Fix a integer overflow in SimplifyCFG's look up table formation logic.

If the width is very large it gets truncated from uint64_t to uint32_t when
passed to TD->fitsInLegalInteger. The truncated value can fit in a register.
This manifested in massive memory usage or crashes (PR13946).

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@164784 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp
index 299c0596da3581b473018161ccd9264e761d3511..065325b7c25e889f4ebb15c2e0ab5870be29aa98 100644 (file)
--- a/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3414,6 +3414,10 @@ bool SwitchLookupTable::WouldFitInRegister(const TargetData *TD,
     return false;
   // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
   // are <= 15, we could try to narrow the type.
+
+  // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
+  if (TableSize >= UINT_MAX/IT->getBitWidth())
+    return false;
   return TD->fitsInLegalInteger(TableSize * IT->getBitWidth());
 }
 

diff --git a/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll b/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
index ef9d51669d3a27d23d43599b6f26598a036bc9db..134ac4eeb1440b0f20a07f192a51c03fcaf0fad3 100644 (file)
--- a/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
+++ b/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
@@ -231,3 +231,41 @@ lor.end:
 ; CHECK-NEXT: %lor.ext = zext i1 %1 to i32
 ; CHECK-NEXT: ret i32 %lor.ext
 }
+
+; PR13946
+define i32 @overflow(i32 %type) nounwind {
+entry:
+  switch i32 %type, label %sw.default [
+    i32 -2147483648, label %sw.bb
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 -2147483645, label %sw.bb3
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb:
+  br label %if.end
+
+sw.bb1:
+  br label %if.end
+
+sw.bb2:
+  br label %if.end
+
+sw.bb3:
+  br label %if.end
+
+sw.default:
+  br label %if.end
+
+if.else:
+  br label %if.end
+
+if.end:
+  %dirent_type.0 = phi i32 [ 3, %sw.default ], [ 6, %sw.bb3 ], [ 5, %sw.bb2 ], [ 0, %sw.bb1 ], [ 3, %sw.bb ], [ 0, %if.else ]
+  ret i32 %dirent_type.0
+; CHECK: define i32 @overflow
+; CHECK: switch
+; CHECK: phi
+}