[llvm] r374698 - [Attributor][FIX] Do not apply h2s for arbitrary mallocs

[Johannes Doerfert via llvm-commits]

Author: jdoerfert
Date: Sat Oct 12 20:54:08 2019
New Revision: 374698

URL: http://llvm.org/viewvc/llvm-project?rev=374698&view=rev
Log:
[Attributor][FIX] Do not apply h2s for arbitrary mallocs

H2S did apply to mallocs of non-constant sizes if the uses were OK. This
is now forbidden through reording of the "good" and "bad" cases in the
conditional.

Modified:
    llvm/trunk/lib/Transforms/IPO/Attributor.cpp
    llvm/trunk/test/Transforms/FunctionAttrs/heap_to_stack.ll

Modified: llvm/trunk/lib/Transforms/IPO/Attributor.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/Attributor.cpp?rev=374698&r1=374697&r2=374698&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/Attributor.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/Attributor.cpp Sat Oct 12 20:54:08 2019
@@ -3620,30 +3620,36 @@ ChangeStatus AAHeapToStackImpl::updateIm
   };
 
   auto MallocCallocCheck = [&](Instruction &I) {
-    if (isMallocLikeFn(&I, TLI)) {
+    if (BadMallocCalls.count(&I))
+      return true;
+
+    bool IsMalloc = isMallocLikeFn(&I, TLI);
+    bool IsCalloc = !IsMalloc && isCallocLikeFn(&I, TLI);
+    if (!IsMalloc && !IsCalloc) {
+      BadMallocCalls.insert(&I);
+      return true;
+    }
+
+    if (IsMalloc) {
       if (auto *Size = dyn_cast<ConstantInt>(I.getOperand(0)))
-        if (!Size->getValue().sle(MaxHeapToStackSize))
-          return true;
-    } else if (isCallocLikeFn(&I, TLI)) {
+        if (Size->getValue().sle(MaxHeapToStackSize))
+          if (UsesCheck(I)) {
+            MallocCalls.insert(&I);
+            return true;
+          }
+    } else if (IsCalloc) {
       bool Overflow = false;
       if (auto *Num = dyn_cast<ConstantInt>(I.getOperand(0)))
         if (auto *Size = dyn_cast<ConstantInt>(I.getOperand(1)))
-          if (!(Size->getValue().umul_ov(Num->getValue(), Overflow))
+          if ((Size->getValue().umul_ov(Num->getValue(), Overflow))
                    .sle(MaxHeapToStackSize))
-            if (!Overflow)
+            if (!Overflow && UsesCheck(I)) {
+              MallocCalls.insert(&I);
               return true;
-    } else {
-      BadMallocCalls.insert(&I);
-      return true;
+            }
     }
 
-    if (BadMallocCalls.count(&I))
-      return true;
-
-    if (UsesCheck(I))
-      MallocCalls.insert(&I);
-    else
-      BadMallocCalls.insert(&I);
+    BadMallocCalls.insert(&I);
     return true;
   };
 

Modified: llvm/trunk/test/Transforms/FunctionAttrs/heap_to_stack.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/FunctionAttrs/heap_to_stack.ll?rev=374698&r1=374697&r2=374698&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/FunctionAttrs/heap_to_stack.ll (original)
+++ llvm/trunk/test/Transforms/FunctionAttrs/heap_to_stack.ll Sat Oct 12 20:54:08 2019
@@ -316,3 +316,8 @@ define void @test14() {
   ; CHECK: tail call void @free(i8* noalias %1)
   ret void
 }
+
+define void @test15(i64 %S) {
+  %1 = tail call noalias i8* @malloc(i64 %S)
+  ret void
+}