[llvm] 2c8836c - [LV] Don't consider predicated insts as invariant unconditionally in CM.

[Florian Hahn via llvm-commits]

Author: Florian Hahn
Date: 2024-09-29T20:31:24+01:00
New Revision: 2c8836c899015cce49a399a6bc47b260a24a22e7

URL: https://github.com/llvm/llvm-project/commit/2c8836c899015cce49a399a6bc47b260a24a22e7
DIFF: https://github.com/llvm/llvm-project/commit/2c8836c899015cce49a399a6bc47b260a24a22e7.diff

LOG: [LV] Don't consider predicated insts as invariant unconditionally in CM.

Predicated instructions cannot hoisted trivially, so don't treat them as
uniform value in the cost model.

This fixes a difference between legacy and VPlan-based cost model.

Fixes https://github.com/llvm/llvm-project/issues/110295.

Added: 
    llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll

Modified: 
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cb346be8ffe5e2..792e0e17dd8719 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6539,8 +6539,16 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
       Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
     }
     auto Op2Info = TTI.getOperandInfo(Op2);
-    if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
-        Legal->isInvariant(Op2))
+    auto IsInvariant = [this](Value *Op) {
+      if (!Legal->isInvariant(Op))
+        return false;
+      // Consider Op2 invariant, if it is not a predicated instruction in the
+      // loop. In that case, it is not trivially hoistable.
+      return !isa<Instruction>(Op) ||
+             !TheLoop->contains(cast<Instruction>(Op)) ||
+             !isPredicatedInst(cast<Instruction>(Op));
+    };
+    if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue && IsInvariant(Op2))
       Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
 
     SmallVector<const Value *, 4> Operands(I->operand_values());

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
new file mode 100644
index 00000000000000..0072dd95bd0983
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Test case for https://github.com/llvm/llvm-project/issues/110295.
+define void @predicated_urem_shl_cost(ptr %A, i32 %x, i1 %c) {
+; CHECK-LABEL: define void @predicated_urem_shl_cost(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[X:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 2, [[X]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[L]], [[REM]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[SHL]], %[[THEN]] ]
+; CHECK-NEXT:    store i32 [[P]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 0
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %A, i32 %iv
+  %l = load i32, ptr %gep
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %rem = urem i32 2, %x
+  %shl = shl i32 %l, %rem
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i32 [ 0, %loop.header ], [ %shl, %then ]
+  store i32 %p, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv, 0
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}