[llvm] [LV] Fix the cost of first order recurrence splice (PR #192473)

[via llvm-commits]

llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-vectorizers

Author: Sander de Smalen (sdesmalen-arm)

<details>
<summary>Changes</summary>

The index had the wrong sign (for splice.right, the sign is negative), which meant that it calculates the cost of a splice.left operation. For SVE this makes a difference because a splice.left is lowered using an unpredicated EXT instruction, whereas a splice.right is lowered using a predicated SPLICE instruction, which needs a slightly higher cost.

The change in `reduction-recurrence-costs-sve.ll` happens because the vector loop is now less profitable (higher cost) and therefore requires a higher trip-count to be profitable (hence the extra umax).

---
Full diff: https://github.com/llvm/llvm-project/pull/192473.diff


4 Files Affected:

- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+2-4) 
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+2-5) 
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll (+2-1) 
- (added) llvm/test/Transforms/LoopVectorize/AArch64/splice-cost.ll (+31) 


``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3bf3f599c9828..4ad4731b59be6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6035,12 +6035,10 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
 
     // First-order recurrences are replaced by vector shuffles inside the loop.
     if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
-      SmallVector<int> Mask(VF.getKnownMinValue());
-      std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
       return TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
                                 cast<VectorType>(VectorTy),
-                                cast<VectorType>(VectorTy), Mask, CostKind,
-                                VF.getKnownMinValue() - 1);
+                                cast<VectorType>(VectorTy), {}, CostKind,
+                                -(VF.getKnownMinValue() - 1));
     }
 
     // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cbb44198fe362..24aef9766f3b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1218,14 +1218,11 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
   }
   case VPInstruction::FirstOrderRecurrenceSplice: {
     assert(VF.isVector() && "Scalar FirstOrderRecurrenceSplice?");
-    SmallVector<int> Mask(VF.getKnownMinValue());
-    std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
     Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
-
     return Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
                                   cast<VectorType>(VectorTy),
-                                  cast<VectorType>(VectorTy), Mask,
-                                  Ctx.CostKind, VF.getKnownMinValue() - 1);
+                                  cast<VectorType>(VectorTy), {},
+                                  Ctx.CostKind, -(VF.getKnownMinValue() - 1));
   }
   case VPInstruction::ActiveLaneMask: {
     Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 8179adfec8201..3c88b9213c9a8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -52,7 +52,8 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
 ; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; VSCALEFORTUNING2-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 16)
+; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[UMAX]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VSCALEFORTUNING2:       [[VECTOR_PH]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/splice-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/splice-cost.ll
new file mode 100644
index 0000000000000..f99e61d70404a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/splice-cost.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -passes=loop-vectorize -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; REQUIRES: asserts
+target triple = "aarch64"
+
+; CHECK: Cost of 1 for VF 2: EMIT vp<{{.*}}> = first-order splice
+; CHECK: Cost of 3 for VF vscale x 2: EMIT vp<{{.*}}> = first-order splice
+
+define void @foo(ptr noalias %in, ptr noalias %out, i64 %n) "target-features"="+sve" {
+entry:
+  %load.prev = load i64, ptr %in, align 8
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %prev = phi i64 [ %load.prev, %entry ], [ %load.cur, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx.load = getelementptr inbounds nuw i64, ptr %in, i64 %indvars.iv.next
+  %load.cur = load i64, ptr %arrayidx.load
+  %add = add i64 %load.cur, %prev
+  %arrayidx.store = getelementptr inbounds nuw i64, ptr %out, i64 %indvars.iv
+  store i64 %add, ptr %arrayidx.store
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
+
+for.cond.cleanup:
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}

``````````

</details>


https://github.com/llvm/llvm-project/pull/192473