[llvm] [VPlan] Use SCEV optimize VecTC-materialization for scalable (PR #193703)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 02:36:04 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Ramkumar Ramachandra (artagnon)
<details>
<summary>Changes</summary>
Follow up on 6e4b52 ([VPlan] Use MaxRuntimeStep in materializeVectorTC to simplify middle br, #<!-- -->193067) to optimize for non-constant trip-counts using SCEV.
---
Patch is 485.99 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/193703.diff
46 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+3-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+26-13)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+14-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll (+3-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll (+4-7)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll (+4-7)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll (+4-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll (+80-134)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll (+53-94)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll (+32-50)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll (+14-22)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll (+45-88)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/streaming-compatible-sve-no-maximize-bandwidth.ll (+6-10)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll (+6-15)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll (+29-55)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll (+3-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll (+8-20)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll (+7-24)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll (+5-11)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll (+20-35)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll (+24-52)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll (+31-70)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll (+6-10)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll (+5-8)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll (+5-7)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll (+16-28)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/ordered-reduction.ll (+8-14)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll (+18-36)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-unroll.ll (+8-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll (+12-16)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll (+33-63)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll (+41-74)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll (+30-54)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll (+20-35)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll (+84-150)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll (+20-34)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-intermediate-store.ll (+8-14)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll (+4-7)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll (+4-7)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll (+4-7)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll (+80-146)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll (+4-7)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c6ff52081b136..840fde247522f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7019,11 +7019,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// which may be needed for epilogue vectorization.
VPlanTransforms::removeBranchOnConst(BestVPlan, /*OnlyLatches=*/true);
VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
- std::optional<uint64_t> MaxRuntimeStep;
- if (auto MaxVScale = getMaxVScale(*CM.TheFunction, CM.TTI))
- MaxRuntimeStep = uint64_t(*MaxVScale) * BestVF.getKnownMinValue() * BestUF;
+ uint64_t MaxRuntimeStep = BestVF.getKnownMinValue() * BestUF *
+ getMaxVScale(*CM.TheFunction, CM.TTI).value_or(0);
VPlanTransforms::materializeVectorTripCount(
- BestVPlan, VectorPH, CM.foldTailByMasking(),
+ BestVPlan, PSE, VectorPH, CM.foldTailByMasking(),
CM.requiresScalarEpilogue(BestVF.isVector()), &BestVPlan.getVFxUF(),
MaxRuntimeStep);
VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9b2e3d762fe15..6cc851c03cb50 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5135,9 +5135,8 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
}
void VPlanTransforms::materializeVectorTripCount(
- VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking,
- bool RequiresScalarEpilogue, VPValue *Step,
- std::optional<uint64_t> MaxRuntimeStep) {
+ VPlan &Plan, VPBasicBlock *VectorPHVPBB,
+ bool TailByMasking, bool RequiresScalarEpilogue, VPValue *Step) {
VPSymbolicValue &VectorTC = Plan.getVectorTripCount();
// There's nothing to do if there are no users of the vector trip count or its
// IR value has already been set.
@@ -5155,16 +5154,6 @@ void VPlanTransforms::materializeVectorTripCount(
}
VPBuilder Builder(VectorPHVPBB, InsertPt);
- // For scalable steps, if TC is a constant and is divisible by the maximum
- // possible runtime step, then TC % Step == 0 for all valid vscale values
- // and the vector trip count equals TC directly.
- const APInt *TCVal;
- if (!RequiresScalarEpilogue && match(TC, m_APInt(TCVal)) && MaxRuntimeStep &&
- TCVal->getZExtValue() % *MaxRuntimeStep == 0) {
- VectorTC.replaceAllUsesWith(TC);
- return;
- }
-
// If the tail is to be folded by masking, round the number of iterations N
// up to a multiple of Step instead of rounding down. This is done by first
// adding Step-1 and then rounding down. Note that it's ok if this addition
@@ -5205,6 +5194,30 @@ void VPlanTransforms::materializeVectorTripCount(
VectorTC.replaceAllUsesWith(Res);
}
+void VPlanTransforms::materializeVectorTripCount(
+ VPlan &Plan, PredicatedScalarEvolution &PSE, VPBasicBlock *VectorPHVPBB,
+ bool TailByMasking, bool RequiresScalarEpilogue, VPValue *Step,
+ uint64_t MaxRuntimeStep) {
+ VPSymbolicValue &VectorTC = Plan.getVectorTripCount();
+ if (VectorTC.getNumUsers() == 0 || VectorTC.getUnderlyingValue())
+ return;
+
+ VPValue *TC = Plan.getTripCount();
+ // For scalable steps, if TC is a constant and is divisible by the maximum
+ // possible runtime step, then TC is divisible by Step for all valid
+ // vscale values and the vector trip count equals TC directly.
+ ScalarEvolution *SE = PSE.getSE();
+ SmallVector<const SCEVPredicate *, 2> Assume;
+ if (!RequiresScalarEpilogue && MaxRuntimeStep &&
+ SE->isKnownMultipleOf(vputils::getSCEVExprForVPValue(TC, PSE),
+ MaxRuntimeStep, Assume)) {
+ PSE.addPredicate(SCEVUnionPredicate(Assume, *SE));
+ return VectorTC.replaceAllUsesWith(TC);
+ }
+ materializeVectorTripCount(Plan, VectorPHVPBB, TailByMasking,
+ RequiresScalarEpilogue, Step);
+}
+
void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
ElementCount VFEC) {
// If VF and VFxUF have already been materialized (no remaining users),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 991f7ec5cf4bb..98870d467d667 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -427,12 +427,20 @@ struct VPlanTransforms {
/// Materialize vector trip count computations to a set of VPInstructions.
/// \p Step is used as the step value for the trip count computation.
- /// \p MaxRuntimeStep is the maximum possible runtime value of Step, used to
- /// prove the trip count is divisible by the step for scalable VFs.
- static void materializeVectorTripCount(
- VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking,
- bool RequiresScalarEpilogue, VPValue *Step,
- std::optional<uint64_t> MaxRuntimeStep = std::nullopt);
+ static void materializeVectorTripCount(VPlan &Plan,
+ VPBasicBlock *VectorPHVPBB,
+ bool TailByMasking,
+ bool RequiresScalarEpilogue,
+ VPValue *Step);
+
+ /// A variant that also takes \p MaxRuntimeStep, the maximum possible runtime
+ /// value of \p Step, to optimize when the trip count is divisible by Step for
+ /// scalable VFs. Zero is an invalid value, and disables the optimization.
+ static void
+ materializeVectorTripCount(VPlan &Plan, PredicatedScalarEvolution &PSE,
+ VPBasicBlock *VectorPHVPBB, bool TailByMasking,
+ bool RequiresScalarEpilogue, VPValue *Step,
+ uint64_t MaxRuntimeStep);
/// Materialize the backedge-taken count to be computed explicitly using
/// VPInstructions.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index fd0503a3f41c8..5d9c3c395f915 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -723,8 +723,6 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; DEFAULT-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE27:.*]] ]
@@ -844,10 +842,10 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; DEFAULT: [[PRED_STORE_CONTINUE27]]:
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; DEFAULT-NEXT: [[TMP80:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP80]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; DEFAULT-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP0]]
+; DEFAULT-NEXT: br i1 [[TMP54]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[TMP0]]
; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 607f424d5c5a7..8c882e7e5a73f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -319,8 +319,6 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
-; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -332,10 +330,10 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8
; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP0]]
; COST1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[TMP0]]
; COST1-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
; COST1: [[SCALAR_PH]]:
;
@@ -346,8 +344,6 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
-; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
-; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -357,10 +353,10 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST10-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]]
; COST10-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP0]]
; COST10-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[TMP0]]
; COST10-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
; COST10: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index dfff45df2d9db..6e5984c4a92dd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -13,8 +13,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; TFNONE: vector.ph:
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
-; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]]
; TFNONE: vector.body:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -27,13 +25,12 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
; TFNONE-NEXT: store double [[TMP14]], ptr [[P:%.*]], align 8
; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP0]]
+; TFNONE-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TFNONE: middle.block:
-; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
+; TFNONE-NEXT: br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
; TFNONE: scalar.ph:
-; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; TFNONE-NEXT: br label [[LOOP:%.*]]
; TFNONE: loop:
; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_END:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 0ad66c480a56f..c3f52c1d95d4f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -852,8 +852,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
-; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -866,13 +864,12 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
; TFNONE-NEXT: store double [[TMP14]], ptr [[P]], align 8
; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP0]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
-; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; TFNONE-NEXT: br i1 true, label %[[END:.*]], label %[[SCALAR_PH]]
; TFNONE: [[SCALAR_PH]]:
-; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; TFNONE-NEXT: br label %[[LOOP:.*]]
; TFNONE: [[LOOP]]:
; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
index 9e31403ff75d5..080018fbbca7e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked_ldst_sme.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -50,8 +51,6 @@ define void @wombat(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4, ptr %a
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[ARG6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -75,13 +74,13 @@ define void @wombat(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4, ptr %a
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD30]], [[TMP12]]
; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP5]]), !alias.scope [[META11]], !noalias [[META12]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[ZEXT]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[ZEXT]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[BB24:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[BB7]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[ZEXT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[BB7]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[BB8:.*]]
; CHECK: [[BB8]]:
; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD22:%.*]], %[[BB21:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index e54d1e9ddb1a3..7435b2fcec599 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -17,8 +17,6 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
; CHECK-NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK-NEON: vector.ph:
-; CHECK-NEON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
-; CHECK-NEON-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-NEON-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-NEON: vector.body:
; CHECK-NEON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -38,11 +36,11 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
; CHECK-NEON-NEXT: [[TMP8:%.*]] = sub <16 x i32> zeroinitializer, [[TMP7]]
; CHECK-NEON-NEXT: [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP8]])
; CHECK-NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEON-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEON-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEON-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEON-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-NEON: middle.block:
; CHECK-NEON-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
-; CHECK-NEON-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; C...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/193703
More information about the llvm-commits
mailing list