[llvm] [VPlan] Optz WideCanIV with SIVSteps over CanIV (PR #191276)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 01:13:57 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/191276
>From 4a742c98919cd434b1f546a60a2c43187aab40f8 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 9 Apr 2026 20:04:44 +0100
Subject: [PATCH 1/4] [VPlan] Optz WideCanIV with SIVSteps over CanIV
Improve removeRedundantCanonicalIVs to replace WideCanonicalIV with
CanonicalIV when possible.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 96 +++++++++++--------
.../ARM/tail-folding-counting-down.ll | 4 +-
.../LoopVectorize/first-order-recurrence.ll | 4 +-
.../tail-folding-vectorization-factor-1.ll | 15 ++-
4 files changed, 66 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5e6d1bbcd5a7c..637fad509249a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -655,11 +655,51 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
}
}
+static VPScalarIVStepsRecipe *
+createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
+ Instruction::BinaryOps InductionOpcode,
+ FPMathOperator *FPBinOp, Instruction *TruncI,
+ VPIRValue *StartV, VPValue *Step, DebugLoc DL,
+ VPBuilder &Builder) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
+ Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
+
+ // Truncate base induction if needed.
+ VPTypeAnalysis TypeInfo(Plan);
+ Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
+ if (TruncI) {
+ Type *TruncTy = TruncI->getType();
+ assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
+ BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
+ ResultTy = TruncTy;
+ }
+
+ // Truncate step if needed.
+ Type *StepTy = TypeInfo.inferScalarType(Step);
+ if (ResultTy != StepTy) {
+ assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
+ auto *VecPreheader =
+ cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(VecPreheader);
+ Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
+ }
+ return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
+ &Plan.getVF(), DL);
+}
+
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
/// recipe, if it exists.
static void removeRedundantCanonicalIVs(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ VPRegionValue *CanonicalIV = LoopRegion->getCanonicalIV();
auto *WidenNewIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
if (!WidenNewIV)
@@ -689,6 +729,20 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
return;
}
}
+
+ if (!Plan.hasScalarVFOnly() && !vputils::onlyScalarValuesUsed(WidenNewIV))
+ return;
+
+ // Replace the wide canonical IV with a scalar-iv-steps over the canonical
+ // IV.
+ Type *CanonicalIVTy = LoopRegion->getCanonicalIVType();
+ VPBuilder Builder(WidenNewIV);
+ WidenNewIV->replaceAllUsesWith(createScalarIVSteps(
+ Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
+ nullptr, Plan.getZero(CanonicalIVTy),
+ Plan.getConstantInt(CanonicalIVTy, 1), CanonicalIV->getDebugLoc(),
+ Builder));
+ WidenNewIV->eraseFromParent();
}
/// Returns true if \p R is dead and can be removed.
@@ -739,46 +793,6 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
}
}
-static VPScalarIVStepsRecipe *
-createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
- Instruction::BinaryOps InductionOpcode,
- FPMathOperator *FPBinOp, Instruction *TruncI,
- VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
- VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
- Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
-
- // Truncate base induction if needed.
- VPTypeAnalysis TypeInfo(Plan);
- Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
- if (TruncI) {
- Type *TruncTy = TruncI->getType();
- assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
- BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
- ResultTy = TruncTy;
- }
-
- // Truncate step if needed.
- Type *StepTy = TypeInfo.inferScalarType(Step);
- if (ResultTy != StepTy) {
- assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
- auto *VecPreheader =
- cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
- VPBuilder::InsertPointGuard Guard(Builder);
- Builder.setInsertPoint(VecPreheader);
- Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
- }
- return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
- &Plan.getVF(), DL);
-}
-
static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
SetVector<VPUser *> Users(llvm::from_range, V->users());
for (unsigned I = 0; I != Users.size(); ++I) {
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index bd584a22c3d05..1a1fcd6f535b7 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -17,8 +17,8 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
-; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i64 0
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)
+; CHECK-TF: %[[INDEX:.*]] = phi i32
+; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[INDEX]], i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index fbb4ec0ee14d3..861f59906e451 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2720,11 +2720,11 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if3:
+; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue4:
+; UNROLL-NO-VF: pred.udiv.continue3:
; UNROLL-NO-VF-NEXT: [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index b8d6667f965af..1d9fe51c92f16 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -90,11 +90,10 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[VEC_IV]], 14
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
@@ -104,20 +103,20 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK: pred.store.if4:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
+; CHECK: pred.store.continue5:
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.if6:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
+; CHECK: pred.store.continue7:
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.if8:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
>From 5d39bfda5a4761b7e3bf7aa0a7dc3574270cc9e6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 27 Apr 2026 17:35:29 +0100
Subject: [PATCH 2/4] [VPlan] Fix an edge case along with scalable-TF test
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
.../RISCV/tail-folding-counting-down.ll | 66 +++++++++++++++++++
.../LoopVectorize/first-order-recurrence.ll | 4 +-
3 files changed, 69 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 637fad509249a..42a04eb234fc0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -730,7 +730,7 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
}
}
- if (!Plan.hasScalarVFOnly() && !vputils::onlyScalarValuesUsed(WidenNewIV))
+ if (!vputils::onlyFirstLaneUsed(WidenNewIV))
return;
// Replace the wide canonical IV with a scalar-iv-steps over the canonical
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
new file mode 100644
index 0000000000000..bf4eab92ee7bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
+
+define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %N) {
+; CHECK-LABEL: define void @tail_folding_scalable(
+; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP1]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 16 x i8> [[VP_OP_LOAD3]], [[VP_OP_LOAD]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[WHILE_END]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp5 = icmp sgt i32 %N, 0
+ br i1 %cmp5, label %while.body.preheader, label %while.end
+
+while.body.preheader:
+ br label %while.body
+
+while.body:
+ %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
+ %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
+ %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
+ %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
+ %dec = add nsw i32 %N.addr.09, -1
+ %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
+ %0 = load i8, ptr %a.addr.06, align 1
+ %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
+ %1 = load i8, ptr %b.addr.07, align 1
+ %add = add i8 %1, %0
+ %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
+ store i8 %add, ptr %c.addr.08, align 1
+ %cmp = icmp sgt i32 %N.addr.09, 1
+ br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:
+ br label %while.end
+
+while.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 861f59906e451..fbb4ec0ee14d3 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2720,11 +2720,11 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if2:
+; UNROLL-NO-VF: pred.udiv.if3:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue3:
+; UNROLL-NO-VF: pred.udiv.continue4:
; UNROLL-NO-VF-NEXT: [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
>From f782dfe310e9506f2271181decb90e5ae1988346 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 28 Apr 2026 08:54:18 +0100
Subject: [PATCH 3/4] [VPlan] Move test under AArch64, add assert
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +-
.../AArch64/tail-folding-counting-down.ll | 64 ++++++++++++++++++
.../RISCV/tail-folding-counting-down.ll | 66 -------------------
3 files changed, 69 insertions(+), 67 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 42a04eb234fc0..1f921a5f220dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -730,8 +730,12 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
}
}
- if (!vputils::onlyFirstLaneUsed(WidenNewIV))
+ if (!vputils::onlyFirstLaneUsed(WidenNewIV)) {
+ assert(!vputils::onlyScalarValuesUsed(WidenNewIV) &&
+ "Lanes other than first lane being used should imply that not just "
+ "scalars are used");
return;
+ }
// Replace the wide canonical IV with a scalar-iv-steps over the canonical
// IV.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
new file mode 100644
index 0000000000000..2ac223130467a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
+
+define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %N) {
+; CHECK-LABEL: define void @tail_folding_scalable(
+; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[PH:.*]]
+; CHECK: [[PH]]:
+; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[SMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP1]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP5]], ptr align 1 [[NEXT_GEP]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP2]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %ph
+
+ph:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %iv.next, %loop ], [ %N, %ph ]
+ %ind.ptr.c = phi ptr [ %ind.ptr.a.next, %loop ], [ %c, %ph ]
+ %ind.ptr.b = phi ptr [ %ind.ptr.b.next, %loop ], [ %b, %ph ]
+ %ind.ptr.a = phi ptr [ %ind.ptr.c.next, %loop ], [ %a, %ph ]
+ %iv.next = add nsw i32 %iv, -1
+ %ind.ptr.c.next = getelementptr inbounds i8, ptr %ind.ptr.a, i32 1
+ %ld.ind.a = load i8, ptr %ind.ptr.a, align 1
+ %ind.ptr.b.next = getelementptr inbounds i8, ptr %ind.ptr.b, i32 1
+ %ld.ind.b = load i8, ptr %ind.ptr.b, align 1
+ %res = add i8 %ld.ind.b, %ld.ind.a
+ %ind.ptr.a.next = getelementptr inbounds i8, ptr %ind.ptr.c, i32 1
+ store i8 %res, ptr %ind.ptr.c, align 1
+ %ec = icmp sgt i32 %iv, 1
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
deleted file mode 100644
index bf4eab92ee7bd..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-counting-down.ll
+++ /dev/null
@@ -1,66 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
-
-define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %N) {
-; CHECK-LABEL: define void @tail_folding_scalable(
-; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
-; CHECK: [[WHILE_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP1]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 16 x i8> [[VP_OP_LOAD3]], [[VP_OP_LOAD]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
-; CHECK: [[WHILE_END_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[WHILE_END]]
-; CHECK: [[WHILE_END]]:
-; CHECK-NEXT: ret void
-;
-entry:
- %cmp5 = icmp sgt i32 %N, 0
- br i1 %cmp5, label %while.body.preheader, label %while.end
-
-while.body.preheader:
- br label %while.body
-
-while.body:
- %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
- %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
- %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
- %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
- %dec = add nsw i32 %N.addr.09, -1
- %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
- %0 = load i8, ptr %a.addr.06, align 1
- %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
- %1 = load i8, ptr %b.addr.07, align 1
- %add = add i8 %1, %0
- %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
- store i8 %add, ptr %c.addr.08, align 1
- %cmp = icmp sgt i32 %N.addr.09, 1
- br i1 %cmp, label %while.body, label %while.end.loopexit
-
-while.end.loopexit:
- br label %while.end
-
-while.end:
- ret void
-}
>From 1c85c2b7121f1a30b22a554f9897a421edb049df Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 28 Apr 2026 09:12:41 +0100
Subject: [PATCH 4/4] [LV/test] Fix a|b|c confusion
---
.../AArch64/tail-folding-counting-down.ll | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
index 2ac223130467a..d866802f9ba6d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
-define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %N) {
+define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %n) {
; CHECK-LABEL: define void @tail_folding_scalable(
; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -20,9 +20,9 @@ define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP1]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -44,17 +44,17 @@ ph:
br label %loop
loop:
- %iv = phi i32 [ %iv.next, %loop ], [ %N, %ph ]
- %ind.ptr.c = phi ptr [ %ind.ptr.a.next, %loop ], [ %c, %ph ]
+ %iv = phi i32 [ %iv.next, %loop ], [ %n, %ph ]
+ %ind.ptr.a = phi ptr [ %ind.ptr.a.next, %loop ], [ %a, %ph ]
%ind.ptr.b = phi ptr [ %ind.ptr.b.next, %loop ], [ %b, %ph ]
- %ind.ptr.a = phi ptr [ %ind.ptr.c.next, %loop ], [ %a, %ph ]
+ %ind.ptr.c = phi ptr [ %ind.ptr.c.next, %loop ], [ %c, %ph ]
%iv.next = add nsw i32 %iv, -1
- %ind.ptr.c.next = getelementptr inbounds i8, ptr %ind.ptr.a, i32 1
+ %ind.ptr.a.next = getelementptr inbounds i8, ptr %ind.ptr.a, i32 1
%ld.ind.a = load i8, ptr %ind.ptr.a, align 1
%ind.ptr.b.next = getelementptr inbounds i8, ptr %ind.ptr.b, i32 1
%ld.ind.b = load i8, ptr %ind.ptr.b, align 1
%res = add i8 %ld.ind.b, %ld.ind.a
- %ind.ptr.a.next = getelementptr inbounds i8, ptr %ind.ptr.c, i32 1
+ %ind.ptr.c.next = getelementptr inbounds i8, ptr %ind.ptr.c, i32 1
store i8 %res, ptr %ind.ptr.c, align 1
%ec = icmp sgt i32 %iv, 1
br i1 %ec, label %loop, label %exit
More information about the llvm-commits
mailing list