[llvm] [VPlan] Optz WideCanIV with SIVSteps over CanIV (PR #191276)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 27 03:21:51 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/191276
>From 75496149ac8579d0c42f913376c463fd16e08523 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 9 Apr 2026 20:04:44 +0100
Subject: [PATCH 1/2] [VPlan] Optz WideCanIV with SIVSteps over CanIV
Improve removeRedundantCanonicalIVs to replace WideCanonicalIV with
CanonicalIV when possible.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 117 ++++++++++--------
.../ARM/tail-folding-counting-down.ll | 3 +-
.../LoopVectorize/first-order-recurrence.ll | 4 +-
.../tail-folding-vectorization-factor-1.ll | 15 ++-
4 files changed, 74 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5e6d1bbcd5a7c..a693c15e98720 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -655,40 +655,91 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
}
}
+static VPScalarIVStepsRecipe *
+createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
+ Instruction::BinaryOps InductionOpcode,
+ FPMathOperator *FPBinOp, Instruction *TruncI,
+ VPIRValue *StartV, VPValue *Step, DebugLoc DL,
+ VPBuilder &Builder) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
+ Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
+
+ // Truncate base induction if needed.
+ VPTypeAnalysis TypeInfo(Plan);
+ Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
+ if (TruncI) {
+ Type *TruncTy = TruncI->getType();
+ assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
+ BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
+ ResultTy = TruncTy;
+ }
+
+ // Truncate step if needed.
+ Type *StepTy = TypeInfo.inferScalarType(Step);
+ if (ResultTy != StepTy) {
+ assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
+ auto *VecPreheader =
+ cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(VecPreheader);
+ Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
+ }
+ return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
+ &Plan.getVF(), DL);
+}
+
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
/// recipe, if it exists.
static void removeRedundantCanonicalIVs(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
- auto *WidenNewIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
-
- if (!WidenNewIV)
+ VPRegionValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ auto *WideCanIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
+ if (!WideCanIV)
return;
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
- auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
-
- if (!WidenOriginalIV || !WidenOriginalIV->isCanonical())
+ auto *WideInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+ if (!WideInd || !WideInd->isCanonical())
continue;
// Replace WidenNewIV with WidenOriginalIV if WidenOriginalIV provides
// everything WidenNewIV's users need. That is, WidenOriginalIV will
// generate a vector phi or all users of WidenNewIV demand the first lane
// only.
- if (Plan.hasScalarVFOnly() ||
- !vputils::onlyScalarValuesUsed(WidenOriginalIV) ||
- vputils::onlyFirstLaneUsed(WidenNewIV)) {
+ if (Plan.hasScalarVFOnly() || !vputils::onlyScalarValuesUsed(WideInd) ||
+ vputils::onlyFirstLaneUsed(WideCanIV)) {
// We are replacing a wide canonical iv with a suitable wide induction.
// This is used to compute header mask, hence all lanes will be used and
// we need to drop wrap flags only applying to lanes guranteed to execute
// in the original scalar loop.
- WidenOriginalIV->dropPoisonGeneratingFlags();
- WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
- WidenNewIV->eraseFromParent();
+ WideInd->dropPoisonGeneratingFlags();
+ WideCanIV->replaceAllUsesWith(WideInd);
+ WideCanIV->eraseFromParent();
return;
}
}
+
+ if (!Plan.hasScalarVFOnly() && !vputils::onlyScalarValuesUsed(WideCanIV))
+ return;
+
+ // Replace the wide canonical IV with a scalar-iv-steps over the canonical
+ // IV.
+ Type *CanonicalIVTy = LoopRegion->getCanonicalIVType();
+ VPBuilder Builder(WideCanIV);
+ WideCanIV->replaceAllUsesWith(createScalarIVSteps(
+ Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
+ nullptr, Plan.getZero(CanonicalIVTy),
+ Plan.getConstantInt(CanonicalIVTy, 1), CanonicalIV->getDebugLoc(),
+ Builder));
+ WideCanIV->eraseFromParent();
}
/// Returns true if \p R is dead and can be removed.
@@ -739,46 +790,6 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
}
}
-static VPScalarIVStepsRecipe *
-createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
- Instruction::BinaryOps InductionOpcode,
- FPMathOperator *FPBinOp, Instruction *TruncI,
- VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
- VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
- Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
-
- // Truncate base induction if needed.
- VPTypeAnalysis TypeInfo(Plan);
- Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
- if (TruncI) {
- Type *TruncTy = TruncI->getType();
- assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
- BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
- ResultTy = TruncTy;
- }
-
- // Truncate step if needed.
- Type *StepTy = TypeInfo.inferScalarType(Step);
- if (ResultTy != StepTy) {
- assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
- auto *VecPreheader =
- cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
- VPBuilder::InsertPointGuard Guard(Builder);
- Builder.setInsertPoint(VecPreheader);
- Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
- }
- return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
- &Plan.getVF(), DL);
-}
-
static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
SetVector<VPUser *> Users(llvm::from_range, V->users());
for (unsigned I = 0; I != Users.size(); ++I) {
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index bd584a22c3d05..7743de65a72b3 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -17,8 +17,7 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
-; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i64 0
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)
+; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %{{.*}}, i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index fbb4ec0ee14d3..861f59906e451 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2720,11 +2720,11 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if3:
+; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue4:
+; UNROLL-NO-VF: pred.udiv.continue3:
; UNROLL-NO-VF-NEXT: [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index b8d6667f965af..1d9fe51c92f16 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -90,11 +90,10 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[VEC_IV]], 14
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
@@ -104,20 +103,20 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK: pred.store.if4:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
+; CHECK: pred.store.continue5:
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.if6:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
+; CHECK: pred.store.continue7:
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.if8:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
>From 365a69cd8a30975d677e9e88f5ef742afb7ce494 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 27 Apr 2026 11:15:24 +0100
Subject: [PATCH 2/2] [VPlan] Minimize diff for now, update a test
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 29 ++++++++++---------
.../ARM/tail-folding-counting-down.ll | 3 +-
2 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a693c15e98720..637fad509249a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -700,46 +700,49 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
static void removeRedundantCanonicalIVs(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPRegionValue *CanonicalIV = LoopRegion->getCanonicalIV();
- auto *WideCanIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
- if (!WideCanIV)
+ auto *WidenNewIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
+
+ if (!WidenNewIV)
return;
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
- auto *WideInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
- if (!WideInd || !WideInd->isCanonical())
+ auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+
+ if (!WidenOriginalIV || !WidenOriginalIV->isCanonical())
continue;
// Replace WidenNewIV with WidenOriginalIV if WidenOriginalIV provides
// everything WidenNewIV's users need. That is, WidenOriginalIV will
// generate a vector phi or all users of WidenNewIV demand the first lane
// only.
- if (Plan.hasScalarVFOnly() || !vputils::onlyScalarValuesUsed(WideInd) ||
- vputils::onlyFirstLaneUsed(WideCanIV)) {
+ if (Plan.hasScalarVFOnly() ||
+ !vputils::onlyScalarValuesUsed(WidenOriginalIV) ||
+ vputils::onlyFirstLaneUsed(WidenNewIV)) {
// We are replacing a wide canonical iv with a suitable wide induction.
// This is used to compute header mask, hence all lanes will be used and
// we need to drop wrap flags only applying to lanes guranteed to execute
// in the original scalar loop.
- WideInd->dropPoisonGeneratingFlags();
- WideCanIV->replaceAllUsesWith(WideInd);
- WideCanIV->eraseFromParent();
+ WidenOriginalIV->dropPoisonGeneratingFlags();
+ WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
+ WidenNewIV->eraseFromParent();
return;
}
}
- if (!Plan.hasScalarVFOnly() && !vputils::onlyScalarValuesUsed(WideCanIV))
+ if (!Plan.hasScalarVFOnly() && !vputils::onlyScalarValuesUsed(WidenNewIV))
return;
// Replace the wide canonical IV with a scalar-iv-steps over the canonical
// IV.
Type *CanonicalIVTy = LoopRegion->getCanonicalIVType();
- VPBuilder Builder(WideCanIV);
- WideCanIV->replaceAllUsesWith(createScalarIVSteps(
+ VPBuilder Builder(WidenNewIV);
+ WidenNewIV->replaceAllUsesWith(createScalarIVSteps(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
nullptr, Plan.getZero(CanonicalIVTy),
Plan.getConstantInt(CanonicalIVTy, 1), CanonicalIV->getDebugLoc(),
Builder));
- WideCanIV->eraseFromParent();
+ WidenNewIV->eraseFromParent();
}
/// Returns true if \p R is dead and can be removed.
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index 7743de65a72b3..1a1fcd6f535b7 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -17,7 +17,8 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %{{.*}}, i32 %N)
+; CHECK-TF: %[[INDEX:.*]] = phi i32
+; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[INDEX]], i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
More information about the llvm-commits
mailing list