[llvm] 251ed1e - [VPlan] Optz WideCanIV with SIVSteps over CanIV (#191276)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 08:46:55 PDT 2026
Author: Ramkumar Ramachandra
Date: 2026-04-28T15:46:48Z
New Revision: 251ed1eb8435d4c34163d288a0cba1af29f825ce
URL: https://github.com/llvm/llvm-project/commit/251ed1eb8435d4c34163d288a0cba1af29f825ce
DIFF: https://github.com/llvm/llvm-project/commit/251ed1eb8435d4c34163d288a0cba1af29f825ce.diff
LOG: [VPlan] Optz WideCanIV with SIVSteps over CanIV (#191276)
Replace WideCanonicalIV with a ScalarIVSteps over the CanonicalIV when
only the first lane is used. This is a preparatory step in enabling
expansion of WideCanonicalIV into executable recipes.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
Modified:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5e6d1bbcd5a7c..7eab86e8fd7ca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -655,11 +655,51 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
}
}
+static VPScalarIVStepsRecipe *
+createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
+ Instruction::BinaryOps InductionOpcode,
+ FPMathOperator *FPBinOp, Instruction *TruncI,
+ VPIRValue *StartV, VPValue *Step, DebugLoc DL,
+ VPBuilder &Builder) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
+ Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
+
+ // Truncate base induction if needed.
+ VPTypeAnalysis TypeInfo(Plan);
+ Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
+ if (TruncI) {
+ Type *TruncTy = TruncI->getType();
+ assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
+ BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
+ ResultTy = TruncTy;
+ }
+
+ // Truncate step if needed.
+ Type *StepTy = TypeInfo.inferScalarType(Step);
+ if (ResultTy != StepTy) {
+ assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
+ "Not truncating.");
+ assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
+ auto *VecPreheader =
+ cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(VecPreheader);
+ Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
+ }
+ return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
+ &Plan.getVF(), DL);
+}
+
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
/// recipe, if it exists.
static void removeRedundantCanonicalIVs(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
+ VPRegionValue *CanonicalIV = LoopRegion->getCanonicalIV();
auto *WidenNewIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(CanonicalIV);
if (!WidenNewIV)
@@ -689,6 +729,24 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
return;
}
}
+
+ if (!vputils::onlyFirstLaneUsed(WidenNewIV) && !Plan.hasScalarVFOnly()) {
+ assert(!vputils::onlyScalarValuesUsed(WidenNewIV) &&
+ "Lanes other than first lane being used should imply that not just "
+ "scalars are used");
+ return;
+ }
+
+ // Replace the wide canonical IV with a scalar-iv-steps over the canonical
+ // IV.
+ Type *CanonicalIVTy = LoopRegion->getCanonicalIVType();
+ VPBuilder Builder(WidenNewIV);
+ WidenNewIV->replaceAllUsesWith(createScalarIVSteps(
+ Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
+ nullptr, Plan.getZero(CanonicalIVTy),
+ Plan.getConstantInt(CanonicalIVTy, 1), CanonicalIV->getDebugLoc(),
+ Builder));
+ WidenNewIV->eraseFromParent();
}
/// Returns true if \p R is dead and can be removed.
@@ -739,46 +797,6 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
}
}
-static VPScalarIVStepsRecipe *
-createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
- Instruction::BinaryOps InductionOpcode,
- FPMathOperator *FPBinOp, Instruction *TruncI,
- VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
- VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
- Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
-
- // Truncate base induction if needed.
- VPTypeAnalysis TypeInfo(Plan);
- Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
- if (TruncI) {
- Type *TruncTy = TruncI->getType();
- assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
- BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
- ResultTy = TruncTy;
- }
-
- // Truncate step if needed.
- Type *StepTy = TypeInfo.inferScalarType(Step);
- if (ResultTy != StepTy) {
- assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
- "Not truncating.");
- assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
- auto *VecPreheader =
- cast<VPBasicBlock>(HeaderVPBB->getSingleHierarchicalPredecessor());
- VPBuilder::InsertPointGuard Guard(Builder);
- Builder.setInsertPoint(VecPreheader);
- Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
- }
- return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step,
- &Plan.getVF(), DL);
-}
-
static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
SetVector<VPUser *> Users(llvm::from_range, V->users());
for (unsigned I = 0; I != Users.size(); ++I) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
new file mode 100644
index 0000000000000..20eb447ac1aa6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-counting-down.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
+
+define void @tail_folding_scalable(ptr %a, ptr noalias %b, ptr noalias %c, i32 %n) {
+; CHECK-LABEL: define void @tail_folding_scalable(
+; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[SMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[NEXT_GEP1]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP5]], ptr align 1 [[NEXT_GEP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP2]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %n, %entry ], [ %iv.next, %loop ]
+ %ind.ptr.a = phi ptr [ %a, %entry ], [ %ind.ptr.a.next, %loop ]
+ %ind.ptr.b = phi ptr [ %b, %entry ], [ %ind.ptr.b.next, %loop ]
+ %ind.ptr.c = phi ptr [ %c, %entry ], [ %ind.ptr.c.next, %loop ]
+ %iv.next = add nsw i32 %iv, -1
+ %ind.ptr.a.next = getelementptr inbounds i8, ptr %ind.ptr.a, i32 1
+ %ld.ind.a = load i8, ptr %ind.ptr.a, align 1
+ %ind.ptr.b.next = getelementptr inbounds i8, ptr %ind.ptr.b, i32 1
+ %ld.ind.b = load i8, ptr %ind.ptr.b, align 1
+ %res = add i8 %ld.ind.b, %ld.ind.a
+ %ind.ptr.c.next = getelementptr inbounds i8, ptr %ind.ptr.c, i32 1
+ store i8 %res, ptr %ind.ptr.c, align 1
+ %ec = icmp sgt i32 %iv, 1
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index bd584a22c3d05..1a1fcd6f535b7 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -17,8 +17,8 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
-; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i64 0
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)
+; CHECK-TF: %[[INDEX:.*]] = phi i32
+; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[INDEX]], i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index fbb4ec0ee14d3..861f59906e451 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2720,11 +2720,11 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if3:
+; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue4:
+; UNROLL-NO-VF: pred.udiv.continue3:
; UNROLL-NO-VF-NEXT: [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index b8d6667f965af..1d9fe51c92f16 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -90,11 +90,10 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[VEC_IV]], 14
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
@@ -104,20 +103,20 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK: pred.store.if4:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
+; CHECK: pred.store.continue5:
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.if6:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
+; CHECK: pred.store.continue7:
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.if8:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
More information about the llvm-commits
mailing list