[llvm] [VPlan] Preserve WideIVStep wrap-flags (PR #191284)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 04:13:41 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/191284
>From ba09b64595575d8dc8581ca5bed2c5b2cfbe4774 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 9 Apr 2026 21:03:18 +0100
Subject: [PATCH 1/4] [VPlan] Fix WideIVStep VectorStep wrap-flags
Extend WideIVStep to carry wrap-flags, and inherit it from the
IntOrFpInduction in the final mul between the vector-step and the
scalar-step when expanding it.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 6 ++++--
.../LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll | 2 +-
.../test/Transforms/LoopVectorize/RISCV/strided-accesses.ll | 2 +-
llvm/test/Transforms/LoopVectorize/induction.ll | 6 +++---
6 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a2445b3dac77d..2d12dc4b2f2b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2158,7 +2158,8 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
case OperationType::OverflowingBinOp:
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
Opcode == Instruction::Mul || Opcode == Instruction::Shl ||
- Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
+ Opcode == VPInstruction::CanonicalIVIncrementForPart ||
+ Opcode == VPInstruction::WideIVStep;
case OperationType::Trunc:
return Opcode == Instruction::Trunc;
case OperationType::DisjointOp:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 126937bf32cb9..da5fc1d0008e0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4197,7 +4197,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
Flags = VPI->getFastMathFlags();
} else {
MulOpc = Instruction::Mul;
- Flags = VPIRFlags::getDefaultFlags(MulOpc);
+ Flags = VPI->hasNoWrapFlags() ? VPI->getNoWrapFlags()
+ : VPIRFlags::getDefaultFlags(MulOpc);
}
VPInstruction *Mul = Builder.createNaryOp(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index df4ca4fe89355..d116c3c902440 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -182,11 +182,13 @@ void UnrollState::unrollWidenInductionByUF(
auto &ID = IV->getInductionDescriptor();
FastMathFlags FMF;
VPIRFlags::WrapFlagsTy WrapFlags(false, false);
+ VPIRFlags Flags;
if (auto *IntOrFPInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(IV)) {
if (IntOrFPInd->hasFastMathFlags())
FMF = IntOrFPInd->getFastMathFlags();
if (IntOrFPInd->hasNoWrapFlags())
WrapFlags = IntOrFPInd->getNoWrapFlags();
+ Flags = *IntOrFPInd;
}
VPValue *ScalarStep = IV->getStepValue();
@@ -194,8 +196,8 @@ void UnrollState::unrollWidenInductionByUF(
Type *VectorStepTy =
IVTy->isPointerTy() ? TypeInfo.inferScalarType(ScalarStep) : IVTy;
VPInstruction *VectorStep = Builder.createNaryOp(
- VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, VectorStepTy, FMF,
- IV->getDebugLoc());
+ VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, VectorStepTy,
+ Flags, IV->getDebugLoc());
ToSkip.insert(VectorStep);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
index af75087bd76fe..aca101af87590 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -293,7 +293,7 @@ define void @test_invar_gep_var_start_step_2(i64 %start, ptr %dst) #0 {
; IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; IC2-NEXT: [[TMP10:%.*]] = mul i64 [[N_VEC]], 2
; IC2-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[TMP10]]
-; IC2-NEXT: [[TMP13:%.*]] = shl <vscale x 4 x i64> [[BROADCAST_SPLAT1]], splat (i64 1)
+; IC2-NEXT: [[TMP13:%.*]] = shl nsw <vscale x 4 x i64> [[BROADCAST_SPLAT1]], splat (i64 1)
; IC2-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[START]], i64 0
; IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 567f54cd3bf8f..c9912cd11d90b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -149,7 +149,7 @@ define void @single_constant_stride_int_iv(ptr %p) {
; CHECK-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
; CHECK-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; CHECK-UF2-NEXT: [[TMP5:%.*]] = mul i64 [[N_VEC]], 64
-; CHECK-UF2-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i64> [[BROADCAST_SPLAT]], splat (i64 6)
+; CHECK-UF2-NEXT: [[TMP6:%.*]] = shl nuw nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], splat (i64 6)
; CHECK-UF2-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-UF2-NEXT: [[TMP8:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP7]], splat (i64 64)
; CHECK-UF2-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index ff72e0d428d64..ed4781f6420b3 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -6034,7 +6034,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
; UNROLL-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
; UNROLL-NEXT: [[IND_END:%.*]] = mul i32 [[STEP]], [[DOTCAST]]
-; UNROLL-NEXT: [[TMP16:%.*]] = shl <2 x i32> [[DOTSPLAT]], splat (i32 1)
+; UNROLL-NEXT: [[TMP16:%.*]] = shl nsw <2 x i32> [[DOTSPLAT]], splat (i32 1)
; UNROLL-NEXT: [[TMP17:%.*]] = mul nuw nsw <2 x i32> [[DOTSPLAT]], <i32 0, i32 1>
; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
; UNROLL: vector.body:
@@ -6110,7 +6110,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i32 [[DOTCAST]], [[STEP]]
-; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = mul <2 x i32> splat (i32 2), [[BROADCAST_SPLAT]]
+; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = mul nsw <2 x i32> splat (i32 2), [[BROADCAST_SPLAT]]
; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = mul nsw <2 x i32> <i32 0, i32 1>, [[BROADCAST_SPLAT]]
; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]]
; UNROLL-NO-IC: vector.body:
@@ -6183,7 +6183,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
; INTERLEAVE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
; INTERLEAVE-NEXT: [[IND_END:%.*]] = mul i32 [[STEP]], [[DOTCAST]]
-; INTERLEAVE-NEXT: [[TMP16:%.*]] = shl <4 x i32> [[DOTSPLAT]], splat (i32 2)
+; INTERLEAVE-NEXT: [[TMP16:%.*]] = shl nsw <4 x i32> [[DOTSPLAT]], splat (i32 2)
; INTERLEAVE-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3>
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
>From 381e5c3cb69fa737a1b85e964c50d8c1d7811f1c Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 17 Apr 2026 10:43:04 +0100
Subject: [PATCH 2/4] [VPlanUnroll] Avoid duplicate Flags vars (NFC)
---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d116c3c902440..974ec4116186d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -180,16 +180,9 @@ void UnrollState::unrollWidenInductionByUF(
IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor());
Type *IVTy = TypeInfo.inferScalarType(IV);
auto &ID = IV->getInductionDescriptor();
- FastMathFlags FMF;
- VPIRFlags::WrapFlagsTy WrapFlags(false, false);
VPIRFlags Flags;
- if (auto *IntOrFPInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(IV)) {
- if (IntOrFPInd->hasFastMathFlags())
- FMF = IntOrFPInd->getFastMathFlags();
- if (IntOrFPInd->hasNoWrapFlags())
- WrapFlags = IntOrFPInd->getNoWrapFlags();
+ if (auto *IntOrFPInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(IV))
Flags = *IntOrFPInd;
- }
VPValue *ScalarStep = IV->getStepValue();
VPBuilder Builder(PH);
@@ -216,16 +209,14 @@ void UnrollState::unrollWidenInductionByUF(
VPValue *Prev = IV;
Builder.setInsertPoint(IV->getParent(), InsertPtForPhi);
unsigned AddOpc;
- VPIRFlags AddFlags;
+ VPIRFlags AddFlags = Flags;
if (IVTy->isPointerTy()) {
AddOpc = VPInstruction::WidePtrAdd;
AddFlags = GEPNoWrapFlags::none();
} else if (IVTy->isFloatingPointTy()) {
AddOpc = ID.getInductionOpcode();
- AddFlags = FMF;
} else {
AddOpc = Instruction::Add;
- AddFlags = WrapFlags;
if (cast<VPWidenIntOrFpInductionRecipe>(IV)->isCanonical())
AddFlags = VPIRFlags::WrapFlagsTy(/*NUW=*/true, /*NSW=*/false);
}
>From 912fe835b79127ea80c480995c1d95e42a69dfc0 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 17 Apr 2026 11:17:26 +0100
Subject: [PATCH 3/4] [VPlan] Variable naming nit
---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 974ec4116186d..194bc33acef18 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -180,9 +180,9 @@ void UnrollState::unrollWidenInductionByUF(
IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor());
Type *IVTy = TypeInfo.inferScalarType(IV);
auto &ID = IV->getInductionDescriptor();
- VPIRFlags Flags;
+ VPIRFlags IVFlags;
if (auto *IntOrFPInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(IV))
- Flags = *IntOrFPInd;
+ IVFlags = *IntOrFPInd;
VPValue *ScalarStep = IV->getStepValue();
VPBuilder Builder(PH);
@@ -190,7 +190,7 @@ void UnrollState::unrollWidenInductionByUF(
IVTy->isPointerTy() ? TypeInfo.inferScalarType(ScalarStep) : IVTy;
VPInstruction *VectorStep = Builder.createNaryOp(
VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, VectorStepTy,
- Flags, IV->getDebugLoc());
+ IVFlags, IV->getDebugLoc());
ToSkip.insert(VectorStep);
@@ -209,7 +209,7 @@ void UnrollState::unrollWidenInductionByUF(
VPValue *Prev = IV;
Builder.setInsertPoint(IV->getParent(), InsertPtForPhi);
unsigned AddOpc;
- VPIRFlags AddFlags = Flags;
+ VPIRFlags AddFlags = IVFlags;
if (IVTy->isPointerTy()) {
AddOpc = VPInstruction::WidePtrAdd;
AddFlags = GEPNoWrapFlags::none();
>From fdb348afdbf2010792c376b5523ea1659685e3b2 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 17 Apr 2026 12:12:52 +0100
Subject: [PATCH 4/4] [VPlanUnroll] Overwrite IVFlags, elim AddFlags
---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 194bc33acef18..1968be2e8966b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -209,28 +209,26 @@ void UnrollState::unrollWidenInductionByUF(
VPValue *Prev = IV;
Builder.setInsertPoint(IV->getParent(), InsertPtForPhi);
unsigned AddOpc;
- VPIRFlags AddFlags = IVFlags;
if (IVTy->isPointerTy()) {
AddOpc = VPInstruction::WidePtrAdd;
- AddFlags = GEPNoWrapFlags::none();
+ IVFlags = GEPNoWrapFlags::none();
} else if (IVTy->isFloatingPointTy()) {
AddOpc = ID.getInductionOpcode();
} else {
AddOpc = Instruction::Add;
if (cast<VPWidenIntOrFpInductionRecipe>(IV)->isCanonical())
- AddFlags = VPIRFlags::WrapFlagsTy(/*NUW=*/true, /*NSW=*/false);
+ IVFlags = VPIRFlags::WrapFlagsTy(/*NUW=*/true, /*NSW=*/false);
}
for (unsigned Part = 1; Part != UF; ++Part) {
std::string Name =
Part > 1 ? "step.add." + std::to_string(Part) : "step.add";
- VPInstruction *Add =
- Builder.createNaryOp(AddOpc,
- {
- Prev,
- VectorStep,
- },
- AddFlags, IV->getDebugLoc(), Name);
+ VPInstruction *Add = Builder.createNaryOp(AddOpc,
+ {
+ Prev,
+ VectorStep,
+ },
+ IVFlags, IV->getDebugLoc(), Name);
ToSkip.insert(Add);
addRecipeForPart(IV, Add, Part);
Prev = Add;
More information about the llvm-commits
mailing list