[llvm] [VPlan] Use DerivedIV in convertToStridedAccesses (PR #209611)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 09:53:05 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/209611
>From 1353eb45a4b0c46141dc78896dda28c4ebb722e1 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 3 Aug 2026 17:37:11 +0100
Subject: [PATCH 1/3] [LV] Pre-commit strided-accesses-derivediv test
---
.../RISCV/strided-accesses-derivediv.ll | 106 ++++++++++++++++++
1 file changed, 106 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
new file mode 100644
index 0000000000000..aad7a0a0f32d8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block" --version 6
+; RUN: opt -mtriple=riscv64 -mattr=+v -p loop-vectorize -force-vector-interleave=2 -S %s | FileCheck %s
+
+define void @strided_load_min_profitable_tc(ptr noalias %p1, ptr noalias %p2) {
+; CHECK-LABEL: define void @strided_load_min_profitable_tc(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 30)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
+; CHECK-NEXT: br i1 [[TMP4]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P2]], i64 1021
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[P2]], i64 8168
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP1]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX9]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i32 [[TMP0]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i32 [[TMP5]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 256, [[TMP6]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 256, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[N_VEC]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = shl i16 [[TMP7]], 2
+; CHECK-NEXT: [[TMP9:%.*]] = trunc <vscale x 2 x i32> [[BROADCAST_SPLAT]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP10:%.*]] = shl <vscale x 2 x i16> [[TMP9]], splat (i16 2)
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i16> @llvm.stepvector.nxv2i16()
+; CHECK-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i16> [[TMP11]], splat (i16 4)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i16> [ [[TMP12]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add <vscale x 2 x i16> [[VEC_IND]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = sext <vscale x 2 x i16> [[VEC_IND]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP14:%.*]] = sext <vscale x 2 x i16> [[STEP_ADD]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP15:%.*]] = zext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP16:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 16
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP17]], i64 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP20]], i64 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT: [[TMP23:%.*]] = trunc <vscale x 2 x i32> [[TMP21]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP24:%.*]] = trunc <vscale x 2 x i32> [[TMP22]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr [1 x [1 x i16]], ptr [[P2]], <vscale x 2 x i64> [[TMP13]]
+; CHECK-NEXT: [[WIDE_GEP10:%.*]] = getelementptr [1 x [1 x i16]], ptr [[P2]], <vscale x 2 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i16.nxv2p0(<vscale x 2 x i16> [[TMP23]], <vscale x 2 x ptr> align 2 [[WIDE_GEP]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i16.nxv2p0(<vscale x 2 x i16> [[TMP24]], <vscale x 2 x ptr> align 2 [[WIDE_GEP10]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT: [[TMP25:%.*]] = trunc <vscale x 2 x i32> [[TMP21]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[TMP26:%.*]] = trunc <vscale x 2 x i32> [[TMP22]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr i8, ptr [[P2]], <vscale x 2 x i64> [[TMP13]]
+; CHECK-NEXT: [[WIDE_GEP12:%.*]] = getelementptr i8, ptr [[P2]], <vscale x 2 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP25]], <vscale x 2 x ptr> align 1 [[WIDE_GEP11]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META6:![0-9]+]], !noalias [[META7:![0-9]+]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP26]], <vscale x 2 x ptr> align 1 [[WIDE_GEP12]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META6]], !noalias [[META7]]
+; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr [1 x i64], ptr [[P2]], <vscale x 2 x i64> [[TMP13]]
+; CHECK-NEXT: [[WIDE_GEP14:%.*]] = getelementptr [1 x i64], ptr [[P2]], <vscale x 2 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x ptr> align 8 [[WIDE_GEP13]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META7]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x ptr> align 8 [[WIDE_GEP14]], <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i16> [[STEP_ADD]], [[TMP10]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.ext = sext i16 %iv to i64
+ %gep1 = getelementptr i32, ptr %p1, i64 %iv.ext
+ %v1 = load i32, ptr %gep1, align 4
+ %v1.trunc = trunc i32 %v1 to i16
+ %gep2 = getelementptr [1 x [1 x i16]], ptr %p2, i64 %iv.ext
+ store i16 %v1.trunc, ptr %gep2, align 2
+ %v1.trunc.i8 = trunc i32 %v1 to i8
+ %gep3 = getelementptr i8, ptr %p2, i64 %iv.ext
+ store i8 %v1.trunc.i8, ptr %gep3, align 1
+ %gep4 = getelementptr [1 x i64], ptr %p2, i64 %iv.ext
+ store i64 0, ptr %gep4, align 8
+ %iv.next = add i16 %iv, 4
+ %iv.next.ext = sext i16 %iv.next to i32
+ %ec = icmp ne i32 %iv.next.ext, 1024
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
>From d101acaa0883d50f52d0f837759157446fc29dc1 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 14 Jul 2026 19:38:36 +0100
Subject: [PATCH 2/3] [VPlan] Use DerivedIV in convertToStridedAccesses (NFC)
Introduce BinOpGEPFlags to package up WrapFlagsTy and GEPNoWrapFlags as
one of the choices of the VPIRFlags union, and use it in migrating
convertToStridedAccesses to use DerivedIV. The end result is that the
DerivedIV expanded by convertToConcreteRecipes produces identical IR.
---
.../Vectorize/LoopVectorizationPlanner.h | 8 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 48 +++++++++++++++++--
.../Transforms/Vectorize/VPlanLowering.cpp | 9 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 31 +++++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 31 ++++++------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 2 +-
.../RISCV/strided-accesses-derivediv.ll | 2 +-
8 files changed, 91 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fa317f290022e..0ac0be7b0090a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -387,10 +387,10 @@ class VPBuilder {
}
/// Convert \p Current to \p Start + \p Current * \p Step.
- VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
- FPMathOperator *FPBinOp, VPValue *Start,
- VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {}) {
+ VPDerivedIVRecipe *
+ createDerivedIV(InductionDescriptor::InductionKind Kind,
+ FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current,
+ VPValue *Step, const VPIRFlags::BinOpGEPFlagsTy Flags = {}) {
return tryInsertInstruction(
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Flags));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..8f1751c8adacf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -705,6 +705,7 @@ class VPIRFlags {
enum class OperationType : unsigned char {
Cmp,
FCmp,
+ BinOpGEP,
OverflowingBinOp,
Trunc,
DisjointOp,
@@ -742,6 +743,18 @@ class VPIRFlags {
NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
};
+ /// Holds both the no-wrap flags and GEP no-wrap flags for computations
+ /// involving BinOps and a GEPs.
+ struct BinOpGEPFlagsTy {
+ WrapFlagsTy WrapFlags;
+ uint8_t RawGEPFlags;
+ BinOpGEPFlagsTy() = default;
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(0) {}
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags, GEPNoWrapFlags GEPFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(GEPFlags.getRaw()) {}
+ };
+
private:
struct ExactFlagsTy {
char IsExact : 1;
@@ -792,6 +805,7 @@ class VPIRFlags {
NonNegFlagsTy NonNegFlags;
FastMathFlagsTy FMFs;
FCmpFlagsTy FCmpFlags;
+ BinOpGEPFlagsTy BinOpGEPFlags;
ReductionFlagsTy ReductionFlags;
uint8_t AllFlags[2];
};
@@ -849,6 +863,12 @@ class VPIRFlags {
FCmpFlags.FMFs = FMFs;
}
+ VPIRFlags(BinOpGEPFlagsTy Flags)
+ : OpType(OperationType::BinOpGEP), AllFlags() {
+ BinOpGEPFlags.WrapFlags = Flags.WrapFlags;
+ BinOpGEPFlags.RawGEPFlags = Flags.RawGEPFlags;
+ }
+
VPIRFlags(WrapFlagsTy WrapFlags)
: OpType(OperationType::OverflowingBinOp), AllFlags() {
this->WrapFlags = WrapFlags;
@@ -929,6 +949,11 @@ class VPIRFlags {
case OperationType::NonNegOp:
NonNegFlags.NonNeg = false;
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW = false;
+ BinOpGEPFlags.WrapFlags.HasNSW = false;
+ BinOpGEPFlags.RawGEPFlags = 0;
+ break;
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -972,7 +997,8 @@ class VPIRFlags {
I.setNonNeg(NonNegFlags.NonNeg);
break;
case OperationType::ReductionOp:
- llvm_unreachable("reduction ops should not use applyFlags");
+ case OperationType::BinOpGEP:
+ llvm_unreachable("Reduction and BinOpGEP ops should not use applyFlags");
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -998,7 +1024,17 @@ class VPIRFlags {
}
GEPNoWrapFlags getGEPNoWrapFlags() const {
- return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ assert(
+ is_contained({OperationType::GEPOp, OperationType::BinOpGEP}, OpType) &&
+ "Recipe doesn't have GEP no-wrap flags");
+ switch (OpType) {
+ case OperationType::GEPOp:
+ return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ case OperationType::BinOpGEP:
+ return GEPNoWrapFlags::fromRaw(BinOpGEPFlags.RawGEPFlags);
+ default:
+ return GEPNoWrapFlags::none();
+ }
}
/// Returns true if the recipe has a comparison predicate.
@@ -1023,6 +1059,7 @@ class VPIRFlags {
bool hasNoUnsignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
+ case OperationType::BinOpGEP:
return WrapFlags.HasNUW;
case OperationType::Trunc:
return TruncFlags.HasNUW;
@@ -1034,6 +1071,7 @@ class VPIRFlags {
bool hasNoSignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
+ case OperationType::BinOpGEP:
return WrapFlags.HasNSW;
case OperationType::Trunc:
return TruncFlags.HasNSW;
@@ -1046,6 +1084,7 @@ class VPIRFlags {
switch (OpType) {
case OperationType::OverflowingBinOp:
case OperationType::Trunc:
+ case OperationType::BinOpGEP:
return {hasNoUnsignedWrap(), hasNoSignedWrap()};
default:
return {};
@@ -4182,7 +4221,7 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPValue *Start,
VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {})
+ const VPIRFlags::BinOpGEPFlagsTy &Flags = {})
: VPRecipeWithIRFlags(VPRecipeBase::VPDerivedIVSC, {Start, Current, Step},
Start->getScalarType(), Flags),
Kind(Kind), FPBinOp(FPBinOp) {}
@@ -4191,7 +4230,8 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe *clone() override {
return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
- getStepValue(), getNoWrapFlags());
+ getStepValue(),
+ {getNoWrapFlags(), getGEPNoWrapFlags()});
}
VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 72b6503157d68..8e62607dad04a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -61,7 +61,8 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
nullptr, Plan.getZero(CanIVTy), Plan.getConstantInt(CanIVTy, 1),
WideCanIV->getDebugLoc(), Builder,
- {static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false}));
+ VPIRFlags::WrapFlagsTy(
+ static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false)));
WideCanIV->eraseFromParent();
return;
}
@@ -422,6 +423,7 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
: Builder.createScalarCast(Instruction::SIToFP, Index, StepTy,
DebugLoc::getCompilerGenerated());
VPIRFlags::WrapFlagsTy Flags = R->getNoWrapFlags();
+ GEPNoWrapFlags GEPFlags = R->getGEPNoWrapFlags();
switch (R->getInductionKind()) {
case InductionDescriptor::IK_IntInduction: {
assert(Index->getScalarType() == Start->getScalarType() &&
@@ -432,9 +434,10 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
DebugLoc::getUnknown(), "", Flags));
}
case InductionDescriptor::IK_PtrInduction:
- return R->replaceAllUsesWith(Builder.createPtrAdd(
+ return R->replaceAllUsesWith(Builder.createNoWrapPtrAdd(
Start,
- Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags)));
+ Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags),
+ GEPFlags));
case InductionDescriptor::IK_FpInduction: {
assert(StepTy->isFloatingPointTy() && "Expected FP Step value");
const FPMathOperator *FPBinOp = R->getFPBinOp();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..9cf8480d59a61 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -394,6 +394,11 @@ void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
"Cannot change IsInLoop");
getFMFsRef() = getFastMathFlagsOrNone() & Other.getFastMathFlagsOrNone();
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW &= Other.BinOpGEPFlags.WrapFlags.HasNUW;
+ BinOpGEPFlags.WrapFlags.HasNSW &= Other.BinOpGEPFlags.WrapFlags.HasNSW;
+ BinOpGEPFlags.RawGEPFlags &= Other.BinOpGEPFlags.RawGEPFlags;
+ break;
case OperationType::Other:
break;
}
@@ -2577,6 +2582,12 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
case OperationType::ReductionOp:
return Opcode == VPInstruction::ComputeReductionResult;
+ case OperationType::BinOpGEP:
+ return Opcode == Instruction::GetElementPtr ||
+ Opcode == VPInstruction::PtrAdd ||
+ Opcode == VPInstruction::WidePtrAdd || Opcode == Instruction::Add ||
+ Opcode == Instruction::Sub || Opcode == Instruction::Mul ||
+ Opcode == Instruction::Shl;
case OperationType::Other:
return true;
}
@@ -2704,21 +2715,19 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
if (ExactFlags.IsExact)
O << " exact";
break;
- case OperationType::OverflowingBinOp:
- if (WrapFlags.HasNUW)
- O << " nuw";
- if (WrapFlags.HasNSW)
- O << " nsw";
+ case OperationType::FPMathOp:
+ getFastMathFlagsOrNone().print(O);
break;
+ case OperationType::OverflowingBinOp:
case OperationType::Trunc:
- if (TruncFlags.HasNUW)
+ case OperationType::BinOpGEP:
+ if (hasNoUnsignedWrap())
O << " nuw";
- if (TruncFlags.HasNSW)
+ if (hasNoSignedWrap())
O << " nsw";
- break;
- case OperationType::FPMathOp:
- getFastMathFlagsOrNone().print(O);
- break;
+ if (OpType != OperationType::BinOpGEP)
+ break;
+ [[fallthrough]];
case OperationType::GEPOp: {
GEPNoWrapFlags Flags = getGEPNoWrapFlags();
if (Flags.isInBounds())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 83e23449df5f5..3ce5327062512 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5804,40 +5804,37 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Ctx.invalidateWideningDecision(&LoadR->getIngredient(), VF);
// Get VF as i32 for the vector length operand.
+ VPBuilder PHBuilder(Plan.getVectorPreheader());
if (!I32VF) {
- VPBuilder Builder(Plan.getVectorPreheader());
- I32VF = Builder.createScalarZExtOrTrunc(
+ I32VF = PHBuilder.createScalarZExtOrTrunc(
&Plan.getVF(), Type::getInt32Ty(Plan.getContext()),
DebugLoc::getUnknown());
}
- VPBuilder Builder(LoadR);
// Create the base pointer of strided access.
- // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
- // supports a general VPValue as the start value.
VPValue *StartVPV =
- VPSCEVExpander(Builder, *PSE.getSE(), LoadR->getDebugLoc())
+ VPSCEVExpander(PHBuilder, *PSE.getSE(), LoadR->getDebugLoc())
.tryToExpand(Start);
if (!StartVPV)
StartVPV = VPBuilder(Plan.getEntry()).createExpandSCEV(Start);
+ VPBuilder Builder(LoadR);
VPValue *StrideInBytes = Plan.getOrAddLiveIn(Step->getValue());
Type *IndexTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
assert(IndexTy == StrideInBytes->getScalarType() &&
"Stride type from SCEV must match the index type");
- VPValue *CanIV = Builder.createScalarZExtOrTrunc(
- VectorLoop->getCanonicalIV(), IndexTy, DebugLoc::getUnknown());
- auto *AddRecPtr = cast<SCEVAddRecExpr>(PtrSCEV);
- auto *Offset = Builder.createOverflowingOp(
- Instruction::Mul, {CanIV, StrideInBytes},
- {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
- GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
- ? GEPNoWrapFlags::noUnsignedWrap()
- : GEPNoWrapFlags::none();
- VPValue *BasePtr = Builder.createNoWrapPtrAdd(StartVPV, Offset, NWFlags);
+ // The no-wrap flags and GEP no-wrap flags should be derived from the nuw
+ // on the AddRecPtr.
+ bool AddRecNUW = cast<SCEVAddRecExpr>(PtrSCEV)->hasNoUnsignedWrap();
+ VPIRFlags::WrapFlagsTy WrapFlags = {AddRecNUW, false};
+ GEPNoWrapFlags GEPFlags =
+ AddRecNUW ? GEPNoWrapFlags::noUnsignedWrap() : GEPNoWrapFlags::none();
+ VPDerivedIVRecipe *BasePtr = Builder.createDerivedIV(
+ InductionDescriptor::IK_PtrInduction, nullptr, StartVPV,
+ VectorLoop->getCanonicalIV(), StrideInBytes, {WrapFlags, GEPFlags});
// Create a new vector pointer for strided access.
VPValue *NewPtr = Builder.createVectorPointer(
- BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, NWFlags,
+ BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, GEPFlags,
LoadR->getDebugLoc());
VPValue *Mask = LoadR->getMask();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 93b18b31e9e7d..5988ad5f0b9cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -597,7 +597,7 @@ VPScalarIVStepsRecipe *vputils::createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags) {
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..7d0ea0878613c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -196,7 +196,7 @@ VPScalarIVStepsRecipe *createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags = {});
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags = {});
/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd
/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
index aad7a0a0f32d8..332b521995688 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
@@ -7,7 +7,7 @@ define void @strided_load_min_profitable_tc(ptr noalias %p1, ptr noalias %p2) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 30)
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 20)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
>From 8010b33e9dd5bfe8f5cecbdb21ecf3eaa9f0c656 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 3 Aug 2026 17:48:45 +0100
Subject: [PATCH 3/3] [VPlan] Fix thinko in getting NUW/NSW
---
llvm/lib/Transforms/Vectorize/VPlan.h | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8f1751c8adacf..8ee542bef4687 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1024,16 +1024,13 @@ class VPIRFlags {
}
GEPNoWrapFlags getGEPNoWrapFlags() const {
- assert(
- is_contained({OperationType::GEPOp, OperationType::BinOpGEP}, OpType) &&
- "Recipe doesn't have GEP no-wrap flags");
switch (OpType) {
case OperationType::GEPOp:
return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
case OperationType::BinOpGEP:
return GEPNoWrapFlags::fromRaw(BinOpGEPFlags.RawGEPFlags);
default:
- return GEPNoWrapFlags::none();
+ llvm_unreachable("Recipe doesn't have GEP no-wrap flags");
}
}
@@ -1059,8 +1056,9 @@ class VPIRFlags {
bool hasNoUnsignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
- case OperationType::BinOpGEP:
return WrapFlags.HasNUW;
+ case OperationType::BinOpGEP:
+ return BinOpGEPFlags.WrapFlags.HasNUW;
case OperationType::Trunc:
return TruncFlags.HasNUW;
default:
@@ -1071,8 +1069,9 @@ class VPIRFlags {
bool hasNoSignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
- case OperationType::BinOpGEP:
return WrapFlags.HasNSW;
+ case OperationType::BinOpGEP:
+ return BinOpGEPFlags.WrapFlags.HasNSW;
case OperationType::Trunc:
return TruncFlags.HasNSW;
default:
More information about the llvm-commits
mailing list