[llvm] [VPlan] Use DerivedIV in convertToStridedAccesses (PR #209611)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 03:13:46 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/209611
>From 147322c996e17fddc39e445d51c7bbb9a067bd17 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 3 Aug 2026 17:37:11 +0100
Subject: [PATCH 1/2] [LV] Pre-commit strided-accesses-derivediv test
---
.../RISCV/strided-accesses-derivediv.ll | 101 ++++++++++++++++++
1 file changed, 101 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
new file mode 100644
index 0000000000000..03d6fdaa37270
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-derivediv.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block" --version 6
+; RUN: opt -mtriple=riscv64 -mattr=+v -p loop-vectorize -force-vector-interleave=2 -S %s | FileCheck %s
+
+define void @strided_load_min_profitable_tc(ptr noalias %p1, ptr noalias %p2) {
+; CHECK-LABEL: define void @strided_load_min_profitable_tc(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 18)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
+; CHECK-NEXT: br i1 [[TMP4]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P2]], i64 1021
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[P2]], i64 8168
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP1]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX9]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i32 [[TMP0]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i32 [[TMP5]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 256, [[TMP6]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 256, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[N_VEC]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = shl i16 [[TMP7]], 2
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP11]], i64 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP14]], i64 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT: [[TMP17:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP18:%.*]] = trunc <vscale x 2 x i32> [[TMP16]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP19:%.*]] = shl i64 [[TMP9]], 3
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP12]], 8
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP20]], i64 [[TMP21]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i16.p0.i64(<vscale x 2 x i16> [[TMP17]], ptr align 2 [[TMP20]], i64 8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i16.p0.i64(<vscale x 2 x i16> [[TMP18]], ptr align 2 [[TMP22]], i64 8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT: [[TMP23:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[TMP24:%.*]] = trunc <vscale x 2 x i32> [[TMP16]] to <vscale x 2 x i8>
+; CHECK-NEXT: [[TMP25:%.*]] = shl i64 [[TMP9]], 2
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP26]], i64 [[TMP27]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i8.p0.i64(<vscale x 2 x i8> [[TMP23]], ptr align 1 [[TMP26]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META6:![0-9]+]], !noalias [[META7:![0-9]+]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i8.p0.i64(<vscale x 2 x i8> [[TMP24]], ptr align 1 [[TMP28]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META6]], !noalias [[META7]]
+; CHECK-NEXT: [[TMP29:%.*]] = shl i64 [[TMP9]], 5
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP29]]
+; CHECK-NEXT: [[TMP31:%.*]] = mul i64 [[TMP12]], 32
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP30]], i64 [[TMP31]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i64.p0.i64(<vscale x 2 x i64> zeroinitializer, ptr align 8 [[TMP30]], i64 32, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META7]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i64.p0.i64(<vscale x 2 x i64> zeroinitializer, ptr align 8 [[TMP32]], i64 32, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.ext = sext i16 %iv to i64
+ %gep1 = getelementptr i32, ptr %p1, i64 %iv.ext
+ %v1 = load i32, ptr %gep1, align 4
+ %v1.trunc = trunc i32 %v1 to i16
+ %gep2 = getelementptr [1 x [1 x i16]], ptr %p2, i64 %iv.ext
+ store i16 %v1.trunc, ptr %gep2, align 2
+ %v1.trunc.i8 = trunc i32 %v1 to i8
+ %gep3 = getelementptr i8, ptr %p2, i64 %iv.ext
+ store i8 %v1.trunc.i8, ptr %gep3, align 1
+ %gep4 = getelementptr [1 x i64], ptr %p2, i64 %iv.ext
+ store i64 0, ptr %gep4, align 8
+ %iv.next = add i16 %iv, 4
+ %iv.next.ext = sext i16 %iv.next to i32
+ %ec = icmp ne i32 %iv.next.ext, 1024
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
>From 0898853d1d9e6b427896f4d265f426c12b694b1f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 14 Jul 2026 19:38:36 +0100
Subject: [PATCH 2/2] [VPlan] Use DerivedIV in convertToStridedAccesses
Introduce BinOpGEPFlags to package up WrapFlagsTy and GEPNoWrapFlags as
one of the choices of the VPIRFlags union, and use it in migrating
convertToStridedAccesses to use DerivedIV. The end result is that the
DerivedIV expanded by convertToConcreteRecipes produces identical IR.
---
.../Vectorize/LoopVectorizationPlanner.h | 8 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 47 +++++++++++++++++--
.../Transforms/Vectorize/VPlanLowering.cpp | 9 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 31 +++++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 34 +++++++-------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 2 +-
.../LoopVectorize/RISCV/strided-accesses.ll | 3 +-
8 files changed, 92 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..abee845fada34 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -387,10 +387,10 @@ class VPBuilder {
}
/// Convert \p Current to \p Start + \p Current * \p Step.
- VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
- FPMathOperator *FPBinOp, VPValue *Start,
- VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {}) {
+ VPDerivedIVRecipe *
+ createDerivedIV(InductionDescriptor::InductionKind Kind,
+ FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current,
+ VPValue *Step, const VPIRFlags::BinOpGEPFlagsTy Flags = {}) {
return tryInsertInstruction(
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Flags));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b2e87a8f4f52d..1e437fe7a7a4a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -705,6 +705,7 @@ class VPIRFlags {
enum class OperationType : unsigned char {
Cmp,
FCmp,
+ BinOpGEP,
OverflowingBinOp,
Trunc,
DisjointOp,
@@ -742,6 +743,18 @@ class VPIRFlags {
NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
};
+ /// Holds both the no-wrap flags and GEP no-wrap flags for computations
+ /// involving BinOps and a GEPs.
+ struct BinOpGEPFlagsTy {
+ WrapFlagsTy WrapFlags;
+ uint8_t RawGEPFlags;
+ BinOpGEPFlagsTy() = default;
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(0) {}
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags, GEPNoWrapFlags GEPFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(GEPFlags.getRaw()) {}
+ };
+
private:
struct ExactFlagsTy {
char IsExact : 1;
@@ -792,6 +805,7 @@ class VPIRFlags {
NonNegFlagsTy NonNegFlags;
FastMathFlagsTy FMFs;
FCmpFlagsTy FCmpFlags;
+ BinOpGEPFlagsTy BinOpGEPFlags;
ReductionFlagsTy ReductionFlags;
uint8_t AllFlags[2];
};
@@ -849,6 +863,12 @@ class VPIRFlags {
FCmpFlags.FMFs = FMFs;
}
+ VPIRFlags(BinOpGEPFlagsTy Flags)
+ : OpType(OperationType::BinOpGEP), AllFlags() {
+ BinOpGEPFlags.WrapFlags = Flags.WrapFlags;
+ BinOpGEPFlags.RawGEPFlags = Flags.RawGEPFlags;
+ }
+
VPIRFlags(WrapFlagsTy WrapFlags)
: OpType(OperationType::OverflowingBinOp), AllFlags() {
this->WrapFlags = WrapFlags;
@@ -929,6 +949,11 @@ class VPIRFlags {
case OperationType::NonNegOp:
NonNegFlags.NonNeg = false;
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW = false;
+ BinOpGEPFlags.WrapFlags.HasNSW = false;
+ BinOpGEPFlags.RawGEPFlags = 0;
+ break;
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -972,7 +997,8 @@ class VPIRFlags {
I.setNonNeg(NonNegFlags.NonNeg);
break;
case OperationType::ReductionOp:
- llvm_unreachable("reduction ops should not use applyFlags");
+ case OperationType::BinOpGEP:
+ llvm_unreachable("Reduction and BinOpGEP ops should not use applyFlags");
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -998,7 +1024,14 @@ class VPIRFlags {
}
GEPNoWrapFlags getGEPNoWrapFlags() const {
- return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ switch (OpType) {
+ case OperationType::GEPOp:
+ return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ case OperationType::BinOpGEP:
+ return GEPNoWrapFlags::fromRaw(BinOpGEPFlags.RawGEPFlags);
+ default:
+ llvm_unreachable("Recipe doesn't have GEP no-wrap flags");
+ }
}
/// Returns true if the recipe has a comparison predicate.
@@ -1024,6 +1057,8 @@ class VPIRFlags {
switch (OpType) {
case OperationType::OverflowingBinOp:
return WrapFlags.HasNUW;
+ case OperationType::BinOpGEP:
+ return BinOpGEPFlags.WrapFlags.HasNUW;
case OperationType::Trunc:
return TruncFlags.HasNUW;
default:
@@ -1035,6 +1070,8 @@ class VPIRFlags {
switch (OpType) {
case OperationType::OverflowingBinOp:
return WrapFlags.HasNSW;
+ case OperationType::BinOpGEP:
+ return BinOpGEPFlags.WrapFlags.HasNSW;
case OperationType::Trunc:
return TruncFlags.HasNSW;
default:
@@ -1046,6 +1083,7 @@ class VPIRFlags {
switch (OpType) {
case OperationType::OverflowingBinOp:
case OperationType::Trunc:
+ case OperationType::BinOpGEP:
return {hasNoUnsignedWrap(), hasNoSignedWrap()};
default:
return {};
@@ -4191,7 +4229,7 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPValue *Start,
VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {})
+ const VPIRFlags::BinOpGEPFlagsTy &Flags = {})
: VPRecipeWithIRFlags(VPRecipeBase::VPDerivedIVSC, {Start, Current, Step},
Start->getScalarType(), Flags),
Kind(Kind), FPBinOp(FPBinOp) {}
@@ -4200,7 +4238,8 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe *clone() override {
return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
- getStepValue(), getNoWrapFlags());
+ getStepValue(),
+ {getNoWrapFlags(), getGEPNoWrapFlags()});
}
VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index c1687e49824ed..f8df7dd887687 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -61,7 +61,8 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
nullptr, Plan.getZero(CanIVTy), Plan.getConstantInt(CanIVTy, 1),
WideCanIV->getDebugLoc(), Builder,
- {static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false}));
+ VPIRFlags::WrapFlagsTy(
+ static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false)));
WideCanIV->eraseFromParent();
return;
}
@@ -415,6 +416,7 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
: Builder.createScalarCast(Instruction::SIToFP, Index, StepTy,
DebugLoc::getCompilerGenerated());
VPIRFlags::WrapFlagsTy Flags = R->getNoWrapFlags();
+ GEPNoWrapFlags GEPFlags = R->getGEPNoWrapFlags();
switch (R->getInductionKind()) {
case InductionDescriptor::IK_IntInduction: {
assert(Index->getScalarType() == Start->getScalarType() &&
@@ -425,9 +427,10 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
DebugLoc::getUnknown(), "", Flags));
}
case InductionDescriptor::IK_PtrInduction:
- return R->replaceAllUsesWith(Builder.createPtrAdd(
+ return R->replaceAllUsesWith(Builder.createNoWrapPtrAdd(
Start,
- Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags)));
+ Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags),
+ GEPFlags));
case InductionDescriptor::IK_FpInduction: {
assert(StepTy->isFloatingPointTy() && "Expected FP Step value");
const FPMathOperator *FPBinOp = R->getFPBinOp();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 53ab77fbd28b1..a76438877ab40 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -397,6 +397,11 @@ void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
"Cannot change IsInLoop");
getFMFsRef() = getFastMathFlagsOrNone() & Other.getFastMathFlagsOrNone();
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW &= Other.BinOpGEPFlags.WrapFlags.HasNUW;
+ BinOpGEPFlags.WrapFlags.HasNSW &= Other.BinOpGEPFlags.WrapFlags.HasNSW;
+ BinOpGEPFlags.RawGEPFlags &= Other.BinOpGEPFlags.RawGEPFlags;
+ break;
case OperationType::Other:
break;
}
@@ -2623,6 +2628,12 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
case OperationType::ReductionOp:
return Opcode == VPInstruction::ComputeReductionResult;
+ case OperationType::BinOpGEP:
+ return Opcode == Instruction::GetElementPtr ||
+ Opcode == VPInstruction::PtrAdd ||
+ Opcode == VPInstruction::WidePtrAdd || Opcode == Instruction::Add ||
+ Opcode == Instruction::Sub || Opcode == Instruction::Mul ||
+ Opcode == Instruction::Shl;
case OperationType::Other:
return true;
}
@@ -2750,21 +2761,19 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
if (ExactFlags.IsExact)
O << " exact";
break;
- case OperationType::OverflowingBinOp:
- if (WrapFlags.HasNUW)
- O << " nuw";
- if (WrapFlags.HasNSW)
- O << " nsw";
+ case OperationType::FPMathOp:
+ getFastMathFlagsOrNone().print(O);
break;
+ case OperationType::OverflowingBinOp:
case OperationType::Trunc:
- if (TruncFlags.HasNUW)
+ case OperationType::BinOpGEP:
+ if (hasNoUnsignedWrap())
O << " nuw";
- if (TruncFlags.HasNSW)
+ if (hasNoSignedWrap())
O << " nsw";
- break;
- case OperationType::FPMathOp:
- getFastMathFlagsOrNone().print(O);
- break;
+ if (OpType != OperationType::BinOpGEP)
+ break;
+ [[fallthrough]];
case OperationType::GEPOp: {
GEPNoWrapFlags Flags = getGEPNoWrapFlags();
if (Flags.isInBounds())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index eb5958c37043d..cc04cc1a10317 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5769,39 +5769,37 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Ctx.invalidateWideningDecision(&MemR->getIngredient(), VF);
// Get VF as i32 for the vector length operand.
+ VPBuilder PHBuilder(Plan.getVectorPreheader());
if (!I32VF) {
- VPBuilder Builder(Plan.getVectorPreheader());
- I32VF = Builder.createScalarZExtOrTrunc(
+ I32VF = PHBuilder.createScalarZExtOrTrunc(
&Plan.getVF(), Type::getInt32Ty(Plan.getContext()),
DebugLoc::getUnknown());
}
- VPBuilder Builder(&R);
// Create the base pointer of strided access.
- // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
- // supports a general VPValue as the start value.
- VPValue *StartVPV = VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc())
- .tryToExpand(Start);
+ VPValue *StartVPV =
+ VPSCEVExpander(PHBuilder, *PSE.getSE(), R.getDebugLoc())
+ .tryToExpand(Start);
if (!StartVPV)
StartVPV = VPBuilder(Plan.getEntry()).createExpandSCEV(Start);
+ VPBuilder Builder(&R);
VPValue *StrideInBytes = Plan.getOrAddLiveIn(Step->getValue());
Type *IndexTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
assert(IndexTy == StrideInBytes->getScalarType() &&
"Stride type from SCEV must match the index type");
- VPValue *CanIV = Builder.createScalarZExtOrTrunc(
- VectorLoop->getCanonicalIV(), IndexTy, DebugLoc::getUnknown());
- auto *AddRecPtr = cast<SCEVAddRecExpr>(PtrSCEV);
- auto *Offset = Builder.createOverflowingOp(
- Instruction::Mul, {CanIV, StrideInBytes},
- {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
- GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
- ? GEPNoWrapFlags::noUnsignedWrap()
- : GEPNoWrapFlags::none();
- VPValue *BasePtr = Builder.createNoWrapPtrAdd(StartVPV, Offset, NWFlags);
+ // The no-wrap flags and GEP no-wrap flags should be derived from the nuw
+ // on the AddRecPtr.
+ bool AddRecNUW = cast<SCEVAddRecExpr>(PtrSCEV)->hasNoUnsignedWrap();
+ VPIRFlags::WrapFlagsTy WrapFlags = {AddRecNUW, false};
+ GEPNoWrapFlags GEPFlags =
+ AddRecNUW ? GEPNoWrapFlags::noUnsignedWrap() : GEPNoWrapFlags::none();
+ VPDerivedIVRecipe *BasePtr = Builder.createDerivedIV(
+ InductionDescriptor::IK_PtrInduction, nullptr, StartVPV,
+ VectorLoop->getCanonicalIV(), StrideInBytes, {WrapFlags, GEPFlags});
// Create a new vector pointer for strided access.
VPValue *NewPtr = Builder.createVectorPointer(
- BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, NWFlags,
+ BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, GEPFlags,
R.getDebugLoc());
VPValue *Mask = MemR->getMask();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index fe44ae5c1f846..9f48c54831414 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -597,7 +597,7 @@ VPScalarIVStepsRecipe *vputils::createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags) {
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..7d0ea0878613c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -196,7 +196,7 @@ VPScalarIVStepsRecipe *createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags = {});
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags = {});
/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd
/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 805ec6f1b4004..d1d5d3ab3ca3a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -255,8 +255,7 @@ define void @single_constant_stride_ptr_iv(ptr %p) {
; CHECK-UF2-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
; CHECK-UF2-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[TMP13]], splat (i32 1)
; CHECK-UF2-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[TMP15]], splat (i32 1)
-; CHECK-UF2-NEXT: [[TMP23:%.*]] = shl nuw i64 [[INDEX]], 3
-; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[TMP23]]
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[TMP9]]
; CHECK-UF2-NEXT: [[TMP24:%.*]] = mul i64 [[TMP3]], 8
; CHECK-UF2-NEXT: [[TMP20:%.*]] = getelementptr nuw i8, ptr [[TMP18]], i64 [[TMP24]]
; CHECK-UF2-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[TMP16]], ptr align 4 [[TMP18]], i64 8, <vscale x 4 x i1> splat (i1 true), i32 [[TMP21]])
More information about the llvm-commits
mailing list