[llvm] [VPlan] Use DerivedIV in convertToStridedAccesses (NFC) (PR #209611)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 05:28:40 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/209611
>From 156e3b4c5bee260023bd2ebfb57f21c5b1562acf Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 14 Jul 2026 19:38:36 +0100
Subject: [PATCH] [VPlan] Use DerivedIV in convertToStridedAccesses (NFC)
Introduce BinOpGEPFlags to package up WrapFlagsTy and GEPNoWrapFlags as
one of the choices of the VPIRFlags union, and use it in migrating
convertToStridedAccesses to use DerivedIV. The end result is that the
DerivedIV expanded by convertToConcreteRecipes produces identical IR.
---
.../Vectorize/LoopVectorizationPlanner.h | 8 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 48 +++++++++++++++++--
.../Transforms/Vectorize/VPlanLowering.cpp | 9 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 31 +++++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 31 ++++++------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 2 +-
7 files changed, 90 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fa317f290022e..0ac0be7b0090a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -387,10 +387,10 @@ class VPBuilder {
}
/// Convert \p Current to \p Start + \p Current * \p Step.
- VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
- FPMathOperator *FPBinOp, VPValue *Start,
- VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {}) {
+ VPDerivedIVRecipe *
+ createDerivedIV(InductionDescriptor::InductionKind Kind,
+ FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current,
+ VPValue *Step, const VPIRFlags::BinOpGEPFlagsTy Flags = {}) {
return tryInsertInstruction(
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Flags));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..8f1751c8adacf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -705,6 +705,7 @@ class VPIRFlags {
enum class OperationType : unsigned char {
Cmp,
FCmp,
+ BinOpGEP,
OverflowingBinOp,
Trunc,
DisjointOp,
@@ -742,6 +743,18 @@ class VPIRFlags {
NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
};
+ /// Holds both the no-wrap flags and GEP no-wrap flags for computations
+ /// involving BinOps and a GEPs.
+ struct BinOpGEPFlagsTy {
+ WrapFlagsTy WrapFlags;
+ uint8_t RawGEPFlags;
+ BinOpGEPFlagsTy() = default;
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(0) {}
+ BinOpGEPFlagsTy(WrapFlagsTy WrapFlags, GEPNoWrapFlags GEPFlags)
+ : WrapFlags(WrapFlags), RawGEPFlags(GEPFlags.getRaw()) {}
+ };
+
private:
struct ExactFlagsTy {
char IsExact : 1;
@@ -792,6 +805,7 @@ class VPIRFlags {
NonNegFlagsTy NonNegFlags;
FastMathFlagsTy FMFs;
FCmpFlagsTy FCmpFlags;
+ BinOpGEPFlagsTy BinOpGEPFlags;
ReductionFlagsTy ReductionFlags;
uint8_t AllFlags[2];
};
@@ -849,6 +863,12 @@ class VPIRFlags {
FCmpFlags.FMFs = FMFs;
}
+ VPIRFlags(BinOpGEPFlagsTy Flags)
+ : OpType(OperationType::BinOpGEP), AllFlags() {
+ BinOpGEPFlags.WrapFlags = Flags.WrapFlags;
+ BinOpGEPFlags.RawGEPFlags = Flags.RawGEPFlags;
+ }
+
VPIRFlags(WrapFlagsTy WrapFlags)
: OpType(OperationType::OverflowingBinOp), AllFlags() {
this->WrapFlags = WrapFlags;
@@ -929,6 +949,11 @@ class VPIRFlags {
case OperationType::NonNegOp:
NonNegFlags.NonNeg = false;
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW = false;
+ BinOpGEPFlags.WrapFlags.HasNSW = false;
+ BinOpGEPFlags.RawGEPFlags = 0;
+ break;
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -972,7 +997,8 @@ class VPIRFlags {
I.setNonNeg(NonNegFlags.NonNeg);
break;
case OperationType::ReductionOp:
- llvm_unreachable("reduction ops should not use applyFlags");
+ case OperationType::BinOpGEP:
+ llvm_unreachable("Reduction and BinOpGEP ops should not use applyFlags");
case OperationType::Cmp:
case OperationType::Other:
break;
@@ -998,7 +1024,17 @@ class VPIRFlags {
}
GEPNoWrapFlags getGEPNoWrapFlags() const {
- return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ assert(
+ is_contained({OperationType::GEPOp, OperationType::BinOpGEP}, OpType) &&
+ "Recipe doesn't have GEP no-wrap flags");
+ switch (OpType) {
+ case OperationType::GEPOp:
+ return GEPNoWrapFlags::fromRaw(GEPFlagsStorage);
+ case OperationType::BinOpGEP:
+ return GEPNoWrapFlags::fromRaw(BinOpGEPFlags.RawGEPFlags);
+ default:
+ return GEPNoWrapFlags::none();
+ }
}
/// Returns true if the recipe has a comparison predicate.
@@ -1023,6 +1059,7 @@ class VPIRFlags {
bool hasNoUnsignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
+ case OperationType::BinOpGEP:
return WrapFlags.HasNUW;
case OperationType::Trunc:
return TruncFlags.HasNUW;
@@ -1034,6 +1071,7 @@ class VPIRFlags {
bool hasNoSignedWrap() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
+ case OperationType::BinOpGEP:
return WrapFlags.HasNSW;
case OperationType::Trunc:
return TruncFlags.HasNSW;
@@ -1046,6 +1084,7 @@ class VPIRFlags {
switch (OpType) {
case OperationType::OverflowingBinOp:
case OperationType::Trunc:
+ case OperationType::BinOpGEP:
return {hasNoUnsignedWrap(), hasNoSignedWrap()};
default:
return {};
@@ -4182,7 +4221,7 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPValue *Start,
VPValue *Current, VPValue *Step,
- const VPIRFlags::WrapFlagsTy &Flags = {})
+ const VPIRFlags::BinOpGEPFlagsTy &Flags = {})
: VPRecipeWithIRFlags(VPRecipeBase::VPDerivedIVSC, {Start, Current, Step},
Start->getScalarType(), Flags),
Kind(Kind), FPBinOp(FPBinOp) {}
@@ -4191,7 +4230,8 @@ class VPDerivedIVRecipe : public VPRecipeWithIRFlags {
VPDerivedIVRecipe *clone() override {
return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
- getStepValue(), getNoWrapFlags());
+ getStepValue(),
+ {getNoWrapFlags(), getGEPNoWrapFlags()});
}
VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 72b6503157d68..8e62607dad04a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -61,7 +61,8 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
nullptr, Plan.getZero(CanIVTy), Plan.getConstantInt(CanIVTy, 1),
WideCanIV->getDebugLoc(), Builder,
- {static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false}));
+ VPIRFlags::WrapFlagsTy(
+ static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false)));
WideCanIV->eraseFromParent();
return;
}
@@ -422,6 +423,7 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
: Builder.createScalarCast(Instruction::SIToFP, Index, StepTy,
DebugLoc::getCompilerGenerated());
VPIRFlags::WrapFlagsTy Flags = R->getNoWrapFlags();
+ GEPNoWrapFlags GEPFlags = R->getGEPNoWrapFlags();
switch (R->getInductionKind()) {
case InductionDescriptor::IK_IntInduction: {
assert(Index->getScalarType() == Start->getScalarType() &&
@@ -432,9 +434,10 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
DebugLoc::getUnknown(), "", Flags));
}
case InductionDescriptor::IK_PtrInduction:
- return R->replaceAllUsesWith(Builder.createPtrAdd(
+ return R->replaceAllUsesWith(Builder.createNoWrapPtrAdd(
Start,
- Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags)));
+ Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags),
+ GEPFlags));
case InductionDescriptor::IK_FpInduction: {
assert(StepTy->isFloatingPointTy() && "Expected FP Step value");
const FPMathOperator *FPBinOp = R->getFPBinOp();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..9cf8480d59a61 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -394,6 +394,11 @@ void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
"Cannot change IsInLoop");
getFMFsRef() = getFastMathFlagsOrNone() & Other.getFastMathFlagsOrNone();
break;
+ case OperationType::BinOpGEP:
+ BinOpGEPFlags.WrapFlags.HasNUW &= Other.BinOpGEPFlags.WrapFlags.HasNUW;
+ BinOpGEPFlags.WrapFlags.HasNSW &= Other.BinOpGEPFlags.WrapFlags.HasNSW;
+ BinOpGEPFlags.RawGEPFlags &= Other.BinOpGEPFlags.RawGEPFlags;
+ break;
case OperationType::Other:
break;
}
@@ -2577,6 +2582,12 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
case OperationType::ReductionOp:
return Opcode == VPInstruction::ComputeReductionResult;
+ case OperationType::BinOpGEP:
+ return Opcode == Instruction::GetElementPtr ||
+ Opcode == VPInstruction::PtrAdd ||
+ Opcode == VPInstruction::WidePtrAdd || Opcode == Instruction::Add ||
+ Opcode == Instruction::Sub || Opcode == Instruction::Mul ||
+ Opcode == Instruction::Shl;
case OperationType::Other:
return true;
}
@@ -2704,21 +2715,19 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
if (ExactFlags.IsExact)
O << " exact";
break;
- case OperationType::OverflowingBinOp:
- if (WrapFlags.HasNUW)
- O << " nuw";
- if (WrapFlags.HasNSW)
- O << " nsw";
+ case OperationType::FPMathOp:
+ getFastMathFlagsOrNone().print(O);
break;
+ case OperationType::OverflowingBinOp:
case OperationType::Trunc:
- if (TruncFlags.HasNUW)
+ case OperationType::BinOpGEP:
+ if (hasNoUnsignedWrap())
O << " nuw";
- if (TruncFlags.HasNSW)
+ if (hasNoSignedWrap())
O << " nsw";
- break;
- case OperationType::FPMathOp:
- getFastMathFlagsOrNone().print(O);
- break;
+ if (OpType != OperationType::BinOpGEP)
+ break;
+ [[fallthrough]];
case OperationType::GEPOp: {
GEPNoWrapFlags Flags = getGEPNoWrapFlags();
if (Flags.isInBounds())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4d6f40ec32a62..13feda7e4c514 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5787,40 +5787,37 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
Ctx.invalidateWideningDecision(&LoadR->getIngredient(), VF);
// Get VF as i32 for the vector length operand.
+ VPBuilder PHBuilder(Plan.getVectorPreheader());
if (!I32VF) {
- VPBuilder Builder(Plan.getVectorPreheader());
- I32VF = Builder.createScalarZExtOrTrunc(
+ I32VF = PHBuilder.createScalarZExtOrTrunc(
&Plan.getVF(), Type::getInt32Ty(Plan.getContext()),
DebugLoc::getUnknown());
}
- VPBuilder Builder(LoadR);
// Create the base pointer of strided access.
- // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
- // supports a general VPValue as the start value.
VPValue *StartVPV =
- VPSCEVExpander(Builder, *PSE.getSE(), LoadR->getDebugLoc())
+ VPSCEVExpander(PHBuilder, *PSE.getSE(), LoadR->getDebugLoc())
.tryToExpand(Start);
if (!StartVPV)
StartVPV = VPBuilder(Plan.getEntry()).createExpandSCEV(Start);
+ VPBuilder Builder(LoadR);
VPValue *StrideInBytes = Plan.getOrAddLiveIn(Step->getValue());
Type *IndexTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
assert(IndexTy == StrideInBytes->getScalarType() &&
"Stride type from SCEV must match the index type");
- VPValue *CanIV = Builder.createScalarZExtOrTrunc(
- VectorLoop->getCanonicalIV(), IndexTy, DebugLoc::getUnknown());
- auto *AddRecPtr = cast<SCEVAddRecExpr>(PtrSCEV);
- auto *Offset = Builder.createOverflowingOp(
- Instruction::Mul, {CanIV, StrideInBytes},
- {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
- GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
- ? GEPNoWrapFlags::noUnsignedWrap()
- : GEPNoWrapFlags::none();
- VPValue *BasePtr = Builder.createNoWrapPtrAdd(StartVPV, Offset, NWFlags);
+ // The no-wrap flags and GEP no-wrap flags should be derived from the nuw
+ // on the AddRecPtr.
+ bool AddRecNUW = cast<SCEVAddRecExpr>(PtrSCEV)->hasNoUnsignedWrap();
+ VPIRFlags::WrapFlagsTy WrapFlags = {AddRecNUW, false};
+ GEPNoWrapFlags GEPFlags =
+ AddRecNUW ? GEPNoWrapFlags::noUnsignedWrap() : GEPNoWrapFlags::none();
+ VPDerivedIVRecipe *BasePtr = Builder.createDerivedIV(
+ InductionDescriptor::IK_PtrInduction, nullptr, StartVPV,
+ VectorLoop->getCanonicalIV(), StrideInBytes, {WrapFlags, GEPFlags});
// Create a new vector pointer for strided access.
VPValue *NewPtr = Builder.createVectorPointer(
- BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, NWFlags,
+ BasePtr, Type::getInt8Ty(Plan.getContext()), StrideInBytes, GEPFlags,
LoadR->getDebugLoc());
VPValue *Mask = LoadR->getMask();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 93b18b31e9e7d..5988ad5f0b9cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -597,7 +597,7 @@ VPScalarIVStepsRecipe *vputils::createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags) {
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..7d0ea0878613c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -196,7 +196,7 @@ VPScalarIVStepsRecipe *createScalarIVSteps(
VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp,
Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL,
- VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags = {});
+ VPBuilder &Builder, const VPIRFlags::BinOpGEPFlagsTy &Flags = {});
/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd
/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only
More information about the llvm-commits
mailing list