[llvm] [VPlan] Populate and use VPIRMetadata from VPInstructions (NFC) (PR #167253)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 16 13:41:56 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/167253
>From 49fc7999e185be19c922364b68854e735785bd7a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 16 Nov 2025 13:12:57 +0000
Subject: [PATCH 1/3] [VPlan] Delegate to other VPInstruction constructors.
(NFCI)
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +--
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 ++++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ea88eaa42d945..754c6b50ae028 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1105,8 +1105,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {}
+ : VPInstruction(Opcode, Operands, {}, {}, DL, Name) {}
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD = {},
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index aa85bd435ee9e..10ad2fcedb441 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -538,6 +538,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case Instruction::Load:
case VPInstruction::AnyOf:
case VPInstruction::BranchOnCond:
+ case VPInstruction::Broadcast:
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
case VPInstruction::CalculateTripCountMinusVF:
@@ -548,13 +549,16 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::FirstActiveLane:
case VPInstruction::Not:
+ case VPInstruction::ResumeForEpilogue:
case VPInstruction::Unpack:
return 1;
case Instruction::ICmp:
case Instruction::FCmp:
+ case Instruction::ExtractElement:
case Instruction::Store:
case VPInstruction::BranchOnCount:
case VPInstruction::ComputeReductionResult:
+ case VPInstruction::ExtractLane:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::PtrAdd:
>From cbaf5bffa21e9237a8e0cbe8328894e7a2999905 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 5 Nov 2025 21:35:12 +0000
Subject: [PATCH 2/3] [VPlan] Populate and use VPIRMetadata from VPInstructions
(NFC)
Update VPlan to populate VPIRMetadata during VPInstruction construction
and use it when creating widened recipes, instead of constructing
VPIRMetadata from the underlying IR instruction each time.
This centralizes VPIRMetadata in VPInstructions and ensures metadata is
consistently available throughout VPlan transformations.
---
.../Vectorize/LoopVectorizationPlanner.h | 17 +++----
.../Transforms/Vectorize/LoopVectorize.cpp | 36 +++++++--------
.../Transforms/Vectorize/VPRecipeBuilder.h | 10 +---
llvm/lib/Transforms/Vectorize/VPlan.h | 46 +++++++++----------
.../Vectorize/VPlanConstruction.cpp | 35 ++++++++++----
llvm/lib/Transforms/Vectorize/VPlanSLP.cpp | 3 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 29 ++++++------
.../Transforms/Vectorize/VPlanTransforms.h | 3 +-
.../Transforms/Vectorize/VPlanTest.cpp | 8 ++--
9 files changed, 100 insertions(+), 87 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 04b05627fa769..c65ff446173f7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -65,7 +65,8 @@ class VPBuilder {
VPInstruction *createInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands, DebugLoc DL,
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
+ return tryInsertInstruction(
+ new VPInstruction(Opcode, Operands, {}, DL, Name));
}
public:
@@ -150,11 +151,11 @@ class VPBuilder {
/// its underlying Instruction.
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
Instruction *Inst = nullptr,
+ const VPIRMetadata &MD = {},
+ DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- DebugLoc DL = DebugLoc::getUnknown();
- if (Inst)
- DL = Inst->getDebugLoc();
- VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
+ VPInstruction *NewVPInst =
+ tryInsertInstruction(new VPInstruction(Opcode, Operands, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
@@ -212,7 +213,7 @@ class VPBuilder {
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
+ new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, {}, DL, Name));
}
VPInstruction *
@@ -223,7 +224,7 @@ class VPBuilder {
FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
*FMFs, {}, DL, Name)
: new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- DL, Name);
+ {}, DL, Name);
return tryInsertInstruction(Select);
}
@@ -329,7 +330,7 @@ class VPBuilder {
else if (Opcode == Instruction::ZExt)
Flags = VPIRFlags::NonNegFlagsTy(false);
return tryInsertInstruction(
- new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
+ new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cbfbc29360b0b..ab9481849c49e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7616,14 +7616,13 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
}
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
- return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
- VPIRMetadata(*Load, LVer), I->getDebugLoc());
+ return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, *VPI,
+ VPI->getDebugLoc());
}
StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask,
- Consecutive, Reverse,
- VPIRMetadata(*Store, LVer), VPI->getDebugLoc());
+ Consecutive, Reverse, *VPI, VPI->getDebugLoc());
}
/// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will
@@ -7742,7 +7741,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
},
Range);
if (ShouldUseVectorIntrinsic)
- return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
+ return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
VPI->getDebugLoc());
Function *Variant = nullptr;
@@ -7834,7 +7833,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
- return new VPWidenRecipe(*I, Ops);
+ return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
}
[[fallthrough]];
}
@@ -7880,7 +7879,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
- return new VPWidenRecipe(*I, NewOps);
+ return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(VPI->operands());
@@ -7888,7 +7887,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getConstantInt(32, Idx));
- return new VPWidenRecipe(*I, NewOps);
+ return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
}
};
}
@@ -7972,8 +7971,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
- auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform,
- BlockInMask, VPIRMetadata(*I, LVer));
+ auto *Recipe =
+ new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
return Recipe;
}
@@ -8232,7 +8231,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
- CastR->getResultType(), *CI);
+ CastR->getResultType(), CI, *VPI, *VPI);
}
return tryToWiden(VPI);
@@ -8260,7 +8259,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*ReductionI, Ops);
+ BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
+ ReductionI->getDebugLoc());
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
@@ -8293,7 +8293,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
// candidates built later for specific VF ranges.
auto VPlan0 = VPlanTransforms::buildVPlan0(
OrigLoop, *LI, Legal->getWidestInductionType(),
- getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
+ getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
@@ -8399,7 +8399,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder, BlockMaskCache, LVer);
+ Builder, BlockMaskCache);
// TODO: Handle partial reductions with EVL tail folding.
if (!CM.foldTailWithEVL())
RecipeBuilder.collectScaledReductions(Range);
@@ -8444,9 +8444,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
- auto *Recipe =
- new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
- nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
+ auto *Recipe = new VPReplicateRecipe(
+ SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
+ *cast<VPInstruction>(SingleDef));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
@@ -8597,7 +8597,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
// addScalarResumePhis.
DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder, BlockMaskCache, nullptr /*LVer*/);
+ Builder, BlockMaskCache);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a7000aff06379..87280b83fc0e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -84,10 +84,6 @@ class VPRecipeBuilder {
/// A mapping of partial reduction exit instructions to their scaling factor.
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
- /// Loop versioning instance for getting noalias metadata guaranteed by
- /// runtime checks.
- LoopVersioning *LVer;
-
/// Check if \p I can be widened at the start of \p Range and possibly
/// decrease the range such that the returned value holds for the entire \p
/// Range. The function should not be called for memory instructions or calls.
@@ -144,11 +140,9 @@ class VPRecipeBuilder {
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
- DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
- LoopVersioning *LVer)
+ DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache),
- LVer(LVer) {}
+ CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 754c6b50ae028..c7e703d2ddfc1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1103,10 +1103,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
#endif
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
- DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPInstruction(Opcode, Operands, {}, {}, DL, Name) {}
-
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "");
@@ -1314,7 +1310,7 @@ class VPPhiAccessors {
struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
VPPhi(ArrayRef<VPValue *> Operands, DebugLoc DL, const Twine &Name = "")
- : VPInstruction(Instruction::PHI, Operands, DL, Name) {}
+ : VPInstruction(Instruction::PHI, Operands, {}, DL, Name) {}
static inline bool classof(const VPUser *U) {
auto *VPI = dyn_cast<VPInstruction>(U);
@@ -1457,9 +1453,12 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode) {}
- VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
- Opcode(I.getOpcode()) {}
+ VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, VPIRFlags(I), DL),
+ VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
+ setUnderlyingValue(&I);
+ }
~VPWidenRecipe() override = default;
@@ -1499,31 +1498,26 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI),
- Opcode(Opcode), ResultTy(ResultTy) {
- assert(UI.getOpcode() == Opcode &&
- "opcode of underlying cast doesn't match");
- }
-
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- const VPIRFlags &Flags = {},
+ CastInst *UI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op,
+ UI ? VPIRFlags(*UI) : Flags,
+ UI ? UI->getDebugLoc() : DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
+ assert((!UI || UI->getOpcode() == Opcode) &&
+ "opcode of underlying cast doesn't match");
+ setUnderlyingValue(UI);
}
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
- *this, getDebugLoc());
- if (auto *UV = getUnderlyingValue())
- New->setUnderlyingValue(UV);
- return New;
+ return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
+ cast_or_null<CastInst>(getUnderlyingValue()),
+ *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
@@ -1567,9 +1561,10 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
+ const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
@@ -1594,7 +1589,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
- operands(), ResultTy, getDebugLoc());
+ operands(), ResultTy, *this,
+ getDebugLoc());
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
getDebugLoc());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4ffd5577d31a4..20519518d3afd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -22,6 +22,7 @@
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
#define DEBUG_TYPE "vplan"
@@ -37,6 +38,9 @@ class PlainCFGBuilder {
// Loop Info analysis.
LoopInfo *LI;
+ // Loop versioning for alias metadata.
+ LoopVersioning *LVer;
+
// Vectorization plan that we are working on.
std::unique_ptr<VPlan> Plan;
@@ -65,8 +69,8 @@ class PlainCFGBuilder {
void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
public:
- PlainCFGBuilder(Loop *Lp, LoopInfo *LI)
- : TheLoop(Lp), LI(LI), Plan(std::make_unique<VPlan>(Lp)) {}
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer)
+ : TheLoop(Lp), LI(LI), LVer(LVer), Plan(std::make_unique<VPlan>(Lp)) {}
/// Build plain CFG for TheLoop and connect it to Plan's entry.
std::unique_ptr<VPlan> buildPlainCFG();
@@ -186,7 +190,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
- VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst);
+ VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst,
+ VPIRMetadata(*Inst), Inst->getDebugLoc());
}
// Skip the rest of the Instruction processing for Branch instructions.
@@ -200,7 +205,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
- VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst);
+ VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst,
+ VPIRMetadata(*Inst), Inst->getDebugLoc());
continue;
}
@@ -228,6 +234,18 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
}
} else {
+ // Build VPIRMetadata from the instruction and add loop versioning
+ // metadata for loads and stores.
+ VPIRMetadata MD(*Inst);
+ if (isa<LoadInst, StoreInst>(Inst) && LVer) {
+ const auto &[AliasScopeMD, NoAliasMD] =
+ LVer->getNoAliasMetadataFor(Inst);
+ if (AliasScopeMD)
+ MD.addMetadata(LLVMContext::MD_alias_scope, AliasScopeMD);
+ if (NoAliasMD)
+ MD.addMetadata(LLVMContext::MD_noalias, NoAliasMD);
+ }
+
// Translate LLVM-IR operands into VPValue operands and set them in the
// new VPInstruction.
SmallVector<VPValue *, 4> VPOperands;
@@ -236,12 +254,12 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0],
- CI->getType(), CI->getDebugLoc());
+ CI->getType(), CI->getDebugLoc(), {}, MD);
NewR->setUnderlyingValue(CI);
} else {
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
- NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst);
+ NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, Inst->getDebugLoc());
}
}
@@ -537,8 +555,9 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
std::unique_ptr<VPlan>
VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- DebugLoc IVDL, PredicatedScalarEvolution &PSE) {
- PlainCFGBuilder Builder(TheLoop, &LI);
+ DebugLoc IVDL, PredicatedScalarEvolution &PSE,
+ LoopVersioning *LVer) {
+ PlainCFGBuilder Builder(TheLoop, &LI, LVer);
std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop);
return VPlan0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index bab7a9e4205f7..6b6d26a68942a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -516,7 +516,8 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef<VPValue *> Values) {
assert(CombinedOperands.size() > 0 && "Need more some operands");
auto *Inst = cast<VPInstruction>(Values[0])->getUnderlyingInstr();
- auto *VPI = new VPInstruction(Opcode, CombinedOperands, Inst->getDebugLoc());
+ auto *VPI =
+ new VPInstruction(Opcode, CombinedOperands, {}, Inst->getDebugLoc());
LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0]
<< "\n");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d05c22e3aeb61..d2e6ea2e3c46a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -80,20 +80,19 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Phi, Start, Step, &Plan.getVF(), *II, Ingredient.getDebugLoc());
}
} else {
- assert(isa<VPInstruction>(&Ingredient) &&
- "only VPInstructions expected here");
+ auto *VPI = cast<VPInstruction>(&Ingredient);
assert(!isa<PHINode>(Inst) && "phis should be handled above");
// Create VPWidenMemoryRecipe for loads and stores.
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/, VPIRMetadata(*Load),
+ false /*Consecutive*/, false /*Reverse*/, *VPI,
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
- nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- VPIRMetadata(*Store), Ingredient.getDebugLoc());
+ nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI,
+ Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -102,15 +101,17 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
return false;
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
- drop_end(Ingredient.operands()), CI->getType(),
+ drop_end(Ingredient.operands()), CI->getType(), *VPI,
CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands());
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
- NewRecipe = new VPWidenCastRecipe(
- CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
+ NewRecipe =
+ new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
+ CI->getType(), CI, {}, *VPI);
} else {
- NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands());
+ NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
+ Ingredient.getDebugLoc());
}
}
@@ -1835,7 +1836,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
// The vector region contains header phis for which we cannot remove the
// loop region yet.
auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()},
- Term->getDebugLoc());
+ {}, Term->getDebugLoc());
ExitingVPBB->appendRecipe(BOC);
}
@@ -3810,15 +3811,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Ext0->getOpcode() == Ext1->getOpcode() &&
IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
auto *NewExt0 = new VPWidenCastRecipe(
- Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
- *Ext0, Ext0->getDebugLoc());
+ Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr,
+ *Ext0, *Ext0, Ext0->getDebugLoc());
NewExt0->insertBefore(Ext0);
VPWidenCastRecipe *NewExt1 = NewExt0;
if (Ext0 != Ext1) {
NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
- Ext->getResultType(), *Ext1, *Ext1,
- Ext1->getDebugLoc());
+ Ext->getResultType(), nullptr, *Ext1,
+ *Ext1, Ext1->getDebugLoc());
NewExt1->insertBefore(Ext1);
}
Mul->setOperand(0, NewExt0);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e3bde8a47dcbc..a44a4f69c917b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -23,6 +23,7 @@ namespace llvm {
class InductionDescriptor;
class Instruction;
+class LoopVersioning;
class PHINode;
class ScalarEvolution;
class PredicatedScalarEvolution;
@@ -99,7 +100,7 @@ struct VPlanTransforms {
/// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks.
LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan>
buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL,
- PredicatedScalarEvolution &PSE);
+ PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr);
/// Update \p Plan to account for all early exits.
LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index bb2991262a505..d7bc3a558ce93 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1007,7 +1007,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()));
+ VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc());
checkVPRecipeCastImpl<VPWidenRecipe, VPUser>(&WidenR);
delete AI;
@@ -1090,7 +1090,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) {
IntegerType *Int64 = IntegerType::get(C, 64);
auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64);
VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast);
+ VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast);
checkVPRecipeCastImpl<VPWidenCastRecipe, VPUser>(&Recipe);
delete Cast;
@@ -1261,7 +1261,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end()));
+ VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc());
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1466,7 +1466,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
- new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end()));
+ new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc());
VPBB1->appendRecipe(WidenR);
{
>From 6521d2ae8bffb7a534272bb05b9a0003678d4296 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 16 Nov 2025 10:46:21 +0000
Subject: [PATCH 3/3] !fixup address comments, more uniform constructors.
---
.../Vectorize/LoopVectorizationPlanner.h | 38 +++++------
.../Transforms/Vectorize/LoopVectorize.cpp | 5 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 63 ++++++++++---------
llvm/lib/Transforms/Vectorize/VPlanSLP.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++----
5 files changed, 70 insertions(+), 64 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index c65ff446173f7..1490d97b2f4f8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -63,10 +63,11 @@ class VPBuilder {
}
VPInstruction *createInstruction(unsigned Opcode,
- ArrayRef<VPValue *> Operands, DebugLoc DL,
+ ArrayRef<VPValue *> Operands,
+ const VPIRMetadata &MD, DebugLoc DL,
const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Opcode, Operands, {}, DL, Name));
+ new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
}
public:
@@ -154,14 +155,14 @@ class VPBuilder {
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- VPInstruction *NewVPInst =
- tryInsertInstruction(new VPInstruction(Opcode, Operands, MD, DL, Name));
+ VPInstruction *NewVPInst = tryInsertInstruction(
+ new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
DebugLoc DL, const Twine &Name = "") {
- return createInstruction(Opcode, Operands, DL, Name);
+ return createInstruction(Opcode, Operands, {}, DL, Name);
}
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags,
@@ -175,8 +176,8 @@ class VPBuilder {
Type *ResultTy, const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
+ return tryInsertInstruction(new VPInstructionWithType(
+ Opcode, Operands, ResultTy, Flags, {}, DL, Name));
}
VPInstruction *createOverflowingOp(unsigned Opcode,
@@ -191,13 +192,14 @@ class VPBuilder {
VPInstruction *createNot(VPValue *Operand,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
+ return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name);
}
VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
+ return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL,
+ Name);
}
VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
@@ -212,20 +214,18 @@ class VPBuilder {
VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return tryInsertInstruction(
- new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, {}, DL, Name));
+ return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);
}
VPInstruction *
createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
- auto *Select =
- FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- *FMFs, {}, DL, Name)
- : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- {}, DL, Name);
- return tryInsertInstruction(Select);
+ if (!FMFs)
+ return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL,
+ Name);
+ return tryInsertInstruction(new VPInstruction(
+ Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name));
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
@@ -308,7 +308,7 @@ class VPBuilder {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}) {
return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Op, ResultTy, DL, Flags, Metadata));
+ new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
}
VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
@@ -330,7 +330,7 @@ class VPBuilder {
else if (Opcode == Instruction::ZExt)
Flags = VPIRFlags::NonNegFlagsTy(false);
return tryInsertInstruction(
- new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));
+ new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ab9481849c49e..a22d0f4db52fb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8225,13 +8225,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
if (VPI->getOpcode() == Instruction::Select)
- return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands());
+ return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
+ *VPI);
if (Instruction::isCast(VPI->getOpcode())) {
auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
- CastR->getResultType(), CI, *VPI, *VPI);
+ CastR->getResultType(), *CI, *VPI);
}
return tryToWiden(VPI);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c7e703d2ddfc1..bdf467cdbb7e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1104,7 +1104,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
- const VPIRFlags &Flags, const VPIRMetadata &MD = {},
+ const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "");
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
@@ -1191,14 +1191,10 @@ class VPInstructionWithType : public VPInstruction {
public:
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
- Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL,
+ Type *ResultTy, const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "")
- : VPInstruction(Opcode, Operands, Flags, {}, DL, Name),
- ResultTy(ResultTy) {}
-
- VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
- Type *ResultTy, DebugLoc DL, const VPIRFlags &Flags,
- const VPIRMetadata &Metadata, const Twine &Name = "")
: VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name),
ResultTy(ResultTy) {}
@@ -1227,7 +1223,7 @@ class VPInstructionWithType : public VPInstruction {
VPInstruction *clone() override {
auto *New =
new VPInstructionWithType(getOpcode(), operands(), getResultType(),
- *this, getDebugLoc(), getName());
+ *this, *this, getDebugLoc(), getName());
New->setUnderlyingValue(getUnderlyingValue());
return New;
}
@@ -1310,7 +1306,7 @@ class VPPhiAccessors {
struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
VPPhi(ArrayRef<VPValue *> Operands, DebugLoc DL, const Twine &Name = "")
- : VPInstruction(Instruction::PHI, Operands, {}, DL, Name) {}
+ : VPInstruction(Instruction::PHI, Operands, {}, {}, DL, Name) {}
static inline bool classof(const VPUser *U) {
auto *VPI = dyn_cast<VPInstruction>(U);
@@ -1455,10 +1451,8 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata, DebugLoc DL)
- : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, VPIRFlags(I), DL),
- VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
- setUnderlyingValue(&I);
- }
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
+ VPIRMetadata(Metadata), Opcode(I.getOpcode()) {}
~VPWidenRecipe() override = default;
@@ -1498,26 +1492,30 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst *UI = nullptr, const VPIRFlags &Flags = {},
+ CastInst &UI, const VPIRMetadata &Metadata)
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
+ VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ assert(UI.getOpcode() == Opcode &&
+ "opcode of underlying cast doesn't match");
+ }
+ VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
+ const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op,
- UI ? VPIRFlags(*UI) : Flags,
- UI ? UI->getDebugLoc() : DL),
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
- assert((!UI || UI->getOpcode() == Opcode) &&
- "opcode of underlying cast doesn't match");
- setUnderlyingValue(UI);
}
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
- cast_or_null<CastInst>(getUnderlyingValue()),
- *this, *this, getDebugLoc());
+ auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
+ *this, getDebugLoc());
+ if (auto *UV = getUnderlyingValue())
+ New->setUnderlyingValue(UV);
+ return New;
}
VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
@@ -1571,9 +1569,13 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
+ const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
+ : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags,
+ DL),
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1592,7 +1594,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
operands(), ResultTy, *this,
getDebugLoc());
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
- getDebugLoc());
+ *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
@@ -1730,15 +1732,16 @@ class VPHistogramRecipe : public VPRecipeBase {
/// instruction.
struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
- VPWidenSelectRecipe(SelectInst &I, ArrayRef<VPValue *> Operands)
+ VPWidenSelectRecipe(SelectInst &I, ArrayRef<VPValue *> Operands,
+ VPIRMetadata &MD)
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
- VPIRMetadata(I) {}
+ VPIRMetadata(MD) {}
~VPWidenSelectRecipe() override = default;
VPWidenSelectRecipe *clone() override {
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
- operands());
+ operands(), *this);
}
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index 6b6d26a68942a..528271b0beadb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -517,7 +517,7 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef<VPValue *> Values) {
assert(CombinedOperands.size() > 0 && "Need more some operands");
auto *Inst = cast<VPInstruction>(Values[0])->getUnderlyingInstr();
auto *VPI =
- new VPInstruction(Opcode, CombinedOperands, {}, Inst->getDebugLoc());
+ new VPInstruction(Opcode, CombinedOperands, {}, {}, Inst->getDebugLoc());
LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0]
<< "\n");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d2e6ea2e3c46a..537701c3fd563 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -104,11 +104,11 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
drop_end(Ingredient.operands()), CI->getType(), *VPI,
CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
- NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands());
+ NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI);
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewRecipe =
new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
- CI->getType(), CI, {}, *VPI);
+ CI->getType(), *CI, *VPI);
} else {
NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
Ingredient.getDebugLoc());
@@ -1696,8 +1696,9 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
Ops.append({ALM, Plan.getOrAddLiveIn(
ConstantInt::get(IntegerType::getInt64Ty(Ctx),
VF.getKnownMinValue() * Part))});
- auto *Ext = new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops,
- IntegerType::getInt1Ty(Ctx), DL);
+ auto *Ext =
+ new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops,
+ IntegerType::getInt1Ty(Ctx), {}, {}, DL);
Extracts[Part] = Ext;
Ext->insertAfter(ALM);
}
@@ -1836,7 +1837,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
// The vector region contains header phis for which we cannot remove the
// loop region yet.
auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()},
- {}, Term->getDebugLoc());
+ {}, {}, Term->getDebugLoc());
ExitingVPBB->appendRecipe(BOC);
}
@@ -2672,13 +2673,13 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
m_Select(m_Specific(HeaderMask), m_VPValue(LHS), m_VPValue(RHS))))
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {Plan->getTrue(), LHS, RHS, &EVL},
- TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc());
+ TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc());
if (match(&CurRecipe, m_Select(m_RemoveMask(HeaderMask, Mask), m_VPValue(LHS),
m_VPValue(RHS))))
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {Mask, LHS, RHS, &EVL},
- TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc());
+ TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc());
return nullptr;
}
@@ -2746,7 +2747,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
Intrinsic::experimental_vp_splice,
{V1, V2, Imm, Plan.getTrue(), PrevEVL, &EVL},
- TypeInfo.inferScalarType(R.getVPSingleValue()), R.getDebugLoc());
+ TypeInfo.inferScalarType(R.getVPSingleValue()), {}, {},
+ R.getDebugLoc());
VPSplice->insertBefore(&R);
R.getVPSingleValue()->replaceAllUsesWith(VPSplice);
ToErase.push_back(&R);
@@ -3811,15 +3813,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Ext0->getOpcode() == Ext1->getOpcode() &&
IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
auto *NewExt0 = new VPWidenCastRecipe(
- Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr,
- *Ext0, *Ext0, Ext0->getDebugLoc());
+ Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
+ *Ext0, Ext0->getDebugLoc());
NewExt0->insertBefore(Ext0);
VPWidenCastRecipe *NewExt1 = NewExt0;
if (Ext0 != Ext1) {
NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
- Ext->getResultType(), nullptr, *Ext1,
- *Ext1, Ext1->getDebugLoc());
+ Ext->getResultType(), *Ext1, *Ext1,
+ Ext1->getDebugLoc());
NewExt1->insertBefore(Ext1);
}
Mul->setOperand(0, NewExt0);
More information about the llvm-commits
mailing list