[llvm] [VPlan] Manage noalias/alias_scope metadata in VPlan. (NFC) (PR #136450)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun May 4 13:16:43 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/136450
>From 499becf62bbcf7dc8bede9931ee32a5eecf4bfe7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 14 Apr 2025 21:04:42 +0200
Subject: [PATCH 1/6] [VPlan] Manage noalias/alias_scope metadata in VPlan.
Use VPIRMetadata added in https://github.com/llvm/llvm-project/pull/135272
to also manage no-alias metadata added by versioning.
Note that this means we have to build the no-alias metadata up-front
once. If it is not used, it will be discarded automatically.
---
.../Vectorize/LoopVectorizationPlanner.h | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 67 +++++++++++--------
.../Transforms/Vectorize/VPRecipeBuilder.h | 14 +++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 14 +---
llvm/lib/Transforms/Vectorize/VPlan.h | 42 +++++++-----
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 15 -----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 10 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 21 ++++--
.../LoopVectorize/AArch64/store-costs-sve.ll | 17 ++---
.../Transforms/Vectorize/VPlanTest.cpp | 7 +-
10 files changed, 107 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 981ff7fc2364d..98cb135243ff7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -36,6 +36,7 @@ class LoopVectorizationLegality;
class LoopVectorizationCostModel;
class PredicatedScalarEvolution;
class LoopVectorizeHints;
+class LoopVersioning;
class OptimizationRemarkEmitter;
class TargetTransformInfo;
class TargetLibraryInfo;
@@ -518,7 +519,7 @@ class LoopVectorizationPlanner {
/// returned VPlan is valid for. If no VPlan can be built for the input range,
/// set the largest included VF to the maximum VF for which no plan could be
/// built.
- VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range);
+ VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range, LoopVersioning *LVer);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// according to the information gathered by Legal when it checked if it is
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2632dac9900ef..d5b10d221a003 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2350,7 +2350,7 @@ static void scalarizeInstruction(const Instruction *Instr,
InputLane = VPLane::getFirstLane();
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
}
- State.addNewMetadata(Cloned, Instr);
+ RepRecipe->applyMetadata(*Cloned);
// Place the cloned scalar in the new loop.
State.Builder.Insert(Cloned);
@@ -7895,24 +7895,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
if (VectorizingEpilogue)
VPlanTransforms::removeDeadRecipes(BestVPlan);
- // Only use noalias metadata when using memory checks guaranteeing no overlap
- // across all iterations.
- const LoopAccessInfo *LAI = Legal->getLAI();
- std::unique_ptr<LoopVersioning> LVer = nullptr;
- if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
- !LAI->getRuntimePointerChecking()->getDiffChecks()) {
-
- // We currently don't use LoopVersioning for the actual loop cloning but we
- // still use it to add the noalias metadata.
- // TODO: Find a better way to re-use LoopVersioning functionality to add
- // metadata.
- LVer = std::make_unique<LoopVersioning>(
- *LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
- PSE.getSE());
- State.LVer = &*LVer;
- State.LVer->prepareNoAliasMetadata();
- }
-
ILV.printDebugTracesAtStart();
//===------------------------------------------------===//
@@ -8503,13 +8485,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
Builder.insert(VectorPtr);
Ptr = VectorPtr;
}
+ auto Metadata = getMetadataToPropagate(I);
if (LoadInst *Load = dyn_cast<LoadInst>(I))
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
- I->getDebugLoc());
+ Metadata, I->getDebugLoc());
StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
- Reverse, I->getDebugLoc());
+ Reverse, Metadata, I->getDebugLoc());
}
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8882,7 +8865,7 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
- auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask);
+ auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask, getMetadataToPropagate(I));
return Recipe;
}
@@ -9003,6 +8986,20 @@ bool VPRecipeBuilder::getScaledReductions(
return false;
}
+SmallVector<std::pair<unsigned, MDNode *>>
+VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ ::getMetadataToPropagate(I, Metadata);
+ if (LVer && isa<LoadInst, StoreInst>(I)) {
+ const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
+ if (AliasScopeMD)
+ Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
+ if (NoAliasMD)
+ Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
+ }
+ return Metadata;
+}
+
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9129,10 +9126,22 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
ElementCount MaxVF) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
+ // Only use noalias metadata when using memory checks guaranteeing no overlap
+ // across all iterations.
+ const LoopAccessInfo *LAI = Legal->getLAI();
+ std::unique_ptr<LoopVersioning> LVer = nullptr;
+ if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
+ !LAI->getRuntimePointerChecking()->getDiffChecks()) {
+ LVer = std::make_unique<LoopVersioning>(
+ *LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
+ PSE.getSE());
+ LVer->prepareNoAliasMetadata();
+ }
+
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
+ if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, LVer.get())) {
bool HasScalarVF = Plan->hasScalarVFOnly();
// Now optimize the initial VPlan.
if (!HasScalarVF)
@@ -9397,7 +9406,8 @@ static void addExitUsersForFirstOrderRecurrences(
}
VPlanPtr
-LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
+LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
+ LoopVersioning *LVer) {
using namespace llvm::VPlanPatternMatch;
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -9453,7 +9463,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
}
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder);
+ Builder, LVer);
// ---------------------------------------------------------------------------
// Pre-construction: record ingredients whose recipes we'll need to further
@@ -9559,8 +9569,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
- auto *Recipe =
- new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */);
+ auto *Recipe = new VPReplicateRecipe(
+ SI, R.operands(), true /* IsUniform */, /*Mask*/ nullptr,
+ RecipeBuilder.getMetadataToPropagate(SI));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
@@ -9742,7 +9753,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
// Collect mapping of IR header phis to header phi recipes, to be used in
// addScalarResumePhis.
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder);
+ Builder, nullptr);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index fd0064a34c4c9..357a68972cb43 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -90,6 +90,10 @@ class VPRecipeBuilder {
/// A mapping of partial reduction exit instructions to their scaling factor.
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
+ /// Loop versioning instance for getting noalias metadata guaranteed by
+ /// runtime checks.
+ LoopVersioning *LVer;
+
/// Check if \p I can be widened at the start of \p Range and possibly
/// decrease the range such that the returned value holds for the entire \p
/// Range. The function should not be called for memory instructions or calls.
@@ -155,9 +159,10 @@ class VPRecipeBuilder {
const TargetTransformInfo *TTI,
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
- PredicatedScalarEvolution &PSE, VPBuilder &Builder)
+ PredicatedScalarEvolution &PSE, VPBuilder &Builder,
+ LoopVersioning *LVer)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(CM), PSE(PSE), Builder(Builder) {}
+ CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
@@ -233,6 +238,11 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
+
+ /// Returns the metatadata that can be preserved from the original instruction
+ /// \p I, including noalias metadata guaranteed by runtime checks.
+ SmallVector<std::pair<unsigned, MDNode *>>
+ getMetadataToPropagate(Instruction *I) const;
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 749ef4fe83bc8..35965571854b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -219,9 +219,9 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
DominatorTree *DT, AssumptionCache *AC,
IRBuilderBase &Builder, VPlan *Plan,
Loop *CurrentParentLoop, Type *CanonicalIVTy)
- : TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan),
- CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
- TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
+ : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
+ CurrentParentLoop(CurrentParentLoop), LVer(nullptr), TypeAnalysis(CanonicalIVTy),
+ VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
if (Def->isLiveIn())
@@ -355,14 +355,6 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
}
-void VPTransformState::addNewMetadata(Instruction *To,
- const Instruction *Orig) {
- // If the loop was versioned with memchecks, add the corresponding no-alias
- // metadata.
- if (LVer && isa<LoadInst, StoreInst>(Orig))
- LVer->annotateInstWithNoAlias(To, Orig);
-}
-
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
const DILocation *DIL = DL;
// When a FSDiscriminator is enabled, we don't need to add the multiply
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 89c77f2189b1f..60bc779a38930 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1194,6 +1194,8 @@ struct VPIRPhi : public VPIRInstruction {
#endif
};
+using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
+
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be propagated.
class VPIRMetadata {
@@ -1202,10 +1204,14 @@ class VPIRMetadata {
protected:
VPIRMetadata() {}
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+ VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
public:
/// Add all metadata to \p I.
void applyMetadata(Instruction &I) const;
+
+ /// Return the IR metadata.
+ MDArrayRef getMetadata() const { return Metadata; }
};
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2467,7 +2473,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
/// copies of the original scalar type, one per lane, instead of producing a
/// single copy of widened type for all lanes. If the instruction is known to be
/// uniform only one copy, per lane zero, will be generated.
-class VPReplicateRecipe : public VPRecipeWithIRFlags {
+class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Indicator if only a single replica per lane is needed.
bool IsUniform;
@@ -2476,9 +2482,9 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
public:
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
- bool IsUniform, VPValue *Mask = nullptr)
+ bool IsUniform, VPValue *Mask = nullptr, ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
- IsUniform(IsUniform), IsPredicated(Mask) {
+ VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
if (Mask)
addOperand(Mask);
}
@@ -2486,9 +2492,9 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
- auto *Copy =
- new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
- isPredicated() ? getMask() : nullptr);
+ auto *Copy = new VPReplicateRecipe(
+ getUnderlyingInstr(), operands(), IsUniform,
+ isPredicated() ? getMask() : nullptr, getMetadata());
Copy->transferFlags(*this);
return Copy;
}
@@ -2648,8 +2654,9 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
std::initializer_list<VPValue *> Operands,
- bool Consecutive, bool Reverse, DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
+ bool Consecutive, bool Reverse, MDArrayRef Metadata,
+ DebugLoc DL)
+ : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
}
@@ -2707,16 +2714,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
/// optional mask.
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
- bool Consecutive, bool Reverse, DebugLoc DL)
+ bool Consecutive, bool Reverse, MDArrayRef Metadata,
+ DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
- Reverse, DL),
+ Reverse, Metadata, DL),
VPValue(this, &Load) {
setMask(Mask);
}
VPWidenLoadRecipe *clone() override {
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, Reverse,
+ getMask(), Consecutive, Reverse, getMetadata(),
getDebugLoc());
}
@@ -2748,7 +2756,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{L.getAddr(), &EVL}, L.isConsecutive(),
- L.isReverse(), L.getDebugLoc()),
+ L.isReverse(), L.getMetadata(), L.getDebugLoc()),
VPValue(this, &getIngredient()) {
setMask(Mask);
}
@@ -2785,16 +2793,17 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
/// to store to and an optional mask.
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
- VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
+ VPValue *Mask, bool Consecutive, bool Reverse,
+ MDArrayRef Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
- Consecutive, Reverse, DL) {
+ Consecutive, Reverse, Metadata, DL) {
setMask(Mask);
}
VPWidenStoreRecipe *clone() override {
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
getStoredValue(), getMask(), Consecutive,
- Reverse, getDebugLoc());
+ Reverse, getMetadata(), getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2828,7 +2837,8 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
{S.getAddr(), S.getStoredValue(), &EVL},
- S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
+ S.isConsecutive(), S.isReverse(), S.getMetadata(),
+ S.getDebugLoc()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index cf8adc15e74bd..5b247106cf67e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -39,7 +39,6 @@ class VPBasicBlock;
class VPRegionBlock;
class VPlan;
class Value;
-class LoopVersioning;
/// Returns a calculation for the total number of elements for a given \p VF.
/// For fixed width vectors this value is a constant, whereas for scalable
@@ -284,13 +283,6 @@ struct VPTransformState {
Iter->second[CacheIdx] = V;
}
- /// Add additional metadata to \p To that was not present on \p Orig.
- ///
- /// Currently this is used to add the noalias annotations based on the
- /// inserted memchecks. Use this for instructions that are *cloned* into the
- /// vector loop.
- void addNewMetadata(Instruction *To, const Instruction *Orig);
-
/// Set the debug location in the builder using the debug location \p DL.
void setDebugLocFrom(DebugLoc DL);
@@ -343,13 +335,6 @@ struct VPTransformState {
/// The parent loop object for the current scope, or nullptr.
Loop *CurrentParentLoop = nullptr;
- /// LoopVersioning. It's only set up (non-null) if memchecks were
- /// used.
- ///
- /// This is currently only used to add no-alias metadata based on the
- /// memchecks. The actually versioning is performed manually.
- LoopVersioning *LVer = nullptr;
-
/// VPlan-based type analysis.
VPTypeAnalysis TypeAnalysis;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2108fbadb51d8..5cfda83eb701f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2722,7 +2722,6 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
}
void VPWidenLoadRecipe::execute(VPTransformState &State) {
-
Type *ScalarDataTy = getLoadStoreType(&Ingredient);
auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
const Align Alignment = getLoadStoreAlignment(&Ingredient);
@@ -2750,9 +2749,7 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
} else {
NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");
}
- // Add metadata to the load, but set the result to the reverse shuffle, if
- // needed.
- State.addNewMetadata(cast<Instruction>(NewLI), &Ingredient);
+ // Add metadata to the load, but setVectorValue to the reverse shuffle.
applyMetadata(*cast<Instruction>(NewLI));
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
@@ -2781,7 +2778,6 @@ static Instruction *createReverseEVL(IRBuilderBase &Builder, Value *Operand,
}
void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
-
Type *ScalarDataTy = getLoadStoreType(&Ingredient);
auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
const Align Alignment = getLoadStoreAlignment(&Ingredient);
@@ -2812,7 +2808,6 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
}
NewLI->addParamAttr(
0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- State.addNewMetadata(NewLI, &Ingredient);
applyMetadata(*NewLI);
Instruction *Res = NewLI;
if (isReverse())
@@ -2887,7 +2882,6 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- State.addNewMetadata(NewSI, &Ingredient);
applyMetadata(*NewSI);
}
@@ -2900,7 +2894,6 @@ void VPWidenStoreRecipe::print(raw_ostream &O, const Twine &Indent,
#endif
void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
-
VPValue *StoredValue = getStoredValue();
bool CreateScatter = !isConsecutive();
const Align Alignment = getLoadStoreAlignment(&Ingredient);
@@ -2934,7 +2927,6 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
}
NewSI->addParamAttr(
1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- State.addNewMetadata(NewSI, &Ingredient);
applyMetadata(*NewSI);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7093d378d8c3e..2740703e9c3e6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -78,15 +78,19 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
assert(!isa<PHINode>(Inst) && "phis should be handled above");
// Create VPWidenMemoryRecipe for loads and stores.
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ ::getMetadataToPropagate(Inst, Metadata);
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/,
+ false /*Consecutive*/, false /*Reverse*/, Metadata,
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ ::getMetadataToPropagate(Inst, Metadata);
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- Ingredient.getDebugLoc());
+ Metadata, Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -183,7 +187,9 @@ static bool sinkScalarOperands(VPlan &Plan) {
// TODO: Handle converting to uniform recipes as separate transform,
// then cloning should be sufficient here.
Instruction *I = SinkCandidate->getUnderlyingInstr();
- Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
+ Clone = new VPReplicateRecipe(
+ I, SinkCandidate->operands(), true, /*Mask*/ nullptr,
+ cast<VPReplicateRecipe>(SinkCandidate)->getMetadata());
// TODO: add ".cloned" suffix to name of Clone's VPValue.
} else {
Clone = SinkCandidate->clone();
@@ -345,7 +351,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(),
make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())),
- PredRecipe->isUniform());
+ PredRecipe->isUniform(), /*Mask*/ nullptr, PredRecipe->getMetadata());
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
@@ -2793,7 +2799,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *L = new VPWidenLoadRecipe(
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
- /*Reverse=*/false, LoadGroup->getDebugLoc());
+ /*Reverse=*/false, {}, LoadGroup->getDebugLoc());
L->insertBefore(LoadGroup);
return L;
}
@@ -2803,7 +2809,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
// Narrow wide load to uniform scalar load, as transformed VPlan will only
// process one original iteration.
auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(),
- WideLoad->operands(), /*IsUniform*/ true);
+ WideLoad->operands(), /*IsUniform*/ true,
+ /*Mask*/ nullptr, WideLoad->getMetadata());
N->insertBefore(WideLoad);
return N;
};
@@ -2824,7 +2831,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *S = new VPWidenStoreRecipe(
*cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos()),
StoreGroup->getAddr(), Res, nullptr, /*Consecutive=*/true,
- /*Reverse=*/false, StoreGroup->getDebugLoc());
+ /*Reverse=*/false, {}, StoreGroup->getDebugLoc());
S->insertBefore(StoreGroup);
StoreGroup->eraseFromParent();
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 9b6a1686eee6e..15c5258b57cc9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -188,17 +188,17 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; DEFAULT-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; DEFAULT: vec.epilog.vector.body:
; DEFAULT-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP16:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META11:![0-9]+]]
+; DEFAULT-NEXT: [[TMP16:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META5]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i64> poison, i64 [[TMP16]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT7]], <8 x i64> poison, <8 x i32> zeroinitializer
; DEFAULT-NEXT: [[TMP18:%.*]] = trunc <8 x i64> [[BROADCAST_SPLAT8]] to <8 x i8>
; DEFAULT-NEXT: [[TMP14:%.*]] = and <8 x i8> [[TMP18]], [[TMP15]]
; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX5]]
; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP26]], i32 0
-; DEFAULT-NEXT: store <8 x i8> [[TMP14]], ptr [[TMP27]], align 1, !alias.scope [[META14:![0-9]+]], !noalias [[META11]]
+; DEFAULT-NEXT: store <8 x i8> [[TMP14]], ptr [[TMP27]], align 1, !alias.scope [[META8]], !noalias [[META5]]
; DEFAULT-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX5]], 8
; DEFAULT-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT8]], 1000
-; DEFAULT-NEXT: br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; DEFAULT: vec.epilog.middle.block:
; DEFAULT-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; DEFAULT: vec.epilog.scalar.ph:
@@ -214,7 +214,7 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; DEFAULT-NEXT: store i8 [[TRUNC]], ptr [[GEP]], align 1
; DEFAULT-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; DEFAULT-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; DEFAULT: exit:
; DEFAULT-NEXT: ret void
;
@@ -313,13 +313,8 @@ attributes #1 = { vscale_range(1,16) "target-features"="+sve" }
; DEFAULT: [[META8]] = !{[[META9:![0-9]+]]}
; DEFAULT: [[META9]] = distinct !{[[META9]], [[META7]]}
; DEFAULT: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; DEFAULT: [[META11]] = !{[[META12:![0-9]+]]}
-; DEFAULT: [[META12]] = distinct !{[[META12]], [[META13:![0-9]+]]}
-; DEFAULT: [[META13]] = distinct !{[[META13]], !"LVerDomain"}
-; DEFAULT: [[META14]] = !{[[META15:![0-9]+]]}
-; DEFAULT: [[META15]] = distinct !{[[META15]], [[META13]]}
-; DEFAULT: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]]}
+; DEFAULT: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; DEFAULT: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
;.
; PRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; PRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index cb7545171744e..eec7b4480b75d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1084,7 +1084,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1201,7 +1201,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_TRUE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1215,7 +1215,8 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
VPValue *StoredV = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
- VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {});
+ VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {},
+ {});
EXPECT_TRUE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_TRUE(Recipe.mayWriteToMemory());
>From 60e1ab483e0633895ee42883f3639d0d96e033a7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 28 Apr 2025 10:52:35 +0100
Subject: [PATCH 2/6] !fixup move applyMetadata;
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d5b10d221a003..6a6890d181d15 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2337,6 +2337,7 @@ static void scalarizeInstruction(const Instruction *Instr,
}
RepRecipe->applyFlags(*Cloned);
+ RepRecipe->applyMetadata(*Cloned);
if (auto DL = RepRecipe->getDebugLoc())
State.setDebugLocFrom(DL);
@@ -2350,7 +2351,6 @@ static void scalarizeInstruction(const Instruction *Instr,
InputLane = VPLane::getFirstLane();
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
}
- RepRecipe->applyMetadata(*Cloned);
// Place the cloned scalar in the new loop.
State.Builder.Insert(Cloned);
>From 9c5fb10352176481fa4e12bd989afbdbe5c9d40d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 29 Apr 2025 13:15:46 +0100
Subject: [PATCH 3/6] !fixup address comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6a6890d181d15..175aab988eb73 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9126,22 +9126,20 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
ElementCount MaxVF) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
- // Only use noalias metadata when using memory checks guaranteeing no overlap
- // across all iterations.
const LoopAccessInfo *LAI = Legal->getLAI();
- std::unique_ptr<LoopVersioning> LVer = nullptr;
- if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
+ LoopVersioning LVer(*LAI, LAI->getRuntimePointerChecking()->getChecks(),
+ OrigLoop, LI, DT, PSE.getSE());
+ if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
- LVer = std::make_unique<LoopVersioning>(
- *LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
- PSE.getSE());
- LVer->prepareNoAliasMetadata();
+ // Only use noalias metadata when using memory checks guaranteeing no
+ // overlap across all iterations.
+ LVer.prepareNoAliasMetadata();
}
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, LVer.get())) {
+ if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, &LVer)) {
bool HasScalarVF = Plan->hasScalarVFOnly();
// Now optimize the initial VPlan.
if (!HasScalarVF)
>From 5083d032a7d60f19f6b9117150b0fa338e3995b8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 29 Apr 2025 22:46:46 +0100
Subject: [PATCH 4/6] !fixup address remaining comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 175aab988eb73..7e06db5379151 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9568,7 +9568,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *Recipe = new VPReplicateRecipe(
- SI, R.operands(), true /* IsUniform */, /*Mask*/ nullptr,
+ SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
RecipeBuilder.getMetadataToPropagate(SI));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
@@ -9751,7 +9751,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
// Collect mapping of IR header phis to header phi recipes, to be used in
// addScalarResumePhis.
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder, nullptr);
+ Builder);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 357a68972cb43..5c95a98961587 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -160,7 +160,7 @@ class VPRecipeBuilder {
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
- LoopVersioning *LVer)
+ LoopVersioning *LVer = nullptr)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
>From 8052f7fb87e979e7821c892128fb64b020ac9c9a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 3 May 2025 11:20:18 +0100
Subject: [PATCH 5/6] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 27 ++++++-------
.../Transforms/Vectorize/VPRecipeBuilder.h | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 39 +++++++++----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 -
.../Transforms/Vectorize/VPlanTransforms.cpp | 14 +++----
5 files changed, 41 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7e06db5379151..d20a171e64eef 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8485,7 +8485,7 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
Builder.insert(VectorPtr);
Ptr = VectorPtr;
}
- auto Metadata = getMetadataToPropagate(I);
+ auto Metadata = getRecipeMetadata(I);
if (LoadInst *Load = dyn_cast<LoadInst>(I))
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
Metadata, I->getDebugLoc());
@@ -8865,7 +8865,8 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
- auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask, getMetadataToPropagate(I));
+ auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
+ getRecipeMetadata(I));
return Recipe;
}
@@ -8986,18 +8987,18 @@ bool VPRecipeBuilder::getScaledReductions(
return false;
}
-SmallVector<std::pair<unsigned, MDNode *>>
-VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
+VPIRMetadata VPRecipeBuilder::getRecipeMetadata(Instruction *I) const {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
::getMetadataToPropagate(I, Metadata);
- if (LVer && isa<LoadInst, StoreInst>(I)) {
- const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
- if (AliasScopeMD)
- Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
- if (NoAliasMD)
- Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
- }
- return Metadata;
+ if (!LVer || !isa<LoadInst, StoreInst>(I))
+ return {};
+
+ const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
+ if (AliasScopeMD)
+ Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
+ if (NoAliasMD)
+ Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
+ return {Metadata};
}
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
@@ -9569,7 +9570,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *Recipe = new VPReplicateRecipe(
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
- RecipeBuilder.getMetadataToPropagate(SI));
+ RecipeBuilder.getRecipeMetadata(SI));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 5c95a98961587..74dd2bf1bd0d0 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -241,8 +241,7 @@ class VPRecipeBuilder {
/// Returns the metatadata that can be preserved from the original instruction
/// \p I, including noalias metadata guaranteed by runtime checks.
- SmallVector<std::pair<unsigned, MDNode *>>
- getMetadataToPropagate(Instruction *I) const;
+ VPIRMetadata getRecipeMetadata(Instruction *I) const;
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 60bc779a38930..8ab5de3ed800a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1194,24 +1194,22 @@ struct VPIRPhi : public VPIRInstruction {
#endif
};
-using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
-
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be propagated.
class VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
protected:
- VPIRMetadata() {}
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
- VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
public:
+ VPIRMetadata() {}
+ VPIRMetadata(ArrayRef<std::pair<unsigned, MDNode *>> Metadata)
+ : Metadata(Metadata) {}
+ VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
+
/// Add all metadata to \p I.
void applyMetadata(Instruction &I) const;
-
- /// Return the IR metadata.
- MDArrayRef getMetadata() const { return Metadata; }
};
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2482,7 +2480,8 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
- bool IsUniform, VPValue *Mask = nullptr, ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
+ bool IsUniform, VPValue *Mask = nullptr,
+ VPIRMetadata Metadata = {})
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
if (Mask)
@@ -2492,9 +2491,9 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
- auto *Copy = new VPReplicateRecipe(
- getUnderlyingInstr(), operands(), IsUniform,
- isPredicated() ? getMask() : nullptr, getMetadata());
+ auto *Copy =
+ new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
+ isPredicated() ? getMask() : nullptr, *this);
Copy->transferFlags(*this);
return Copy;
}
@@ -2654,8 +2653,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
std::initializer_list<VPValue *> Operands,
- bool Consecutive, bool Reverse, MDArrayRef Metadata,
- DebugLoc DL)
+ bool Consecutive, bool Reverse,
+ const VPIRMetadata &Metadata, DebugLoc DL)
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
@@ -2714,8 +2713,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
/// optional mask.
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
- bool Consecutive, bool Reverse, MDArrayRef Metadata,
- DebugLoc DL)
+ bool Consecutive, bool Reverse,
+ const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
Reverse, Metadata, DL),
VPValue(this, &Load) {
@@ -2724,7 +2723,7 @@ struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadRecipe *clone() override {
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, Reverse, getMetadata(),
+ getMask(), Consecutive, Reverse, *this,
getDebugLoc());
}
@@ -2756,7 +2755,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{L.getAddr(), &EVL}, L.isConsecutive(),
- L.isReverse(), L.getMetadata(), L.getDebugLoc()),
+ L.isReverse(), L, L.getDebugLoc()),
VPValue(this, &getIngredient()) {
setMask(Mask);
}
@@ -2794,7 +2793,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
VPValue *Mask, bool Consecutive, bool Reverse,
- MDArrayRef Metadata, DebugLoc DL)
+ const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
Consecutive, Reverse, Metadata, DL) {
setMask(Mask);
@@ -2803,7 +2802,7 @@ struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreRecipe *clone() override {
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
getStoredValue(), getMask(), Consecutive,
- Reverse, getMetadata(), getDebugLoc());
+ Reverse, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2837,7 +2836,7 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
{S.getAddr(), S.getStoredValue(), &EVL},
- S.isConsecutive(), S.isReverse(), S.getMetadata(),
+ S.isConsecutive(), S.isReverse(), S,
S.getDebugLoc()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5cfda83eb701f..5ab0d4c43bfef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2749,7 +2749,6 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
} else {
NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");
}
- // Add metadata to the load, but setVectorValue to the reverse shuffle.
applyMetadata(*cast<Instruction>(NewLI));
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2740703e9c3e6..9fa149ea382b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -82,7 +82,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
::getMetadataToPropagate(Inst, Metadata);
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/, Metadata,
+ false /*Consecutive*/, false /*Reverse*/, {Metadata},
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -90,7 +90,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- Metadata, Ingredient.getDebugLoc());
+ {Metadata}, Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -187,9 +187,9 @@ static bool sinkScalarOperands(VPlan &Plan) {
// TODO: Handle converting to uniform recipes as separate transform,
// then cloning should be sufficient here.
Instruction *I = SinkCandidate->getUnderlyingInstr();
- Clone = new VPReplicateRecipe(
- I, SinkCandidate->operands(), true, /*Mask*/ nullptr,
- cast<VPReplicateRecipe>(SinkCandidate)->getMetadata());
+ Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
+ /*Mask*/ nullptr,
+ *cast<VPReplicateRecipe>(SinkCandidate));
// TODO: add ".cloned" suffix to name of Clone's VPValue.
} else {
Clone = SinkCandidate->clone();
@@ -351,7 +351,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(),
make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())),
- PredRecipe->isUniform(), /*Mask*/ nullptr, PredRecipe->getMetadata());
+ PredRecipe->isUniform(), /*Mask*/ nullptr, *PredRecipe);
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
@@ -2810,7 +2810,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
// process one original iteration.
auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(),
WideLoad->operands(), /*IsUniform*/ true,
- /*Mask*/ nullptr, WideLoad->getMetadata());
+ /*Mask*/ nullptr, *WideLoad);
N->insertBefore(WideLoad);
return N;
};
>From e7a9297906212bf19421485299a0b375a57ba13c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 4 May 2025 21:16:02 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 30 +++++--------------
.../Transforms/Vectorize/VPRecipeBuilder.h | 6 +---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +--
llvm/lib/Transforms/Vectorize/VPlan.h | 15 ++++++----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 19 ++++++------
6 files changed, 42 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d20a171e64eef..01cfa43e56354 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8485,14 +8485,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
Builder.insert(VectorPtr);
Ptr = VectorPtr;
}
- auto Metadata = getRecipeMetadata(I);
if (LoadInst *Load = dyn_cast<LoadInst>(I))
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
- Metadata, I->getDebugLoc());
+ VPIRMetadata(*Load, LVer), I->getDebugLoc());
StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
- Reverse, Metadata, I->getDebugLoc());
+ Reverse, VPIRMetadata(*Store, LVer),
+ I->getDebugLoc());
}
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8866,7 +8866,7 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
- getRecipeMetadata(I));
+ VPIRMetadata(*I, LVer));
return Recipe;
}
@@ -8987,20 +8987,6 @@ bool VPRecipeBuilder::getScaledReductions(
return false;
}
-VPIRMetadata VPRecipeBuilder::getRecipeMetadata(Instruction *I) const {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- ::getMetadataToPropagate(I, Metadata);
- if (!LVer || !isa<LoadInst, StoreInst>(I))
- return {};
-
- const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
- if (AliasScopeMD)
- Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
- if (NoAliasMD)
- Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
- return {Metadata};
-}
-
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9568,9 +9554,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
- auto *Recipe = new VPReplicateRecipe(
- SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
- RecipeBuilder.getRecipeMetadata(SI));
+ auto *Recipe =
+ new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
+ nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
@@ -9752,7 +9738,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
// Collect mapping of IR header phis to header phi recipes, to be used in
// addScalarResumePhis.
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder);
+ Builder, nullptr);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 74dd2bf1bd0d0..caa18e263676b 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -160,7 +160,7 @@ class VPRecipeBuilder {
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
- LoopVersioning *LVer = nullptr)
+ LoopVersioning *LVer)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
@@ -238,10 +238,6 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
-
- /// Returns the metatadata that can be preserved from the original instruction
- /// \p I, including noalias metadata guaranteed by runtime checks.
- VPIRMetadata getRecipeMetadata(Instruction *I) const;
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 35965571854b8..f7a01edeef56b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -219,8 +219,8 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
DominatorTree *DT, AssumptionCache *AC,
IRBuilderBase &Builder, VPlan *Plan,
Loop *CurrentParentLoop, Type *CanonicalIVTy)
- : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
- CurrentParentLoop(CurrentParentLoop), LVer(nullptr), TypeAnalysis(CanonicalIVTy),
+ : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), Plan(Plan),
+ CurrentParentLoop(CurrentParentLoop), TypeAnalysis(CanonicalIVTy),
VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8ab5de3ed800a..524b4b411953c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -65,6 +65,7 @@ class VPReplicateRecipe;
class VPlanSlp;
class Value;
class LoopVectorizationCostModel;
+class LoopVersioning;
struct VPCostContext;
@@ -1199,13 +1200,17 @@ struct VPIRPhi : public VPIRInstruction {
class VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
-protected:
- VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
-
public:
VPIRMetadata() {}
- VPIRMetadata(ArrayRef<std::pair<unsigned, MDNode *>> Metadata)
- : Metadata(Metadata) {}
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I.
+ VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
+ VPIRMetadata(Instruction &I, LoopVersioning *LVer);
+
VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
/// Add all metadata to \p I.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5ab0d4c43bfef..3f7b041d3fe90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -36,6 +36,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <cassert>
@@ -1205,6 +1206,17 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
+VPIRMetadata::VPIRMetadata(Instruction &I, LoopVersioning *LVer)
+ : VPIRMetadata(I) {
+ if (!LVer || !isa<LoadInst, StoreInst>(&I))
+ return;
+ const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(&I);
+ if (AliasScopeMD)
+ Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
+ if (NoAliasMD)
+ Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
+}
+
void VPIRMetadata::applyMetadata(Instruction &I) const {
for (const auto &[Kind, Node] : Metadata)
I.setMetadata(Kind, Node);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9fa149ea382b9..15f583da490c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -82,15 +82,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
::getMetadataToPropagate(Inst, Metadata);
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/, {Metadata},
+ false /*Consecutive*/, false /*Reverse*/, VPIRMetadata(*Load),
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- ::getMetadataToPropagate(Inst, Metadata);
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- {Metadata}, Ingredient.getDebugLoc());
+ VPIRMetadata(*Store), Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -183,13 +181,13 @@ static bool sinkScalarOperands(VPlan &Plan) {
if (ScalarVFOnly)
continue;
VPSingleDefRecipe *Clone;
- if (isa<VPReplicateRecipe>(SinkCandidate)) {
+ if (auto *SinkCandidateRepR =
+ dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
// TODO: Handle converting to uniform recipes as separate transform,
// then cloning should be sufficient here.
Instruction *I = SinkCandidate->getUnderlyingInstr();
Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
- /*Mask*/ nullptr,
- *cast<VPReplicateRecipe>(SinkCandidate));
+ /*Mask*/ nullptr, *SinkCandidateRepR);
// TODO: add ".cloned" suffix to name of Clone's VPValue.
} else {
Clone = SinkCandidate->clone();
@@ -2796,10 +2794,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(R)) {
// Narrow interleave group to wide load, as transformed VPlan will only
// process one original iteration.
+ auto &LI =
+ *cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos());
auto *L = new VPWidenLoadRecipe(
- *cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
- LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
- /*Reverse=*/false, {}, LoadGroup->getDebugLoc());
+ LI, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
+ /*Reverse=*/false, VPIRMetadata(LI), LoadGroup->getDebugLoc());
L->insertBefore(LoadGroup);
return L;
}
More information about the llvm-commits
mailing list