[llvm] [VPlan] Manage instruction medata in VPlan. (PR #135272)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 22 03:50:32 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/135272
>From 3760c4f90be29a2f7b7d36965a33fe40b79692c9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 9 Apr 2025 12:35:39 +0100
Subject: [PATCH 1/4] [VPlan] Manage instruction medata in VPlan.
Add a new helper to manage IR metadata that can be progated to generated
instructions for recipes.
This helps to remove a number of remaining uses of getUnderlyingInstr
during VPlan execution.
Add metadata on cloning.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 34 +++--
.../Transforms/Vectorize/VPRecipeBuilder.h | 5 +
llvm/lib/Transforms/Vectorize/VPlan.cpp | 20 +--
llvm/lib/Transforms/Vectorize/VPlan.h | 132 ++++++++++++------
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 9 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 35 +++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 32 +++--
.../Transforms/Vectorize/VPlanTest.cpp | 24 ++--
8 files changed, 179 insertions(+), 112 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7a5f618d09e95..398e575199b21 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8599,11 +8599,13 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
}
if (LoadInst *Load = dyn_cast<LoadInst>(I))
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
+ getMetadataToPropagate(Load),
I->getDebugLoc());
StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
- Reverse, I->getDebugLoc());
+ Reverse, getMetadataToPropagate(Store),
+ I->getDebugLoc());
}
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8745,6 +8747,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
+ getMetadataToPropagate(CI),
CI->getDebugLoc());
Function *Variant = nullptr;
@@ -8798,7 +8801,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
}
Ops.push_back(Operands.back());
- return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
+ return new VPWidenCallRecipe(CI, Variant, Ops, getMetadataToPropagate(CI),
+ CI->getDebugLoc());
}
return nullptr;
@@ -8836,7 +8840,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
Ops[1] = SafeRHS;
- return new VPWidenRecipe(*I, make_range(Ops.begin(), Ops.end()));
+ return new VPWidenRecipe(*I, make_range(Ops.begin(), Ops.end()),
+ getMetadataToPropagate(I));
}
[[fallthrough]];
}
@@ -8882,7 +8887,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
- return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
+ return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()),
+ getMetadataToPropagate(I));
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(Operands);
@@ -8891,7 +8897,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
- return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
+ return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()),
+ getMetadataToPropagate(I));
}
};
}
@@ -9096,6 +9103,13 @@ bool VPRecipeBuilder::getScaledReductions(
return false;
}
+SmallVector<std::pair<unsigned, MDNode *>>
+VPRecipeBuilder::getMetadataToPropagate(Instruction *I) {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ ::getMetadataToPropagate(I, Metadata);
+ return Metadata;
+}
+
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9168,13 +9182,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
make_range(Operands.begin(), Operands.end()));
if (auto *SI = dyn_cast<SelectInst>(Instr)) {
- return new VPWidenSelectRecipe(
- *SI, make_range(Operands.begin(), Operands.end()));
+ return new VPWidenSelectRecipe(*SI,
+ make_range(Operands.begin(), Operands.end()),
+ getMetadataToPropagate(SI));
}
if (auto *CI = dyn_cast<CastInst>(Instr)) {
return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
- *CI);
+ *CI, getMetadataToPropagate(CI));
}
return tryToWiden(Instr, Operands);
@@ -9200,7 +9215,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*Reduction, make_range(Ops.begin(), Ops.end()));
+ BinOp = new VPWidenRecipe(*Reduction, make_range(Ops.begin(), Ops.end()),
+ getMetadataToPropagate(Reduction));
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index fd0064a34c4c9..a3eae052a25fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -233,6 +233,11 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
+
+ /// Returns the metatadata that can be preserved from the original instruction
+ /// \p I, including noalias metadata guaranteed by runtime checks.
+ SmallVector<std::pair<unsigned, MDNode *>>
+ getMetadataToPropagate(Instruction *I);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fa2d95a44609a..e7eb4b36ced07 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -355,23 +355,13 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
}
-void VPTransformState::addNewMetadata(Instruction *To,
- const Instruction *Orig) {
+void VPTransformState::addNewMetadata(Value *To, const Instruction *Orig) {
+
// If the loop was versioned with memchecks, add the corresponding no-alias
// metadata.
- if (LVer && isa<LoadInst, StoreInst>(Orig))
- LVer->annotateInstWithNoAlias(To, Orig);
-}
-
-void VPTransformState::addMetadata(Value *To, Instruction *From) {
- // No source instruction to transfer metadata from?
- if (!From)
- return;
-
- if (Instruction *ToI = dyn_cast<Instruction>(To)) {
- propagateMetadata(ToI, From);
- addNewMetadata(ToI, From);
- }
+ Instruction *ToI = dyn_cast<Instruction>(To);
+ if (ToI && LVer && isa<LoadInst, StoreInst>(Orig))
+ LVer->annotateInstWithNoAlias(ToI, Orig);
}
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7084676af6d5b..6a9020382999a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1190,28 +1190,55 @@ struct VPIRPhi : public VPIRInstruction {
#endif
};
+using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
+
+/// Helper to manage IR metadata for recipes. It filters out metadata that
+/// cannot be proagated.
+class VPWithIRMetadata {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+
+protected:
+ VPWithIRMetadata() {}
+ VPWithIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
+
+ void addMetadata(MDArrayRef Metadata) {
+ append_range(this->Metadata, Metadata);
+ }
+
+public:
+ /// Add all metadata to \p V if it is an instruction.
+ void setMetadata(Value *V) const;
+
+ void addMetadata(unsigned Kind, MDNode *N) { Metadata.emplace_back(Kind, N); }
+
+ MDArrayRef getMetadata() const { return Metadata; }
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
/// vectorized version of itself.
-class VPWidenRecipe : public VPRecipeWithIRFlags {
+class VPWidenRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
unsigned Opcode;
protected:
template <typename IterT>
VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
- iterator_range<IterT> Operands)
- : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), Opcode(I.getOpcode()) {}
+ iterator_range<IterT> Operands, MDArrayRef Metadata)
+ : VPRecipeWithIRFlags(VPDefOpcode, Operands, I),
+ VPWithIRMetadata(Metadata), Opcode(I.getOpcode()) {}
public:
template <typename IterT>
- VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
- : VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
+ VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands,
+ MDArrayRef Metadata)
+ : VPWidenRecipe(VPDef::VPWidenSC, I, Operands, Metadata) {}
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
- auto *R = new VPWidenRecipe(*getUnderlyingInstr(), operands());
+ auto *R =
+ new VPWidenRecipe(*getUnderlyingInstr(), operands(), getMetadata());
R->transferFlags(*this);
return R;
}
@@ -1236,7 +1263,7 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
};
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
-class VPWidenCastRecipe : public VPRecipeWithIRFlags {
+class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
@@ -1245,23 +1272,23 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
- ResultTy(ResultTy) {
+ CastInst &UI, MDArrayRef Metadata)
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
+ VPWithIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
- ResultTy(ResultTy) {}
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPWithIRMetadata(),
+ Opcode(Opcode), ResultTy(ResultTy) {}
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
if (auto *UV = getUnderlyingValue())
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
- *cast<CastInst>(UV));
+ *cast<CastInst>(UV), getMetadata());
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
}
@@ -1288,7 +1315,8 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
};
/// A recipe for widening vector intrinsics.
-class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
+class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags,
+ public VPWithIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
@@ -1307,18 +1335,19 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
- DebugLoc DL = {})
+ MDArrayRef Metadata, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
- MayReadFromMemory(CI.mayReadFromMemory()),
+ VPWithIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
- DebugLoc DL = {})
+ MDArrayRef Metadata, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
+ VPWithIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1334,7 +1363,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
VPWidenIntrinsicRecipe *clone() override {
return new VPWidenIntrinsicRecipe(*cast<CallInst>(getUnderlyingValue()),
VectorIntrinsicID, {op_begin(), op_end()},
- ResultTy, getDebugLoc());
+ ResultTy, getMetadata(), getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
@@ -1374,7 +1403,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
};
/// A recipe for widening Call instructions using library calls.
-class VPWidenCallRecipe : public VPRecipeWithIRFlags {
+class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
/// between a given VF and the chosen vectorized variant, so there will be a
/// different VPlan for each VF with a valid variant.
@@ -1382,10 +1411,11 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags {
public:
VPWidenCallRecipe(Value *UV, Function *Variant,
- ArrayRef<VPValue *> CallArguments, DebugLoc DL = {})
+ ArrayRef<VPValue *> CallArguments, MDArrayRef Metadata,
+ DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
- Variant(Variant) {
+ VPWithIRMetadata(Metadata), Variant(Variant) {
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1395,7 +1425,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags {
VPWidenCallRecipe *clone() override {
return new VPWidenCallRecipe(getUnderlyingValue(), Variant,
- {op_begin(), op_end()}, getDebugLoc());
+ {op_begin(), op_end()}, getMetadata(),
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
@@ -1471,16 +1502,19 @@ class VPHistogramRecipe : public VPRecipeBase {
};
/// A recipe for widening select instructions.
-struct VPWidenSelectRecipe : public VPRecipeWithIRFlags {
+struct VPWidenSelectRecipe : public VPRecipeWithIRFlags,
+ public VPWithIRMetadata {
template <typename IterT>
- VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I) {}
+ VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands,
+ MDArrayRef Metadata)
+ : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
+ VPWithIRMetadata(Metadata) {}
~VPWidenSelectRecipe() override = default;
VPWidenSelectRecipe *clone() override {
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
- operands());
+ operands(), getMetadata());
}
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
@@ -2602,7 +2636,7 @@ class VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// A common base class for widening memory operations. An optional mask can be
/// provided as the last operand.
-class VPWidenMemoryRecipe : public VPRecipeBase {
+class VPWidenMemoryRecipe : public VPRecipeBase, public VPWithIRMetadata {
protected:
Instruction &Ingredient;
@@ -2625,9 +2659,10 @@ class VPWidenMemoryRecipe : public VPRecipeBase {
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
std::initializer_list<VPValue *> Operands,
- bool Consecutive, bool Reverse, DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), Ingredient(I), Consecutive(Consecutive),
- Reverse(Reverse) {
+ bool Consecutive, bool Reverse, MDArrayRef Metadata,
+ DebugLoc DL)
+ : VPRecipeBase(SC, Operands, DL), VPWithIRMetadata(Metadata),
+ Ingredient(I), Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
}
@@ -2684,17 +2719,20 @@ class VPWidenMemoryRecipe : public VPRecipeBase {
/// optional mask.
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
- bool Consecutive, bool Reverse, DebugLoc DL)
+ bool Consecutive, bool Reverse, MDArrayRef Metadata,
+ DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
- Reverse, DL),
+ Reverse, Metadata, DL),
VPValue(this, &Load) {
setMask(Mask);
}
VPWidenLoadRecipe *clone() override {
- return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, Reverse,
- getDebugLoc());
+ auto *Copy = new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
+ getMask(), Consecutive, Reverse,
+ getMetadata(), getDebugLoc());
+ Copy->addMetadata(getMetadata());
+ return Copy;
}
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC);
@@ -2725,9 +2763,10 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{L.getAddr(), &EVL}, L.isConsecutive(),
- L.isReverse(), L.getDebugLoc()),
+ L.isReverse(), L.getMetadata(), L.getDebugLoc()),
VPValue(this, &getIngredient()) {
setMask(Mask);
+ addMetadata(L.getMetadata());
}
VP_CLASSOF_IMPL(VPDef::VPWidenLoadEVLSC)
@@ -2762,16 +2801,19 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
/// to store to and an optional mask.
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
- VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
+ VPValue *Mask, bool Consecutive, bool Reverse,
+ MDArrayRef Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
- Consecutive, Reverse, DL) {
+ Consecutive, Reverse, Metadata, DL) {
setMask(Mask);
}
VPWidenStoreRecipe *clone() override {
- return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
- getStoredValue(), getMask(), Consecutive,
- Reverse, getDebugLoc());
+ auto *Copy = new VPWidenStoreRecipe(
+ cast<StoreInst>(Ingredient), getAddr(), getStoredValue(), getMask(),
+ Consecutive, Reverse, getMetadata(), getDebugLoc());
+ Copy->addMetadata(getMetadata());
+ return Copy;
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2805,8 +2847,10 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
{S.getAddr(), S.getStoredValue(), &EVL},
- S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
+ S.isConsecutive(), S.isReverse(), S.getMetadata(),
+ S.getDebugLoc()) {
setMask(Mask);
+ addMetadata(S.getMetadata());
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreEVLSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index bebea1915690f..4826dfd562c2d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -288,14 +288,7 @@ struct VPTransformState {
/// Currently this is used to add the noalias annotations based on the
/// inserted memchecks. Use this for instructions that are *cloned* into the
/// vector loop.
- void addNewMetadata(Instruction *To, const Instruction *Orig);
-
- /// Add metadata from one instruction to another.
- ///
- /// This includes both the original MDs from \p From and additional ones (\see
- /// addNewMetadata). Use this for *newly created* instructions in the vector
- /// loop.
- void addMetadata(Value *To, Instruction *From);
+ void addNewMetadata(Value *To, const Instruction *Orig);
/// Set the debug location in the builder using the debug location \p DL.
void setDebugLocFrom(DebugLoc DL);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2cc558f49ccce..06eb03f904ba2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1204,6 +1204,14 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPWithIRMetadata::setMetadata(Value *V) const {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return;
+ for (const auto &[Kind, Node] : Metadata)
+ I->setMetadata(Kind, Node);
+}
+
void VPWidenCallRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
@@ -1234,7 +1242,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
if (!V->getType()->isVoidTy())
State.set(this, V);
- State.addMetadata(V, CI);
+ setMetadata(V);
}
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
@@ -1313,7 +1321,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
if (!V->getType()->isVoidTy())
State.set(this, V);
- State.addMetadata(V, CI);
+ setMetadata(V);
}
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
@@ -1510,7 +1518,7 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
State.set(this, Sel);
if (isa<FPMathOperator>(Sel))
setFlags(cast<Instruction>(Sel));
- State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
+ setMetadata(Sel);
}
InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
@@ -1646,7 +1654,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
// Use this vector value for all users of the original instruction.
State.set(this, V);
- State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
+ setMetadata(V);
break;
}
case Instruction::ExtractValue: {
@@ -1679,7 +1687,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
C = Builder.CreateICmp(getPredicate(), A, B);
}
State.set(this, C);
- State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
+ setMetadata(C);
break;
}
default:
@@ -1795,9 +1803,10 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
Value *A = State.get(Op);
Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
State.set(this, Cast);
- State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
- if (auto *CastOp = dyn_cast<Instruction>(Cast))
+ if (auto *CastOp = dyn_cast<Instruction>(Cast)) {
setFlags(CastOp);
+ setMetadata(CastOp);
+ }
}
InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
@@ -2749,7 +2758,8 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");
}
// Add metadata to the load, but setVectorValue to the reverse shuffle.
- State.addMetadata(NewLI, LI);
+ State.addNewMetadata(NewLI, LI);
+ setMetadata(NewLI);
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
State.set(this, NewLI);
@@ -2809,7 +2819,8 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
}
NewLI->addParamAttr(
0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- State.addMetadata(NewLI, LI);
+ State.addNewMetadata(NewLI, LI);
+ setMetadata(NewLI);
Instruction *Res = NewLI;
if (isReverse())
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
@@ -2885,7 +2896,8 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- State.addMetadata(NewSI, SI);
+ State.addNewMetadata(NewSI, SI);
+ setMetadata(NewSI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2932,7 +2944,8 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
}
NewSI->addParamAttr(
1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- State.addMetadata(NewSI, SI);
+ State.addNewMetadata(NewSI, SI);
+ setMetadata(NewSI);
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b80fe18d1bd66..7dba249085421 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -70,6 +70,8 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenIntOrFpInductionRecipe(
Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
} else {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ getMetadataToPropagate(Inst, Metadata);
assert(isa<VPInstruction>(&Ingredient) &&
"only VPInstructions expected here");
assert(!isa<PHINode>(Inst) && "phis should be handled above");
@@ -77,13 +79,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/,
+ false /*Consecutive*/, false /*Reverse*/, Metadata,
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- Ingredient.getDebugLoc());
+ Metadata, Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -93,14 +95,16 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
{Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(),
- CI->getDebugLoc());
+ Metadata, CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
- NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands());
+ NewRecipe =
+ new VPWidenSelectRecipe(*SI, Ingredient.operands(), Metadata);
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
- NewRecipe = new VPWidenCastRecipe(
- CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
+ NewRecipe =
+ new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
+ CI->getType(), *CI, Metadata);
} else {
- NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands());
+ NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), Metadata);
}
}
@@ -2009,9 +2013,9 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
SmallVector<VPValue *> Ops(Sel->operands());
Ops.push_back(&EVL);
- return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
- TypeInfo.inferScalarType(Sel),
- Sel->getDebugLoc());
+ return new VPWidenIntrinsicRecipe(
+ Intrinsic::vp_select, Ops, TypeInfo.inferScalarType(Sel),
+ Sel->getMetadata(), Sel->getDebugLoc());
})
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
if (VPI->getOpcode() == VPInstruction::FirstOrderRecurrenceSplice) {
@@ -2023,7 +2027,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
Ops.append({MinusOneVPV, &AllOneMask, PrevEVL, &EVL});
return new VPWidenIntrinsicRecipe(Intrinsic::experimental_vp_splice,
Ops, TypeInfo.inferScalarType(VPI),
- VPI->getDebugLoc());
+ {}, VPI->getDebugLoc());
}
VPValue *LHS, *RHS;
@@ -2038,7 +2042,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
// limited to selects whose condition is a header mask.
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
- TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
+ TypeInfo.inferScalarType(LHS), {}, VPI->getDebugLoc());
})
.Default([&](VPRecipeBase *R) { return nullptr; });
}
@@ -2755,7 +2759,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *L = new VPWidenLoadRecipe(
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
- /*Reverse=*/false, LoadGroup->getDebugLoc());
+ /*Reverse=*/false, {}, LoadGroup->getDebugLoc());
L->insertBefore(LoadGroup);
return L;
}
@@ -2786,7 +2790,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *S = new VPWidenStoreRecipe(
*cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos()),
StoreGroup->getAddr(), Res, nullptr, /*Consecutive=*/true,
- /*Reverse=*/false, StoreGroup->getDebugLoc());
+ /*Reverse=*/false, {}, StoreGroup->getDebugLoc());
S->insertBefore(StoreGroup);
StoreGroup->eraseFromParent();
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index cb7545171744e..277e03cbe8588 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -929,7 +929,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()));
+ VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()), {});
EXPECT_TRUE(isa<VPUser>(&WidenR));
VPRecipeBase *WidenRBase = &WidenR;
EXPECT_TRUE(isa<VPUser>(WidenRBase));
@@ -950,7 +950,7 @@ TEST_F(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(CalledFn);
- VPWidenCallRecipe Recipe(Call, Fn, Args);
+ VPWidenCallRecipe Recipe(Call, Fn, Args, {});
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -978,7 +978,7 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
Args.push_back(Op2);
Args.push_back(Op3);
VPWidenSelectRecipe WidenSelectR(*SelectI,
- make_range(Args.begin(), Args.end()));
+ make_range(Args.begin(), Args.end()), {});
EXPECT_TRUE(isa<VPUser>(&WidenSelectR));
VPRecipeBase *BaseR = &WidenSelectR;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1084,7 +1084,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1111,7 +1111,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end()));
+ VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end()), {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1130,7 +1130,8 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
- VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()));
+ VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()),
+ {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1201,7 +1202,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_TRUE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1215,7 +1216,8 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
VPValue *StoredV = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
- VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {});
+ VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {},
+ {});
EXPECT_TRUE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_TRUE(Recipe.mayWriteToMemory());
@@ -1234,7 +1236,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(CalledFn);
- VPWidenCallRecipe Recipe(Call, Fn, Args);
+ VPWidenCallRecipe Recipe(Call, Fn, Args, {});
EXPECT_TRUE(Recipe.mayHaveSideEffects());
EXPECT_TRUE(Recipe.mayReadFromMemory());
EXPECT_TRUE(Recipe.mayWriteToMemory());
@@ -1257,7 +1259,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(CalledFn);
- VPWidenCallRecipe Recipe(Call, TheFn, Args);
+ VPWidenCallRecipe Recipe(Call, TheFn, Args, {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1314,7 +1316,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
- new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end()));
+ new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end()), {});
VPBB1->appendRecipe(WidenR);
{
>From d3edd8957422d6907e6ce8e31975c7918bc77063 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 19 Apr 2025 20:13:23 +0100
Subject: [PATCH 2/4] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/VPRecipeBuilder.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 52 +++++++------------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 25 ++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-
4 files changed, 35 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a3eae052a25fe..aab34834ae9bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -236,7 +236,7 @@ class VPRecipeBuilder {
/// Returns the metatadata that can be preserved from the original instruction
/// \p I, including noalias metadata guaranteed by runtime checks.
- SmallVector<std::pair<unsigned, MDNode *>>
+ static SmallVector<std::pair<unsigned, MDNode *>>
getMetadataToPropagate(Instruction *I);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a9020382999a..0692ad4e3bbfb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1194,23 +1194,17 @@ using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be proagated.
-class VPWithIRMetadata {
+class VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
protected:
- VPWithIRMetadata() {}
- VPWithIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
-
- void addMetadata(MDArrayRef Metadata) {
- append_range(this->Metadata, Metadata);
- }
+ VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
public:
/// Add all metadata to \p V if it is an instruction.
- void setMetadata(Value *V) const;
-
- void addMetadata(unsigned Kind, MDNode *N) { Metadata.emplace_back(Kind, N); }
+ void applyMetadata(Value *V) const;
+ /// Return the IR metadata.
MDArrayRef getMetadata() const { return Metadata; }
};
@@ -1218,15 +1212,15 @@ class VPWithIRMetadata {
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
/// vectorized version of itself.
-class VPWidenRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
+class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
unsigned Opcode;
protected:
template <typename IterT>
VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
iterator_range<IterT> Operands, MDArrayRef Metadata)
- : VPRecipeWithIRFlags(VPDefOpcode, Operands, I),
- VPWithIRMetadata(Metadata), Opcode(I.getOpcode()) {}
+ : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), VPIRMetadata(Metadata),
+ Opcode(I.getOpcode()) {}
public:
template <typename IterT>
@@ -1263,7 +1257,7 @@ class VPWidenRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
};
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
-class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
+class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
@@ -1274,13 +1268,13 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst &UI, MDArrayRef Metadata)
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
- VPWithIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPWithIRMetadata(),
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPIRMetadata({}),
Opcode(Opcode), ResultTy(ResultTy) {}
~VPWidenCastRecipe() override = default;
@@ -1315,8 +1309,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
};
/// A recipe for widening vector intrinsics.
-class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags,
- public VPWithIRMetadata {
+class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
@@ -1337,7 +1330,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments, Type *Ty,
MDArrayRef Metadata, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VPWithIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
@@ -1346,7 +1339,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments, Type *Ty,
MDArrayRef Metadata, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VPWithIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
@@ -1403,7 +1396,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags,
};
/// A recipe for widening Call instructions using library calls.
-class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
+class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
/// between a given VF and the chosen vectorized variant, so there will be a
/// different VPlan for each VF with a valid variant.
@@ -1415,7 +1408,7 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPWithIRMetadata {
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
- VPWithIRMetadata(Metadata), Variant(Variant) {
+ VPIRMetadata(Metadata), Variant(Variant) {
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1502,13 +1495,12 @@ class VPHistogramRecipe : public VPRecipeBase {
};
/// A recipe for widening select instructions.
-struct VPWidenSelectRecipe : public VPRecipeWithIRFlags,
- public VPWithIRMetadata {
+struct VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
template <typename IterT>
VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands,
MDArrayRef Metadata)
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
- VPWithIRMetadata(Metadata) {}
+ VPIRMetadata(Metadata) {}
~VPWidenSelectRecipe() override = default;
@@ -2636,7 +2628,7 @@ class VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// A common base class for widening memory operations. An optional mask can be
/// provided as the last operand.
-class VPWidenMemoryRecipe : public VPRecipeBase, public VPWithIRMetadata {
+class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
protected:
Instruction &Ingredient;
@@ -2661,8 +2653,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPWithIRMetadata {
std::initializer_list<VPValue *> Operands,
bool Consecutive, bool Reverse, MDArrayRef Metadata,
DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPWithIRMetadata(Metadata),
- Ingredient(I), Consecutive(Consecutive), Reverse(Reverse) {
+ : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
+ Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
}
@@ -2731,7 +2723,6 @@ struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
auto *Copy = new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
getMask(), Consecutive, Reverse,
getMetadata(), getDebugLoc());
- Copy->addMetadata(getMetadata());
return Copy;
}
@@ -2766,7 +2757,6 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
L.isReverse(), L.getMetadata(), L.getDebugLoc()),
VPValue(this, &getIngredient()) {
setMask(Mask);
- addMetadata(L.getMetadata());
}
VP_CLASSOF_IMPL(VPDef::VPWidenLoadEVLSC)
@@ -2812,7 +2802,6 @@ struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
auto *Copy = new VPWidenStoreRecipe(
cast<StoreInst>(Ingredient), getAddr(), getStoredValue(), getMask(),
Consecutive, Reverse, getMetadata(), getDebugLoc());
- Copy->addMetadata(getMetadata());
return Copy;
}
@@ -2850,7 +2839,6 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
S.isConsecutive(), S.isReverse(), S.getMetadata(),
S.getDebugLoc()) {
setMask(Mask);
- addMetadata(S.getMetadata());
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreEVLSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 06eb03f904ba2..b5a5d8b7ace7d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1204,7 +1204,7 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPWithIRMetadata::setMetadata(Value *V) const {
+void VPIRMetadata::applyMetadata(Value *V) const {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return;
@@ -1239,10 +1239,10 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
setFlags(V);
+ applyMetadata(V);
if (!V->getType()->isVoidTy())
State.set(this, V);
- setMetadata(V);
}
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
@@ -1318,10 +1318,10 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
setFlags(V);
+ applyMetadata(V);
if (!V->getType()->isVoidTy())
State.set(this, V);
- setMetadata(V);
}
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
@@ -1518,7 +1518,7 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
State.set(this, Sel);
if (isa<FPMathOperator>(Sel))
setFlags(cast<Instruction>(Sel));
- setMetadata(Sel);
+ applyMetadata(Sel);
}
InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
@@ -1649,12 +1649,13 @@ void VPWidenRecipe::execute(VPTransformState &State) {
Value *V = Builder.CreateNAryOp(Opcode, Ops);
- if (auto *VecOp = dyn_cast<Instruction>(V))
+ if (auto *VecOp = dyn_cast<Instruction>(V)) {
setFlags(VecOp);
+ applyMetadata(V);
+ }
// Use this vector value for all users of the original instruction.
State.set(this, V);
- setMetadata(V);
break;
}
case Instruction::ExtractValue: {
@@ -1686,8 +1687,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
} else {
C = Builder.CreateICmp(getPredicate(), A, B);
}
+ applyMetadata(C);
State.set(this, C);
- setMetadata(C);
break;
}
default:
@@ -1805,7 +1806,7 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
State.set(this, Cast);
if (auto *CastOp = dyn_cast<Instruction>(Cast)) {
setFlags(CastOp);
- setMetadata(CastOp);
+ applyMetadata(CastOp);
}
}
@@ -2759,7 +2760,7 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
}
// Add metadata to the load, but setVectorValue to the reverse shuffle.
State.addNewMetadata(NewLI, LI);
- setMetadata(NewLI);
+ applyMetadata(NewLI);
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
State.set(this, NewLI);
@@ -2820,7 +2821,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
NewLI->addParamAttr(
0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
State.addNewMetadata(NewLI, LI);
- setMetadata(NewLI);
+ applyMetadata(NewLI);
Instruction *Res = NewLI;
if (isReverse())
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
@@ -2897,7 +2898,7 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
State.addNewMetadata(NewSI, SI);
- setMetadata(NewSI);
+ applyMetadata(NewSI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2945,7 +2946,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
NewSI->addParamAttr(
1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
State.addNewMetadata(NewSI, SI);
- setMetadata(NewSI);
+ applyMetadata(NewSI);
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7dba249085421..445bcac5919fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -70,8 +70,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenIntOrFpInductionRecipe(
Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
} else {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- getMetadataToPropagate(Inst, Metadata);
+ auto Metadata = VPRecipeBuilder::getMetadataToPropagate(Inst);
assert(isa<VPInstruction>(&Ingredient) &&
"only VPInstructions expected here");
assert(!isa<PHINode>(Inst) && "phis should be handled above");
>From 7ad19f7ddb15fe39f5f8375866eb99c0b71b4e7a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 21 Apr 2025 19:00:12 +0100
Subject: [PATCH 3/4] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 +++++++++----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 41 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 ++++----
.../Transforms/Vectorize/VPlanTest.cpp | 7 ++--
5 files changed, 52 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 522e1232a562c..4dfe68e0ae0a7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8599,13 +8599,11 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
}
if (LoadInst *Load = dyn_cast<LoadInst>(I))
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
- getMetadataToPropagate(Load),
I->getDebugLoc());
StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
- Reverse, getMetadataToPropagate(Store),
- I->getDebugLoc());
+ Reverse, I->getDebugLoc());
}
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0692ad4e3bbfb..2735c68e1b3e8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1199,10 +1199,11 @@ class VPIRMetadata {
protected:
VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
+ VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
public:
- /// Add all metadata to \p V if it is an instruction.
- void applyMetadata(Value *V) const;
+ /// Add all metadata to \p I.
+ void applyMetadata(Instruction &I) const;
/// Return the IR metadata.
MDArrayRef getMetadata() const { return Metadata; }
@@ -1337,10 +1338,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
- MDArrayRef Metadata, DebugLoc DL = {})
+ DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty) {
+ VPIRMetadata({}), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -2651,9 +2651,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
std::initializer_list<VPValue *> Operands,
- bool Consecutive, bool Reverse, MDArrayRef Metadata,
- DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
+ bool Consecutive, bool Reverse, DebugLoc DL)
+ : VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
}
@@ -2711,18 +2710,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
/// optional mask.
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
- bool Consecutive, bool Reverse, MDArrayRef Metadata,
- DebugLoc DL)
+ bool Consecutive, bool Reverse, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
- Reverse, Metadata, DL),
+ Reverse, DL),
VPValue(this, &Load) {
setMask(Mask);
}
VPWidenLoadRecipe *clone() override {
- auto *Copy = new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, Reverse,
- getMetadata(), getDebugLoc());
+ auto *Copy =
+ new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(), getMask(),
+ Consecutive, Reverse, getDebugLoc());
return Copy;
}
@@ -2754,7 +2752,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{L.getAddr(), &EVL}, L.isConsecutive(),
- L.isReverse(), L.getMetadata(), L.getDebugLoc()),
+ L.isReverse(), L.getDebugLoc()),
VPValue(this, &getIngredient()) {
setMask(Mask);
}
@@ -2791,17 +2789,16 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
/// to store to and an optional mask.
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
- VPValue *Mask, bool Consecutive, bool Reverse,
- MDArrayRef Metadata, DebugLoc DL)
+ VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
- Consecutive, Reverse, Metadata, DL) {
+ Consecutive, Reverse, DL) {
setMask(Mask);
}
VPWidenStoreRecipe *clone() override {
- auto *Copy = new VPWidenStoreRecipe(
- cast<StoreInst>(Ingredient), getAddr(), getStoredValue(), getMask(),
- Consecutive, Reverse, getMetadata(), getDebugLoc());
+ auto *Copy = new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
+ getStoredValue(), getMask(),
+ Consecutive, Reverse, getDebugLoc());
return Copy;
}
@@ -2836,8 +2833,7 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
{S.getAddr(), S.getStoredValue(), &EVL},
- S.isConsecutive(), S.isReverse(), S.getMetadata(),
- S.getDebugLoc()) {
+ S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index caa75f71d875e..bc1fbbab7941d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1204,12 +1204,9 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPIRMetadata::applyMetadata(Value *V) const {
- auto *I = dyn_cast<Instruction>(V);
- if (!I)
- return;
+void VPIRMetadata::applyMetadata(Instruction &I) const {
for (const auto &[Kind, Node] : Metadata)
- I->setMetadata(Kind, Node);
+ I.setMetadata(Kind, Node);
}
void VPWidenCallRecipe::execute(VPTransformState &State) {
@@ -1239,7 +1236,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
setFlags(V);
- applyMetadata(V);
+ applyMetadata(*V);
if (!V->getType()->isVoidTy())
State.set(this, V);
@@ -1318,7 +1315,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
setFlags(V);
- applyMetadata(V);
+ applyMetadata(*V);
if (!V->getType()->isVoidTy())
State.set(this, V);
@@ -1516,9 +1513,11 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
State.set(this, Sel);
- if (isa<FPMathOperator>(Sel))
- setFlags(cast<Instruction>(Sel));
- applyMetadata(Sel);
+ if (auto *I = dyn_cast<Instruction>(Sel)) {
+ if (isa<FPMathOperator>(I))
+ setFlags(I);
+ applyMetadata(*I);
+ }
}
InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
@@ -1651,7 +1650,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
if (auto *VecOp = dyn_cast<Instruction>(V)) {
setFlags(VecOp);
- applyMetadata(V);
+ applyMetadata(*VecOp);
}
// Use this vector value for all users of the original instruction.
@@ -1687,7 +1686,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
} else {
C = Builder.CreateICmp(getPredicate(), A, B);
}
- applyMetadata(C);
+ if (auto *I = dyn_cast<Instruction>(C))
+ applyMetadata(*I);
State.set(this, C);
break;
}
@@ -1806,7 +1806,7 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
State.set(this, Cast);
if (auto *CastOp = dyn_cast<Instruction>(Cast)) {
setFlags(CastOp);
- applyMetadata(CastOp);
+ applyMetadata(*CastOp);
}
}
@@ -2749,7 +2749,7 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
}
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateGather);
- Value *NewLI;
+ Instruction *NewLI;
if (CreateGather) {
NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,
"wide.masked.gather");
@@ -2762,10 +2762,9 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
}
// Add metadata to the load, but setVectorValue to the reverse shuffle.
State.addNewMetadata(NewLI, LI);
- applyMetadata(NewLI);
- if (Reverse)
- NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
- State.set(this, NewLI);
+ applyMetadata(*NewLI);
+ State.set(this,
+ Reverse ? Builder.CreateVectorReverse(NewLI, "reverse") : NewLI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2823,7 +2822,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
NewLI->addParamAttr(
0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
State.addNewMetadata(NewLI, LI);
- applyMetadata(NewLI);
+ applyMetadata(*NewLI);
Instruction *Res = NewLI;
if (isReverse())
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
@@ -2900,7 +2899,7 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
State.addNewMetadata(NewSI, SI);
- applyMetadata(NewSI);
+ applyMetadata(*NewSI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2948,7 +2947,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
NewSI->addParamAttr(
1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
State.addNewMetadata(NewSI, SI);
- applyMetadata(NewSI);
+ applyMetadata(*NewSI);
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 445bcac5919fc..54aa4a44ed09a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -78,13 +78,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
NewRecipe = new VPWidenLoadRecipe(
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, false /*Reverse*/, Metadata,
+ false /*Consecutive*/, false /*Reverse*/,
Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
- Metadata, Ingredient.getDebugLoc());
+ Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -2012,9 +2012,9 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
SmallVector<VPValue *> Ops(Sel->operands());
Ops.push_back(&EVL);
- return new VPWidenIntrinsicRecipe(
- Intrinsic::vp_select, Ops, TypeInfo.inferScalarType(Sel),
- Sel->getMetadata(), Sel->getDebugLoc());
+ return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
+ TypeInfo.inferScalarType(Sel),
+ Sel->getDebugLoc());
})
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
if (VPI->getOpcode() == VPInstruction::FirstOrderRecurrenceSplice) {
@@ -2026,7 +2026,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
Ops.append({MinusOneVPV, &AllOneMask, PrevEVL, &EVL});
return new VPWidenIntrinsicRecipe(Intrinsic::experimental_vp_splice,
Ops, TypeInfo.inferScalarType(VPI),
- {}, VPI->getDebugLoc());
+ VPI->getDebugLoc());
}
VPValue *LHS, *RHS;
@@ -2041,7 +2041,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
// limited to selects whose condition is a header mask.
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
- TypeInfo.inferScalarType(LHS), {}, VPI->getDebugLoc());
+ TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
})
.Default([&](VPRecipeBase *R) { return nullptr; });
}
@@ -2758,7 +2758,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *L = new VPWidenLoadRecipe(
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
- /*Reverse=*/false, {}, LoadGroup->getDebugLoc());
+ /*Reverse=*/false, LoadGroup->getDebugLoc());
L->insertBefore(LoadGroup);
return L;
}
@@ -2789,7 +2789,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *S = new VPWidenStoreRecipe(
*cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos()),
StoreGroup->getAddr(), Res, nullptr, /*Consecutive=*/true,
- /*Reverse=*/false, {}, StoreGroup->getDebugLoc());
+ /*Reverse=*/false, StoreGroup->getDebugLoc());
S->insertBefore(StoreGroup);
StoreGroup->eraseFromParent();
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 277e03cbe8588..9732f8da1e592 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1084,7 +1084,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1202,7 +1202,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
- VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
+ VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_TRUE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1216,8 +1216,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
VPValue *StoredV = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
- VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {},
- {});
+ VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {});
EXPECT_TRUE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_TRUE(Recipe.mayWriteToMemory());
>From ef9bcc98f9f1fbe776f960a27387519e61e206ca Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 22 Apr 2025 11:49:48 +0100
Subject: [PATCH 4/4] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 30 +++-----
.../Transforms/Vectorize/VPRecipeBuilder.h | 5 --
llvm/lib/Transforms/Vectorize/VPlan.h | 68 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 ++--
.../Transforms/Vectorize/VPlanTest.cpp | 11 ++-
5 files changed, 46 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 435f27ae20d48..32c3435ccb38d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8745,7 +8745,6 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
- getMetadataToPropagate(CI),
CI->getDebugLoc());
Function *Variant = nullptr;
@@ -8799,8 +8798,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
}
Ops.push_back(Operands.back());
- return new VPWidenCallRecipe(CI, Variant, Ops, getMetadataToPropagate(CI),
- CI->getDebugLoc());
+ return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
}
return nullptr;
@@ -8838,8 +8836,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
Ops[1] = SafeRHS;
- return new VPWidenRecipe(*I, make_range(Ops.begin(), Ops.end()),
- getMetadataToPropagate(I));
+ return new VPWidenRecipe(*I, make_range(Ops.begin(), Ops.end()));
}
[[fallthrough]];
}
@@ -8885,8 +8882,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
- return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()),
- getMetadataToPropagate(I));
+ return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(Operands);
@@ -8895,8 +8891,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
- return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()),
- getMetadataToPropagate(I));
+ return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
}
};
}
@@ -9101,13 +9096,6 @@ bool VPRecipeBuilder::getScaledReductions(
return false;
}
-SmallVector<std::pair<unsigned, MDNode *>>
-VPRecipeBuilder::getMetadataToPropagate(Instruction *I) {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- ::getMetadataToPropagate(I, Metadata);
- return Metadata;
-}
-
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9180,14 +9168,13 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
make_range(Operands.begin(), Operands.end()));
if (auto *SI = dyn_cast<SelectInst>(Instr)) {
- return new VPWidenSelectRecipe(*SI,
- make_range(Operands.begin(), Operands.end()),
- getMetadataToPropagate(SI));
+ return new VPWidenSelectRecipe(
+ *SI, make_range(Operands.begin(), Operands.end()));
}
if (auto *CI = dyn_cast<CastInst>(Instr)) {
return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
- *CI, getMetadataToPropagate(CI));
+ *CI);
}
return tryToWiden(Instr, Operands);
@@ -9213,8 +9200,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*Reduction, make_range(Ops.begin(), Ops.end()),
- getMetadataToPropagate(Reduction));
+ BinOp = new VPWidenRecipe(*Reduction, make_range(Ops.begin(), Ops.end()));
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index aab34834ae9bf..fd0064a34c4c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -233,11 +233,6 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
-
- /// Returns the metatadata that can be preserved from the original instruction
- /// \p I, including noalias metadata guaranteed by runtime checks.
- static SmallVector<std::pair<unsigned, MDNode *>>
- getMetadataToPropagate(Instruction *I);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b66abcd47706d..360cca60a0ec1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1190,23 +1190,18 @@ struct VPIRPhi : public VPIRInstruction {
#endif
};
-using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
-
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be proagated.
class VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
protected:
- VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
+ VPIRMetadata() {}
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
public:
/// Add all metadata to \p I.
void applyMetadata(Instruction &I) const;
-
- /// Return the IR metadata.
- MDArrayRef getMetadata() const { return Metadata; }
};
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -1219,21 +1214,19 @@ class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
protected:
template <typename IterT>
VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
- iterator_range<IterT> Operands, MDArrayRef Metadata)
- : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), VPIRMetadata(Metadata),
+ iterator_range<IterT> Operands)
+ : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), VPIRMetadata(I),
Opcode(I.getOpcode()) {}
public:
template <typename IterT>
- VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands,
- MDArrayRef Metadata)
- : VPWidenRecipe(VPDef::VPWidenSC, I, Operands, Metadata) {}
+ VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
+ : VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
- auto *R =
- new VPWidenRecipe(*getUnderlyingInstr(), operands(), getMetadata());
+ auto *R = new VPWidenRecipe(*getUnderlyingInstr(), operands());
R->transferFlags(*this);
return R;
}
@@ -1267,15 +1260,15 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI, MDArrayRef Metadata)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
- VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ CastInst &UI)
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI),
+ Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPIRMetadata({}),
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPIRMetadata(),
Opcode(Opcode), ResultTy(ResultTy) {}
~VPWidenCastRecipe() override = default;
@@ -1283,7 +1276,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenCastRecipe *clone() override {
if (auto *UV = getUnderlyingValue())
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
- *cast<CastInst>(UV), getMetadata());
+ *cast<CastInst>(UV));
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
}
@@ -1329,10 +1322,10 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
- MDArrayRef Metadata, DebugLoc DL = {})
+ DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()),
+ VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
@@ -1340,7 +1333,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
ArrayRef<VPValue *> CallArguments, Type *Ty,
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VPIRMetadata({}), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
+ VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1356,7 +1349,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
return new VPWidenIntrinsicRecipe(*cast<CallInst>(getUnderlyingValue()),
VectorIntrinsicID, {op_begin(), op_end()},
- ResultTy, getMetadata(), getDebugLoc());
+ ResultTy, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
@@ -1404,11 +1397,10 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCallRecipe(Value *UV, Function *Variant,
- ArrayRef<VPValue *> CallArguments, MDArrayRef Metadata,
- DebugLoc DL = {})
+ ArrayRef<VPValue *> CallArguments, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
- VPIRMetadata(Metadata), Variant(Variant) {
+ VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1418,8 +1410,7 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenCallRecipe *clone() override {
return new VPWidenCallRecipe(getUnderlyingValue(), Variant,
- {op_begin(), op_end()}, getMetadata(),
- getDebugLoc());
+ {op_begin(), op_end()}, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
@@ -1497,16 +1488,15 @@ class VPHistogramRecipe : public VPRecipeBase {
/// A recipe for widening select instructions.
struct VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
template <typename IterT>
- VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands,
- MDArrayRef Metadata)
+ VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands)
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
- VPIRMetadata(Metadata) {}
+ VPIRMetadata(I) {}
~VPWidenSelectRecipe() override = default;
VPWidenSelectRecipe *clone() override {
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
- operands(), getMetadata());
+ operands());
}
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
@@ -2718,10 +2708,9 @@ struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
}
VPWidenLoadRecipe *clone() override {
- auto *Copy =
- new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(), getMask(),
- Consecutive, Reverse, getDebugLoc());
- return Copy;
+ return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
+ getMask(), Consecutive, Reverse,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC);
@@ -2796,10 +2785,9 @@ struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
}
VPWidenStoreRecipe *clone() override {
- auto *Copy = new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
- getStoredValue(), getMask(),
- Consecutive, Reverse, getDebugLoc());
- return Copy;
+ return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
+ getStoredValue(), getMask(), Consecutive,
+ Reverse, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c196d0601ae46..f2dc68b2ea8b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -70,7 +70,6 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenIntOrFpInductionRecipe(
Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
} else {
- auto Metadata = VPRecipeBuilder::getMetadataToPropagate(Inst);
assert(isa<VPInstruction>(&Ingredient) &&
"only VPInstructions expected here");
assert(!isa<PHINode>(Inst) && "phis should be handled above");
@@ -94,16 +93,14 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
{Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(),
- Metadata, CI->getDebugLoc());
+ CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
- NewRecipe =
- new VPWidenSelectRecipe(*SI, Ingredient.operands(), Metadata);
+ NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands());
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
- NewRecipe =
- new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
- CI->getType(), *CI, Metadata);
+ NewRecipe = new VPWidenCastRecipe(
+ CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
} else {
- NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), Metadata);
+ NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands());
}
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 9732f8da1e592..17f74c4b41496 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -929,7 +929,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()), {});
+ VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()));
EXPECT_TRUE(isa<VPUser>(&WidenR));
VPRecipeBase *WidenRBase = &WidenR;
EXPECT_TRUE(isa<VPUser>(WidenRBase));
@@ -978,7 +978,7 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
Args.push_back(Op2);
Args.push_back(Op3);
VPWidenSelectRecipe WidenSelectR(*SelectI,
- make_range(Args.begin(), Args.end()), {});
+ make_range(Args.begin(), Args.end()));
EXPECT_TRUE(isa<VPUser>(&WidenSelectR));
VPRecipeBase *BaseR = &WidenSelectR;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1111,7 +1111,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end()), {});
+ VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end()));
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1130,8 +1130,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
- VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()),
- {});
+ VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()));
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1315,7 +1314,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
- new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end()), {});
+ new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end()));
VPBB1->appendRecipe(WidenR);
{
More information about the llvm-commits
mailing list