[llvm] 5d136f9 - [VPlan] Manage instruction metadata in VPlan. (#135272)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 24 03:57:23 PDT 2025
Author: Florian Hahn
Date: 2025-04-24T11:57:19+01:00
New Revision: 5d136f90a9806daf6d9ac905734a27974fc8f1b7
URL: https://github.com/llvm/llvm-project/commit/5d136f90a9806daf6d9ac905734a27974fc8f1b7
DIFF: https://github.com/llvm/llvm-project/commit/5d136f90a9806daf6d9ac905734a27974fc8f1b7.diff
LOG: [VPlan] Manage instruction metadata in VPlan. (#135272)
Add a new helper to manage IR metadata that can be progated to generated
instructions for recipes.
This helps to remove a number of remaining uses of getUnderlyingInstr
during VPlan execution.
PR: https://github.com/llvm/llvm-project/pull/135272
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanHelpers.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fa2d95a44609a..701e4bfe0623e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -363,17 +363,6 @@ void VPTransformState::addNewMetadata(Instruction *To,
LVer->annotateInstWithNoAlias(To, Orig);
}
-void VPTransformState::addMetadata(Value *To, Instruction *From) {
- // No source instruction to transfer metadata from?
- if (!From)
- return;
-
- if (Instruction *ToI = dyn_cast<Instruction>(To)) {
- propagateMetadata(ToI, From);
- addNewMetadata(ToI, From);
- }
-}
-
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
const DILocation *DIL = DL;
// When a FSDiscriminator is enabled, we don't need to add the multiply
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b662aa0331d67..bd6e15d3fb7a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1190,18 +1190,33 @@ struct VPIRPhi : public VPIRInstruction {
#endif
};
+/// Helper to manage IR metadata for recipes. It filters out metadata that
+/// cannot be propagated.
+class VPIRMetadata {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+
+protected:
+ VPIRMetadata() {}
+ VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+
+public:
+ /// Add all metadata to \p I.
+ void applyMetadata(Instruction &I) const;
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
/// vectorized version of itself.
-class VPWidenRecipe : public VPRecipeWithIRFlags {
+class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
unsigned Opcode;
protected:
template <typename IterT>
VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
iterator_range<IterT> Operands)
- : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), Opcode(I.getOpcode()) {}
+ : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), VPIRMetadata(I),
+ Opcode(I.getOpcode()) {}
public:
template <typename IterT>
@@ -1236,7 +1251,7 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
};
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
-class VPWidenCastRecipe : public VPRecipeWithIRFlags {
+class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
@@ -1246,15 +1261,15 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst &UI)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
- ResultTy(ResultTy) {
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI),
+ Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
- ResultTy(ResultTy) {}
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), VPIRMetadata(),
+ Opcode(Opcode), ResultTy(ResultTy) {}
~VPWidenCastRecipe() override = default;
@@ -1288,7 +1303,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
};
/// A recipe for widening vector intrinsics.
-class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
+class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
@@ -1309,7 +1324,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
ArrayRef<VPValue *> CallArguments, Type *Ty,
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
@@ -1318,7 +1333,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
ArrayRef<VPValue *> CallArguments, Type *Ty,
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
+ VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1374,7 +1389,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
};
/// A recipe for widening Call instructions using library calls.
-class VPWidenCallRecipe : public VPRecipeWithIRFlags {
+class VPWidenCallRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
/// between a given VF and the chosen vectorized variant, so there will be a
///
diff erent VPlan for each VF with a valid variant.
@@ -1385,7 +1400,7 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags {
ArrayRef<VPValue *> CallArguments, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
- Variant(Variant) {
+ VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1471,10 +1486,11 @@ class VPHistogramRecipe : public VPRecipeBase {
};
/// A recipe for widening select instructions.
-struct VPWidenSelectRecipe : public VPRecipeWithIRFlags {
+struct VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
template <typename IterT>
VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I) {}
+ : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
+ VPIRMetadata(I) {}
~VPWidenSelectRecipe() override = default;
@@ -2602,7 +2618,7 @@ class VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// A common base class for widening memory operations. An optional mask can be
/// provided as the last operand.
-class VPWidenMemoryRecipe : public VPRecipeBase {
+class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
protected:
Instruction &Ingredient;
@@ -2626,8 +2642,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase {
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
std::initializer_list<VPValue *> Operands,
bool Consecutive, bool Reverse, DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), Ingredient(I), Consecutive(Consecutive),
- Reverse(Reverse) {
+ : VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
+ Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index bebea1915690f..e860595b473a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -290,13 +290,6 @@ struct VPTransformState {
/// vector loop.
void addNewMetadata(Instruction *To, const Instruction *Orig);
- /// Add metadata from one instruction to another.
- ///
- /// This includes both the original MDs from \p From and additional ones (\see
- /// addNewMetadata). Use this for *newly created* instructions in the vector
- /// loop.
- void addMetadata(Value *To, Instruction *From);
-
/// Set the debug location in the builder using the debug location \p DL.
void setDebugLocFrom(DebugLoc DL);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6720140608d15..8ec092ce9a905 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1205,6 +1205,11 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPIRMetadata::applyMetadata(Instruction &I) const {
+ for (const auto &[Kind, Node] : Metadata)
+ I.setMetadata(Kind, Node);
+}
+
void VPWidenCallRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
assert(Variant != nullptr && "Can't create vector function.");
@@ -1231,11 +1236,11 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
applyFlags(*V);
+ applyMetadata(*V);
V->setCallingConv(Variant->getCallingConv());
if (!V->getType()->isVoidTy())
State.set(this, V);
- State.addMetadata(V, CI);
}
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
@@ -1311,10 +1316,10 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
applyFlags(*V);
+ applyMetadata(*V);
if (!V->getType()->isVoidTy())
State.set(this, V);
- State.addMetadata(V, CI);
}
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
@@ -1509,9 +1514,11 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
State.set(this, Sel);
- if (isa<FPMathOperator>(Sel))
- applyFlags(*cast<Instruction>(Sel));
- State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
+ if (auto *I = dyn_cast<Instruction>(Sel)) {
+ if (isa<FPMathOperator>(I))
+ applyFlags(*I);
+ applyMetadata(*I);
+ }
}
InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
@@ -1642,12 +1649,13 @@ void VPWidenRecipe::execute(VPTransformState &State) {
Value *V = Builder.CreateNAryOp(Opcode, Ops);
- if (auto *VecOp = dyn_cast<Instruction>(V))
+ if (auto *VecOp = dyn_cast<Instruction>(V)) {
applyFlags(*VecOp);
+ applyMetadata(*VecOp);
+ }
// Use this vector value for all users of the original instruction.
State.set(this, V);
- State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
break;
}
case Instruction::ExtractValue: {
@@ -1679,8 +1687,9 @@ void VPWidenRecipe::execute(VPTransformState &State) {
} else {
C = Builder.CreateICmp(getPredicate(), A, B);
}
+ if (auto *I = dyn_cast<Instruction>(C))
+ applyMetadata(*I);
State.set(this, C);
- State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
break;
}
default:
@@ -1796,9 +1805,10 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
Value *A = State.get(Op);
Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
State.set(this, Cast);
- State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
- if (auto *CastOp = dyn_cast<Instruction>(Cast))
+ if (auto *CastOp = dyn_cast<Instruction>(Cast)) {
applyFlags(*CastOp);
+ applyMetadata(*CastOp);
+ }
}
InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
@@ -2750,8 +2760,10 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
} else {
NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");
}
- // Add metadata to the load, but setVectorValue to the reverse shuffle.
- State.addMetadata(NewLI, cast<LoadInst>(&Ingredient));
+ // Add metadata to the load, but set the result to the reverse shuffle, if
+ // needed.
+ State.addNewMetadata(cast<Instruction>(NewLI), &Ingredient);
+ applyMetadata(*cast<Instruction>(NewLI));
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
State.set(this, NewLI);
@@ -2810,7 +2822,8 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
}
NewLI->addParamAttr(
0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- State.addMetadata(NewLI, cast<LoadInst>(&Ingredient));
+ State.addNewMetadata(NewLI, &Ingredient);
+ applyMetadata(*NewLI);
Instruction *Res = NewLI;
if (isReverse())
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
@@ -2884,7 +2897,8 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- State.addMetadata(NewSI, cast<StoreInst>(&Ingredient));
+ State.addNewMetadata(NewSI, &Ingredient);
+ applyMetadata(*NewSI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2930,7 +2944,8 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
}
NewSI->addParamAttr(
1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- State.addMetadata(NewSI, cast<StoreInst>(&Ingredient));
+ State.addNewMetadata(NewSI, &Ingredient);
+ applyMetadata(*NewSI);
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
More information about the llvm-commits
mailing list