[llvm] 577199f - Reapply "[VPlan] Set branch weight metadata on middle term in VPlan (NFC) (#143035)"
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 14 09:22:02 PDT 2025
Author: Florian Hahn
Date: 2025-06-14T17:20:46+01:00
New Revision: 577199f9221ebc805a69372a2b19f4c8ebaf1daf
URL: https://github.com/llvm/llvm-project/commit/577199f9221ebc805a69372a2b19f4c8ebaf1daf
DIFF: https://github.com/llvm/llvm-project/commit/577199f9221ebc805a69372a2b19f4c8ebaf1daf.diff
LOG: Reapply "[VPlan] Set branch weight metadata on middle term in VPlan (NFC) (#143035)"
This reverts commit 0604dc199c019b23746f4a54885ba0c75569cdae.
The recommitted version addresses post-commit comments and adjusts the
place the branch weights are added. It now runs before VPlans are optimized
for VF and UF, which may remove the vector loop region, causing a crash
trying to get the middle block after that. Test case added in
72f99b75afc12bb.
Original message:
Manage branch weights for the BranchOnCond in the middle block in VPlan.
This requires updating VPInstruction to inherit from VPIRMetadata, which
in general makes sense as there are a number of opcodes that could take
metadata.
There are other branches (part of the skeleton) that also need branch
weights adding.
PR: https://github.com/llvm/llvm-project/pull/143035
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7c006ae326ecb..9b5ad16589539 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7300,6 +7300,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
OrigLoop->getHeader()->getContext());
VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
+ if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
+ VPlanTransforms::runPass(VPlanTransforms::addBranchWeightToMiddleTerminator,
+ BestVPlan, BestVF);
VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
VPlanTransforms::narrowInterleaveGroups(
@@ -7309,11 +7312,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::convertToConcreteRecipes(BestVPlan,
*Legal->getWidestInductionType());
- // Retrieve and store the middle block before dissolving regions. Regions are
- // dissolved after optimizing for VF and UF, which completely removes unneeded
- // loop regions first.
- VPBasicBlock *MiddleVPBB =
- BestVPlan.getVectorLoopRegion() ? BestVPlan.getMiddleBlock() : nullptr;
+ // Regions are dissolved after optimizing for VF and UF, which completely
+ // removes unneeded loop regions first.
VPlanTransforms::dissolveLoopRegions(BestVPlan);
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
@@ -7456,20 +7456,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ILV.printDebugTracesAtEnd();
- // 4. Adjust branch weight of the branch in the middle block.
- if (HeaderVPBB) {
- auto *MiddleTerm =
- cast<BranchInst>(State.CFG.VPBB2IRBB[MiddleVPBB]->getTerminator());
- if (MiddleTerm->isConditional() &&
- hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
- // Assume that `Count % VectorTripCount` is equally distributed.
- unsigned TripCount = BestVPlan.getUF() * State.VF.getKnownMinValue();
- assert(TripCount > 0 && "trip count should not be zero");
- const uint32_t Weights[] = {1, TripCount - 1};
- setBranchWeights(*MiddleTerm, Weights, /*IsExpected=*/false);
- }
- }
-
return ExpandedSCEVs;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 53619b39219e6..5a3c4a514a5dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -882,11 +882,40 @@ template <unsigned PartOpIdx> class VPUnrollPartAccessor {
unsigned getUnrollPart(VPUser &U) const;
};
+/// Helper to manage IR metadata for recipes. It filters out metadata that
+/// cannot be propagated.
+class VPIRMetadata {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+
+public:
+ VPIRMetadata() {}
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I.
+ VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
+ VPIRMetadata(Instruction &I, LoopVersioning *LVer);
+
+ /// Copy constructor for cloning.
+ VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
+
+ /// Add all metadata to \p I.
+ void applyMetadata(Instruction &I) const;
+
+ /// Add metadata with kind \p Kind and \p Node.
+ void addMetadata(unsigned Kind, MDNode *Node) {
+ Metadata.emplace_back(Kind, Node);
+ }
+};
+
/// This is a concrete Recipe that models a single VPlan-level instruction.
/// While as any Recipe it may generate a sequence of IR instructions when
/// executed, these instructions would always form a single-def expression as
/// the VPInstruction is also a single def-use vertex.
class VPInstruction : public VPRecipeWithIRFlags,
+ public VPIRMetadata,
public VPUnrollPartAccessor<1> {
friend class VPlanSlp;
@@ -976,7 +1005,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- Opcode(Opcode), Name(Name.str()) {}
+ VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {}
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL = {},
@@ -1268,29 +1297,6 @@ struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors {
const VPRecipeBase *getAsRecipe() const override { return this; }
};
-/// Helper to manage IR metadata for recipes. It filters out metadata that
-/// cannot be propagated.
-class VPIRMetadata {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
-
-public:
- VPIRMetadata() {}
-
- /// Adds metatadata that can be preserved from the original instruction
- /// \p I.
- VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
-
- /// Adds metatadata that can be preserved from the original instruction
- /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
- VPIRMetadata(Instruction &I, LoopVersioning *LVer);
-
- /// Copy constructor for cloning.
- VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
-
- /// Add all metadata to \p I.
- void applyMetadata(Instruction &I) const;
-};
-
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c64bda167b854..3bdfa6724f691 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -409,7 +409,7 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL,
const Twine &Name)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL),
- Opcode(Opcode), Name(Name.str()) {
+ VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
}
@@ -590,7 +590,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case VPInstruction::BranchOnCond: {
Value *Cond = State.get(getOperand(0), VPLane(0));
- return createCondBranch(Cond, getParent(), State);
+ auto *Br = createCondBranch(Cond, getParent(), State);
+ applyMetadata(*Br);
+ return Br;
}
case VPInstruction::BranchOnCount: {
// First create the compare.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dc3c7bfe5cd1a..44a72755b9cf8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -31,6 +31,7 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/TypeSize.h"
@@ -3203,3 +3204,25 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
Plan.getOrAddLiveIn(ConstantInt::get(CanIV->getScalarType(), 1)));
removeDeadRecipes(Plan);
}
+
+/// Add branch weight metadata, if the \p Plan's middle block is terminated by a
+/// BranchOnCond recipe.
+void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
+ ElementCount VF) {
+ VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
+ auto *MiddleTerm =
+ dyn_cast_or_null<VPInstruction>(MiddleVPBB->getTerminator());
+ // Only add branch metadata if there is a (conditional) terminator.
+ if (!MiddleTerm)
+ return;
+
+ assert(MiddleTerm->getOpcode() == VPInstruction::BranchOnCond &&
+ "must have a BranchOnCond");
+ // Assume that `TripCount % VectorStep ` is equally distributed.
+ unsigned VectorStep = Plan.getUF() * VF.getKnownMinValue();
+ assert(VectorStep > 0 && "trip count should not be zero");
+ MDBuilder MDB(Plan.getScalarHeader()->getIRBasicBlock()->getContext());
+ MDNode *BranchWeights =
+ MDB.createBranchWeights({1, VectorStep - 1}, /*IsExpected=*/false);
+ MiddleTerm->addMetadata(LLVMContext::MD_prof, BranchWeights);
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 34e2de4eb3b74..5a03bdb7c6882 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -234,6 +234,10 @@ struct VPlanTransforms {
/// removed in the future.
static DenseMap<VPBasicBlock *, VPValue *>
introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+
+ /// Add branch weight metadata, if the \p Plan's middle block is terminated by
+ /// a BranchOnCond recipe.
+ static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF);
};
} // namespace llvm
More information about the llvm-commits
mailing list