[llvm] 0604dc1 - Revert "[VPlan] Set branch weight metadata on middle term in VPlan (NFC) (#143035)"
Hans Wennborg via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 12 04:53:45 PDT 2025
Author: Hans Wennborg
Date: 2025-06-12T13:52:05+02:00
New Revision: 0604dc199c019b23746f4a54885ba0c75569cdae
URL: https://github.com/llvm/llvm-project/commit/0604dc199c019b23746f4a54885ba0c75569cdae
DIFF: https://github.com/llvm/llvm-project/commit/0604dc199c019b23746f4a54885ba0c75569cdae.diff
LOG: Revert "[VPlan] Set branch weight metadata on middle term in VPlan (NFC) (#143035)"
This caused assertion failures:
llvm/lib/Transforms/Vectorize/VPlan.h:4021:
llvm::VPBasicBlock* llvm::VPlan::getMiddleBlock():
Assertion `LoopRegion && "cannot call the function after vector loop region has been removed"' failed.
See comment on the PR.
> Manage branch weights for the BranchOnCond in the middle block in VPlan.
> This requires updating VPInstruction to inherit from VPIRMetadata, which
> in general makes sense as there are a number of opcodes that could take
> metadata.
>
> There are other branches (part of the skeleton) that also need branch
> weights adding.
>
> PR: https://github.com/llvm/llvm-project/pull/143035
This reverts commit db8d34db26e9ea92c08d6e813eca9cce40c48478.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 474f856d20461..8177b76ad5bdf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7272,33 +7272,6 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
}
-/// Add branch weight metadata, if the \p Plan's middle block is terminated by a
-/// BranchOnCond recipe.
-static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
- Loop *OrigLoop) {
- // 4. Adjust branch weight of the branch in the middle block.
- Instruction *LatchTerm = OrigLoop->getLoopLatch()->getTerminator();
- if (!hasBranchWeightMD(*LatchTerm))
- return;
-
- VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
- auto *MiddleTerm =
- dyn_cast_or_null<VPInstruction>(MiddleVPBB->getTerminator());
- // Only add branch metadata if there is a (conditional) terminator.
- if (!MiddleTerm)
- return;
-
- assert(MiddleTerm->getOpcode() == VPInstruction::BranchOnCond &&
- "must have a BranchOnCond");
- // Assume that `Count % VectorTripCount` is equally distributed.
- unsigned TripCount = Plan.getUF() * VF.getKnownMinValue();
- assert(TripCount > 0 && "trip count should not be zero");
- MDBuilder MDB(LatchTerm->getContext());
- MDNode *BranchWeights =
- MDB.createBranchWeights({1, TripCount - 1}, /*IsExpected=*/false);
- MiddleTerm->addMetadata(LLVMContext::MD_prof, BranchWeights);
-}
-
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue) {
@@ -7321,8 +7294,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::convertToConcreteRecipes(BestVPlan,
*Legal->getWidestInductionType());
-
- addBranchWeightToMiddleTerminator(BestVPlan, BestVF, OrigLoop);
+ // Retrieve and store the middle block before dissolving regions. Regions are
+ // dissolved after optimizing for VF and UF, which completely removes unneeded
+ // loop regions first.
+ VPBasicBlock *MiddleVPBB =
+ BestVPlan.getVectorLoopRegion() ? BestVPlan.getMiddleBlock() : nullptr;
VPlanTransforms::dissolveLoopRegions(BestVPlan);
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
@@ -7465,6 +7441,20 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ILV.printDebugTracesAtEnd();
+ // 4. Adjust branch weight of the branch in the middle block.
+ if (HeaderVPBB) {
+ auto *MiddleTerm =
+ cast<BranchInst>(State.CFG.VPBB2IRBB[MiddleVPBB]->getTerminator());
+ if (MiddleTerm->isConditional() &&
+ hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
+ // Assume that `Count % VectorTripCount` is equally distributed.
+ unsigned TripCount = BestVPlan.getUF() * State.VF.getKnownMinValue();
+ assert(TripCount > 0 && "trip count should not be zero");
+ const uint32_t Weights[] = {1, TripCount - 1};
+ setBranchWeights(*MiddleTerm, Weights, /*IsExpected=*/false);
+ }
+ }
+
return ExpandedSCEVs;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 468284168e9ca..acc861b991975 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -882,39 +882,11 @@ template <unsigned PartOpIdx> class VPUnrollPartAccessor {
unsigned getUnrollPart(VPUser &U) const;
};
-/// Helper to manage IR metadata for recipes. It filters out metadata that
-/// cannot be propagated.
-class VPIRMetadata {
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
-
-public:
- VPIRMetadata() {}
-
- /// Adds metatadata that can be preserved from the original instruction
- /// \p I.
- VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
-
- /// Adds metatadata that can be preserved from the original instruction
- /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
- VPIRMetadata(Instruction &I, LoopVersioning *LVer);
-
- /// Copy constructor for cloning.
- VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
-
- /// Add all metadata to \p I.
- void applyMetadata(Instruction &I) const;
-
- void addMetadata(unsigned Kind, MDNode *Node) {
- Metadata.emplace_back(Kind, Node);
- }
-};
-
/// This is a concrete Recipe that models a single VPlan-level instruction.
/// While as any Recipe it may generate a sequence of IR instructions when
/// executed, these instructions would always form a single-def expression as
/// the VPInstruction is also a single def-use vertex.
class VPInstruction : public VPRecipeWithIRFlags,
- public VPIRMetadata,
public VPUnrollPartAccessor<1> {
friend class VPlanSlp;
@@ -1004,7 +976,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {}
+ Opcode(Opcode), Name(Name.str()) {}
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL = {},
@@ -1296,6 +1268,29 @@ struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors {
const VPRecipeBase *getAsRecipe() const override { return this; }
};
+/// Helper to manage IR metadata for recipes. It filters out metadata that
+/// cannot be propagated.
+class VPIRMetadata {
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+
+public:
+ VPIRMetadata() {}
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I.
+ VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+
+ /// Adds metatadata that can be preserved from the original instruction
+ /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
+ VPIRMetadata(Instruction &I, LoopVersioning *LVer);
+
+ /// Copy constructor for cloning.
+ VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
+
+ /// Add all metadata to \p I.
+ void applyMetadata(Instruction &I) const;
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8863a3fb4b31d..aa6b13c217bd1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -410,7 +410,7 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL,
const Twine &Name)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL),
- VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {
+ Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
}
@@ -591,9 +591,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case VPInstruction::BranchOnCond: {
Value *Cond = State.get(getOperand(0), VPLane(0));
- auto *Br = createCondBranch(Cond, getParent(), State);
- applyMetadata(*Br);
- return Br;
+ return createCondBranch(Cond, getParent(), State);
}
case VPInstruction::BranchOnCount: {
// First create the compare.
More information about the llvm-commits
mailing list