[llvm] [VPlan] Add exit phi operands during initial construction (NFC). (PR #136455)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 21 13:30:01 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/136455
>From e9313098df85360f222c9f39117d1c5a9eb41037 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 23 Mar 2025 14:16:27 +0000
Subject: [PATCH 1/3] [VPlan] Add exit phi operands during initial construction
(NFC).
Add incoming exit phi operands during the initial VPlan construction.
This ensures all users are added to the initial VPlan and is also needed
in preparation to retaining exiting edges during initial construction.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +---
.../Vectorize/VPlanConstruction.cpp | 19 +++++++++++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 28 +++++++++++--------
4 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7a5f618d09e95..f9a7769b76b94 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9392,11 +9392,7 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
continue;
}
- PHINode &ExitPhi = ExitIRI->getIRPhi();
- BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
- Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB);
- VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
- ExitIRI->addOperand(V);
+ VPValue *V = ExitIRI->getOperand(0);
if (V->isLiveIn())
continue;
assert(V->getDefiningRecipe()->getParent()->getEnclosingLoopRegion() &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 9fcccfcf8117f..95f0a91113fb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -352,6 +352,19 @@ std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG(
Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
Plan->getEntry()->setPlan(&*Plan);
+ // Add incoming operands for the VPIRInstructions wrapping the exit phis.
+ for (auto *EB : Plan->getExitBlocks()) {
+ for (VPRecipeBase &R : *EB) {
+ auto *PhiR = dyn_cast<VPIRPhi>(&R);
+ if (!PhiR)
+ break;
+ PHINode &Phi = PhiR->getIRPhi();
+ for (BasicBlock *Pred : predecessors(EB->getIRBasicBlock()))
+ PhiR->addOperand(
+ getOrCreateVPOperand(Phi.getIncomingValueForBlock(Pred)));
+ }
+ }
+
for (const auto &[IRBB, VPB] : BB2VPBB)
VPB2IRBB[VPB] = IRBB;
@@ -464,6 +477,12 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy,
VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader());
if (!RequiresScalarEpilogueCheck) {
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+ // The exit blocks are dead, remove any recipes to make sure no users remain
+ // that may pessimize transforms.
+ for (auto *EB : Plan.getExitBlocks()) {
+ for (VPRecipeBase &R : make_early_inc_range(*EB))
+ R.eraseFromParent();
+ }
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2cc558f49ccce..9e68ab55783ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1139,7 +1139,7 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF,
void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
assert(isa<PHINode>(getInstruction()) &&
"can only add exiting operands to phi nodes");
- assert(getNumOperands() == 1 && "must have a single operand");
+ assert(getNumOperands() > 0 && "must have at least one operand");
VPValue *Exiting = getOperand(0);
if (!Exiting->isLiveIn()) {
LLVMContext &Ctx = getInstruction().getContext();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b80fe18d1bd66..9f5d76c2c856d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2501,20 +2501,24 @@ void VPlanTransforms::handleUncountableEarlyExit(
if (!ExitIRI)
break;
- PHINode &ExitPhi = ExitIRI->getIRPhi();
- VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi.getIncomingValueForBlock(UncountableExitingBlock));
-
+ unsigned EarlyExitIdx = 0;
if (OrigLoop->getUniqueExitBlock()) {
+ // After the transform, the first incoming value is coming from the
+ // orignial loop latch, while the second operand is from the early exit.
+ // Sawp the phi operands, if the first predecessor in the original IR is
+ // not the loop latch.
+ if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) !=
+ OrigLoop->getLoopLatch())
+ ExitIRI->swapOperands();
+
+ EarlyExitIdx = 1;
// If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
// (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
// which is coming from the original latch.
- VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi.getIncomingValueForBlock(OrigLoop->getLoopLatch()));
- ExitIRI->addOperand(IncomingFromLatch);
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
+ VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
auto IsVector = [](ElementCount VF) { return VF.isVector(); };
// When the VFs are vectors, need to add `extract` to get the incoming value
// from early exit. When the range contains scalar VF, limit the range to
@@ -2522,14 +2526,16 @@ void VPlanTransforms::handleUncountableEarlyExit(
// and vector VFs.
if (!IncomingFromEarlyExit->isLiveIn() &&
LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
+ // Add the incoming value from the early exit.
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
"first.active.lane");
- IncomingFromEarlyExit = EarlyExitB.createNaryOp(
- Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane},
- nullptr, "early.exit.value");
+ ExitIRI->setOperand(
+ EarlyExitIdx,
+ EarlyExitB.createNaryOp(Instruction::ExtractElement,
+ {IncomingFromEarlyExit, FirstActiveLane},
+ nullptr, "early.exit.value"));
}
- ExitIRI->addOperand(IncomingFromEarlyExit);
}
MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {IsEarlyExitTaken});
>From 70523376ae33c458adf311d2f4fee6a163f42e78 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 21 Apr 2025 20:45:59 +0100
Subject: [PATCH 2/3] !fixup address comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index aa8ec1783a8ba..afb18881c7b02 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9370,8 +9370,7 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
}
}
-// Collect VPIRInstructions for phis in the exit blocks that are modeled
-// in VPlan and add the exiting VPValue as operand.
+// Collect VPIRInstructions for phis in the exit block from the latch only.
static SetVector<VPIRInstruction *>
collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
VPlan &Plan) {
@@ -9392,6 +9391,7 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
continue;
}
+ assert(ExitIRI->getNumOperands() == 1 && "must have a single operand");
VPValue *V = ExitIRI->getOperand(0);
if (V->isLiveIn())
continue;
>From d0081f77b585b3b176f67e72d80d06a92b56d0e4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 21 Apr 2025 21:29:22 +0100
Subject: [PATCH 3/3] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++---
.../Vectorize/VPlanConstruction.cpp | 15 +++++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 30 ++++++++++---------
5 files changed, 31 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 48db0294fcfe7..6527aec6dc5db 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9417,7 +9417,7 @@ addUsersInExitBlocks(VPlan &Plan,
ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
"exit values from early exits must be fixed when branch to "
"early-exit is added");
- ExitIRI->extractLastLaneOfOperand(B);
+ ExitIRI->extractLastLaneOfFirstOperand(B);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fdcee2bd28997..15f22e0e6cddc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1161,10 +1161,10 @@ class VPIRInstruction : public VPRecipeBase {
return true;
}
- /// Update the recipes single operand to the last lane of the operand using \p
- /// Builder. Must only be used for single operand VPIRInstructions wrapping a
- /// PHINode.
- void extractLastLaneOfOperand(VPBuilder &Builder);
+ /// Update the recipes first operand to the last lane of the operand using \p
+ /// Builder. Must only be used for VPIRInstructions with at least one operand
+ /// wrapping a PHINode.
+ void extractLastLaneOfFirstOperand(VPBuilder &Builder);
};
/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 95f0a91113fb9..6b6108c30e272 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -352,13 +352,14 @@ std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG(
Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
Plan->getEntry()->setPlan(&*Plan);
- // Add incoming operands for the VPIRInstructions wrapping the exit phis.
+ // Fix VPlan loop-closed-ssa exit phi's by add incoming operands to the
+ // VPIRInstructions wrapping them.
for (auto *EB : Plan->getExitBlocks()) {
- for (VPRecipeBase &R : *EB) {
- auto *PhiR = dyn_cast<VPIRPhi>(&R);
- if (!PhiR)
- break;
+ for (VPRecipeBase &R : EB->phis()) {
+ auto *PhiR = cast<VPIRPhi>(&R);
PHINode &Phi = PhiR->getIRPhi();
+ assert(PhiR->getNumOperands() == 0 &&
+ "no phi operands should be added yet");
for (BasicBlock *Pred : predecessors(EB->getIRBasicBlock()))
PhiR->addOperand(
getOrCreateVPOperand(Phi.getIncomingValueForBlock(Pred)));
@@ -477,8 +478,8 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy,
VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader());
if (!RequiresScalarEpilogueCheck) {
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
- // The exit blocks are dead, remove any recipes to make sure no users remain
- // that may pessimize transforms.
+ // The exit blocks are unreachable, remove their recipes to make sure no
+ // users remain that may pessimize transforms.
for (auto *EB : Plan.getExitBlocks()) {
for (VPRecipeBase &R : make_early_inc_range(*EB))
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b1fc38d338d74..f6481aaf55ecc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1137,9 +1137,9 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF,
return 0;
}
-void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
+void VPIRInstruction::extractLastLaneOfFirstOperand(VPBuilder &Builder) {
assert(isa<PHINode>(getInstruction()) &&
- "can only add exiting operands to phi nodes");
+ "can only update exiting operands to phi nodes");
assert(getNumOperands() > 0 && "must have at least one operand");
VPValue *Exiting = getOperand(0);
if (!Exiting->isLiveIn()) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e539c68799228..6d01ea7cce708 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2503,23 +2503,26 @@ void VPlanTransforms::handleUncountableEarlyExit(
// Update the exit phis in the early exit block.
VPBuilder MiddleBuilder(NewMiddle);
VPBuilder EarlyExitB(VectorEarlyExitVPBB);
- for (VPRecipeBase &R : *VPEarlyExitBlock->phis()) {
+ for (VPRecipeBase &R : VPEarlyExitBlock->phis()) {
auto *ExitIRI = cast<VPIRPhi>(&R);
+ // By default, assume early exit operand is first, e.g., when the two exit
+ // blocks are distinct - VPEarlyExitBlock has a single predecessor.
unsigned EarlyExitIdx = 0;
if (OrigLoop->getUniqueExitBlock()) {
- // After the transform, the first incoming value is coming from the
- // orignial loop latch, while the second operand is from the early exit.
- // Sawp the phi operands, if the first predecessor in the original IR is
- // not the loop latch.
+ // The incoming values currently correspond to the original IR
+ // predecessors. After the transform, the first incoming value is coming
+ // from the original loop latch, while the second operand is from the
+ // early exit. Swap the phi operands, if the first predecessor in the
+ // original IR is not the loop latch.
if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) !=
OrigLoop->getLoopLatch())
ExitIRI->swapOperands();
EarlyExitIdx = 1;
// If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
- // (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
- // which is coming from the original latch.
- ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
+ // (MiddleVPBB and NewMiddle). Extract the last lane of the incoming value
+ // from MiddleVPBB which is coming from the original latch.
+ ExitIRI->extractLastLaneOfFirstOperand(MiddleBuilder);
}
VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
@@ -2530,15 +2533,14 @@ void VPlanTransforms::handleUncountableEarlyExit(
// and vector VFs.
if (!IncomingFromEarlyExit->isLiveIn() &&
LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
- // Add the incoming value from the early exit.
+ // Update the incoming value from the early exit.
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
"first.active.lane");
- ExitIRI->setOperand(
- EarlyExitIdx,
- EarlyExitB.createNaryOp(Instruction::ExtractElement,
- {IncomingFromEarlyExit, FirstActiveLane},
- nullptr, "early.exit.value"));
+ IncomingFromEarlyExit = EarlyExitB.createNaryOp(
+ Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane},
+ nullptr, "early.exit.value");
+ ExitIRI->setOperand(EarlyExitIdx, IncomingFromEarlyExit);
}
}
MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {IsEarlyExitTaken});
More information about the llvm-commits
mailing list