[llvm] [VPlan] Handle extracts for middle blocks also used by early exiting blocks. NFC (PR #181789)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 17 01:12:04 PST 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/181789
Currently createExtractsForLiveOuts only handles creating extracts when the middle block has one predecessor, but if an early exit exits to the same block as the latch then it might have multiple predecessors.
This handles the latter case to avoid the need to handle it in VPlanTransforms::handleUncountableEarlyExits. Addresses the comment in https://github.com/llvm/llvm-project/pull/174864#discussion_r2794153217
>From b3c4593eef280e7f24f6595437763ab781cc5593 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 17 Feb 2026 17:09:04 +0800
Subject: [PATCH] [VPlan] Handle extracts for middle blocks also used by early
exiting blocks. NFC
Currently createExtractsForLiveOuts only handles creating extracts when the middle block has one predecessor, but if an early exit exits to the same block as the latch then it might have multiple predecessors.
This handles the latter case to avoid the need to handle it in VPlanTransforms::handleUncountableEarlyExits. Addresses the comment in https://github.com/llvm/llvm-project/pull/174864#discussion_r2794153217
---
llvm/lib/Transforms/Vectorize/VPlan.h | 9 ++++---
.../Vectorize/VPlanConstruction.cpp | 18 ++++++--------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 24 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.cpp | 24 -------------------
4 files changed, 21 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bb36659cdba6c..308a93f3ea9c8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1524,6 +1524,10 @@ class VPPhiAccessors {
/// Returns the incoming value for \p VPBB. \p VPBB must be an incoming block.
VPValue *getIncomingValueForBlock(const VPBasicBlock *VPBB) const;
+ /// Sets the incoming value for \p VPBB to \p V. \p VPBB must be an incoming
+ /// block.
+ void setIncomingValueForBlock(const VPBasicBlock *VPBB, VPValue *V) const;
+
/// Returns the number of incoming values, also number of incoming blocks.
virtual unsigned getNumIncoming() const {
return getAsRecipe()->getNumOperands();
@@ -1656,11 +1660,6 @@ class VPIRInstruction : public VPRecipeBase {
return true;
}
- /// Update the recipe's first operand to the last lane of the last part of the
- /// operand using \p Builder. Must only be used for VPIRInstructions with at
- /// least one operand wrapping a PHINode.
- void extractLastLaneOfLastPartOfFirstOperand(VPBuilder &Builder);
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 80847fcbb77fb..9f28d770f6f93 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -491,21 +491,17 @@ static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
static void createExtractsForLiveOuts(VPlan &Plan, VPBasicBlock *MiddleVPBB) {
VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
for (VPBasicBlock *EB : Plan.getExitBlocks()) {
- if (EB->getSinglePredecessor() != MiddleVPBB)
+ if (!is_contained(EB->predecessors(), MiddleVPBB))
continue;
for (VPRecipeBase &R : EB->phis()) {
auto *ExitIRI = cast<VPIRPhi>(&R);
- for (unsigned Idx = 0; Idx != ExitIRI->getNumIncoming(); ++Idx) {
- VPRecipeBase *Inc = ExitIRI->getIncomingValue(Idx)->getDefiningRecipe();
- if (!Inc)
- continue;
- assert(ExitIRI->getNumOperands() == 1 &&
- ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
- "exit values from early exits must be fixed when branch to "
- "early-exit is added");
- ExitIRI->extractLastLaneOfLastPartOfFirstOperand(B);
- }
+ VPValue *Exiting = ExitIRI->getIncomingValueForBlock(MiddleVPBB);
+ if (isa<VPIRValue>(Exiting))
+ continue;
+ Exiting = B.createNaryOp(VPInstruction::ExtractLastPart, Exiting);
+ Exiting = B.createNaryOp(VPInstruction::ExtractLastLane, Exiting);
+ ExitIRI->setIncomingValueForBlock(MiddleVPBB, Exiting);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bab1fc773fa4b..5818189762b6a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1659,20 +1659,6 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF,
return 0;
}
-void VPIRInstruction::extractLastLaneOfLastPartOfFirstOperand(
- VPBuilder &Builder) {
- assert(isa<PHINode>(getInstruction()) &&
- "can only update exiting operands to phi nodes");
- assert(getNumOperands() > 0 && "must have at least one operand");
- VPValue *Exiting = getOperand(0);
- if (isa<VPIRValue>(Exiting))
- return;
-
- Exiting = Builder.createNaryOp(VPInstruction::ExtractLastPart, Exiting);
- Exiting = Builder.createNaryOp(VPInstruction::ExtractLastLane, Exiting);
- setOperand(0, Exiting);
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPIRInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -1723,6 +1709,16 @@ VPPhiAccessors::getIncomingValueForBlock(const VPBasicBlock *VPBB) const {
llvm_unreachable("VPBB is not an incoming block");
}
+void VPPhiAccessors::setIncomingValueForBlock(const VPBasicBlock *VPBB,
+ VPValue *V) const {
+ for (unsigned Idx = 0; Idx != getNumIncoming(); ++Idx)
+ if (getIncomingBlock(Idx) == VPBB) {
+ const_cast<VPRecipeBase *>(getAsRecipe())->setOperand(Idx, V);
+ return;
+ }
+ llvm_unreachable("VPBB is not an incoming block");
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPPhiAccessors::printPhiOperands(raw_ostream &O,
VPSlotTracker &SlotTracker) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 51208b6aad680..f20d34dce12d6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4168,30 +4168,6 @@ void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
}
- // For exit blocks that also have the middle block as predecessor (latch
- // exits to the same block as an early exit), extract the last lane of the
- // first operand for the middle block's incoming value.
- VPBuilder MiddleBuilder(MiddleVPBB);
- VPBasicBlock *MiddleSuccVPBB =
- cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
- if (MiddleSuccVPBB->getNumPredecessors() > 1) {
- assert(all_of(MiddleSuccVPBB->getPredecessors(),
- [&](VPBlockBase *Pred) {
- return Pred == MiddleVPBB ||
- is_contained(VectorEarlyExitVPBBs, Pred);
- }) &&
- "All predecessors must be either the middle block or early exit "
- "blocks");
-
- for (VPRecipeBase &R : MiddleSuccVPBB->phis()) {
- auto *ExitIRI = cast<VPIRPhi>(&R);
- assert(ExitIRI->getIncomingValueForBlock(MiddleVPBB) ==
- ExitIRI->getOperand(0) &&
- "First operand must come from middle block");
- ExitIRI->extractLastLaneOfLastPartOfFirstOperand(MiddleBuilder);
- }
- }
-
// Chain through exits: for each exit, check if its condition is true at
// the first active lane. If so, take that exit; otherwise, try the next.
// The last exit needs no check since it must be taken if all others fail.
More information about the llvm-commits
mailing list