[llvm] [VPlan] Permit licm-sinking recipes with no users (PR #189957)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 18 03:25:13 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/189957
>From d20b886e0f2013186ec129f3660a7a19685d73f8 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 1 Apr 2026 13:36:54 +0100
Subject: [PATCH 1/2] [VPlan] Permit licm-sinking recipes with no users
The patch is a preparatory step for sinking invariant stores in licm. We
have extended cannotHoistOrSink to accept a flag indicating whether we
are sinking, in order to prevent sinking assumes: without this change,
there is a bad test update in tree. Unfortunately, as the only possible
instructions without users that will not be trivially dce'd, and on
which the vectorizer will not bail out are, are stores and assumes:
since we forbid assume-sinking, and store sinking will only be done in a
follow-up, we have no test changes to show. The patch has not been
marked as an NFC, because the additonal flag in cannotHoistOrSink is
used in more sinking transforms, which would prevent assumes from being
sunk.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 51 +++++++++----------
1 file changed, 24 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 126937bf32cb9..95e4d3b4fd4d0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -290,11 +290,13 @@ collectGroupedReplicateMemOps(
/// Return true if we do not know how to (mechanically) hoist or sink \p R out
/// of a loop region.
-static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
+static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R,
+ bool Sinking = false) {
// Assumes don't alias anything or throw; as long as they're guaranteed to
- // execute, they're safe to hoist.
+ // execute, they're safe to hoist. They should however not be sunk, as it
+ // would destroy information.
if (match(&R, m_Intrinsic<Intrinsic::assume>()))
- return false;
+ return Sinking;
// TODO: Relax checks in the future, e.g. we could also hoist reads, if their
// memory location is not modified in the vector loop.
@@ -324,7 +326,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Candidate))
return;
- if (Candidate->getParent() == SinkTo || cannotHoistOrSinkRecipe(*Candidate))
+ if (Candidate->getParent() == SinkTo ||
+ cannotHoistOrSinkRecipe(*Candidate, /*Sinking=*/true))
return;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
@@ -2331,7 +2334,7 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
VPDT.properlyDominates(Previous, SinkCandidate))
return true;
- if (cannotHoistOrSinkRecipe(*SinkCandidate))
+ if (cannotHoistOrSinkRecipe(*SinkCandidate, /*Sinking=*/true))
return false;
WorkList.push_back(SinkCandidate);
@@ -2709,7 +2712,7 @@ static void licm(VPlan &Plan) {
LoopRegion->getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(POT)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (cannotHoistOrSinkRecipe(R))
+ if (cannotHoistOrSinkRecipe(R, /*Sinking=*/true))
continue;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
@@ -2727,42 +2730,36 @@ static void licm(VPlan &Plan) {
// TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
// support recipes with multiple defined values (e.g., interleaved loads).
auto *Def = cast<VPSingleDefRecipe>(&R);
- // Skip recipes without users as we cannot determine a sink block.
- // TODO: Clone sinkable recipes without users to all exit blocks to reduce
- // their execution frequency.
- if (Def->getNumUsers() == 0)
- continue;
+ // Cannot sink the recipe if the user is defined in a loop region or a
+ // non-successor of the vector loop region. Cannot sink if user is a phi
+ // either.
VPBasicBlock *SinkBB = nullptr;
- // Cannot sink the recipe if any user
- // * is defined in any loop region, or
- // * is a phi, or
- // * multiple users in different blocks.
- if (any_of(Def->users(), [&SinkBB](VPUser *U) {
+ if (any_of(Def->users(), [&SinkBB, &LoopRegion](VPUser *U) {
auto *UserR = cast<VPRecipeBase>(U);
VPBasicBlock *Parent = UserR->getParent();
- // TODO: If the user is a PHI node, we should check the block of
- // incoming value. Support PHI node users if needed.
- if (UserR->isPhi() || Parent->getEnclosingLoopRegion())
- return true;
// TODO: Support sinking when users are in multiple blocks.
if (SinkBB && SinkBB != Parent)
return true;
SinkBB = Parent;
- return false;
+ // TODO: If the user is a PHI node, we should check the block of
+ // incoming value.
+ return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
+ Parent->getSinglePredecessor() != LoopRegion;
}))
continue;
- // Only sink to dedicated exit blocks of the loop region.
- if (SinkBB->getSinglePredecessor() != LoopRegion)
+ // Attempt to set SinkBB to the LoopRegion's single successor, if one
+ // wasn't found.
+ if (!SinkBB && !(SinkBB = cast_or_null<VPBasicBlock>(
+ LoopRegion->getSingleSuccessor())))
continue;
- // TODO: This will need to be a check instead of a assert after
- // conditional branches in vectorized loops are supported.
+ // This will need to be a check instead of a assert after conditional
+ // branches in vectorized loops are supported.
assert(VPDT.properlyDominates(VPBB, SinkBB) &&
"Defining block must dominate sink block");
- // TODO: Clone the recipe if users are on multiple exit paths, instead of
- // just moving.
+ // TODO: Clone the recipe if users are on multiple exit paths.
Def->moveBefore(*SinkBB, SinkBB->getFirstNonPhi());
}
}
>From d293fc73be9c68eafb81e16c1bebf91ecbdc46c9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 18 Apr 2026 11:22:54 +0100
Subject: [PATCH 2/2] [VPlan] Fix comments around licm
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 95e4d3b4fd4d0..79127ee9a895b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -289,7 +289,7 @@ collectGroupedReplicateMemOps(
}
/// Return true if we do not know how to (mechanically) hoist or sink \p R out
-/// of a loop region.
+/// of a loop region. When sinking, \p Sinking must be true.
static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R,
bool Sinking = false) {
// Assumes don't alias anything or throw; as long as they're guaranteed to
@@ -2743,7 +2743,7 @@ static void licm(VPlan &Plan) {
return true;
SinkBB = Parent;
// TODO: If the user is a PHI node, we should check the block of
- // incoming value.
+ // incoming value. Support PHI node users if needed.
return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
Parent->getSinglePredecessor() != LoopRegion;
}))
@@ -2755,11 +2755,12 @@ static void licm(VPlan &Plan) {
LoopRegion->getSingleSuccessor())))
continue;
- // This will need to be a check instead of a assert after conditional
- // branches in vectorized loops are supported.
+ // TODO: This will need to be a check instead of a assert after
+ // conditional branches in vectorized loops are supported.
assert(VPDT.properlyDominates(VPBB, SinkBB) &&
"Defining block must dominate sink block");
- // TODO: Clone the recipe if users are on multiple exit paths.
+ // TODO: Clone the recipe if users are on multiple exit paths, instead of
+ // just moving.
Def->moveBefore(*SinkBB, SinkBB->getFirstNonPhi());
}
}
More information about the llvm-commits
mailing list