[llvm] deaef1c - [LV] Adjust exit recipe detection to run on early vplan (#183318)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 2 09:25:41 PDT 2026
Author: Graham Hunter
Date: 2026-04-02T17:25:35+01:00
New Revision: deaef1c1b71728d321913797b8bb447fa769cb6e
URL: https://github.com/llvm/llvm-project/commit/deaef1c1b71728d321913797b8bb447fa769cb6e
DIFF: https://github.com/llvm/llvm-project/commit/deaef1c1b71728d321913797b8bb447fa769cb6e.diff
LOG: [LV] Adjust exit recipe detection to run on early vplan (#183318)
Splitting out some work from #178454; this covers the enums for
early exit loop type (none, readonly, readwrite) and the style
used (readonly with multiple exit blocks, or masking with the
last iteration done in scalar code), along with changing the early
exit recipe detection to suit moving the transform for handling
early exit readwrite loops earlier in the vplan pipeline.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 29436c09023b5..04b6348665a05 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -470,58 +470,64 @@ unsigned vputils::getVFScaleFactor(VPRecipeBase *R) {
}
std::optional<VPValue *>
-vputils::getRecipesForUncountableExit(VPlan &Plan,
- SmallVectorImpl<VPRecipeBase *> &Recipes,
- SmallVectorImpl<VPRecipeBase *> &GEPs) {
- // Given a VPlan like the following (just including the recipes contributing
- // to loop control exiting here, not the actual work), we're looking to match
- // the recipes contributing to the uncountable exit condition comparison
- // (here, vp<%4>) back to either live-ins or the address nodes for the load
- // used as part of the uncountable exit comparison so that we can copy them
- // to a preheader and rotate the address in the loop to the next vector
- // iteration.
+vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
+ SmallVectorImpl<VPInstruction *> &GEPs,
+ VPBasicBlock *LatchVPBB) {
+ // Given a plain CFG VPlan loop with countable latch exiting block
+ // \p LatchVPBB, we're looking to match the recipes contributing to the
+ // uncountable exit condition comparison (here, vp<%4>) back to either
+ // live-ins or the address nodes for the load used as part of the uncountable
+ // exit comparison so that we can either move them within the loop, or copy
+ // them to the preheader depending on the chosen method for dealing with
+ // stores in uncountable exit loops.
//
// Currently, the address of the load is restricted to a GEP with 2 operands
// and a live-in base address. This constraint may be relaxed later.
//
// VPlan ' for UF>=1' {
- // Live-in vp<%0> = VF
- // Live-in ir<64> = original trip-count
+ // Live-in vp<%0> = VF * UF
+ // Live-in vp<%1> = vector-trip-count
+ // Live-in ir<20> = original trip-count
//
- // entry:
- // Successor(s): preheader, vector.ph
+ // ir-bb<entry>:
+ // Successor(s): scalar.ph, vector.ph
//
// vector.ph:
- // Successor(s): vector loop
+ // Successor(s): for.body
//
- // <x1> vector loop: {
- // vector.body:
- // EMIT vp<%2> = CANONICAL-INDUCTION ir<0>
- // vp<%3> = SCALAR-STEPS vp<%2>, ir<1>, vp<%0>
- // CLONE ir<%ee.addr> = getelementptr ir<0>, vp<%3>
- // WIDEN ir<%ee.load> = load ir<%ee.addr>
- // WIDEN vp<%4> = icmp eq ir<%ee.load>, ir<0>
- // EMIT vp<%5> = any-of vp<%4>
- // EMIT vp<%6> = add vp<%2>, vp<%0>
- // EMIT vp<%7> = icmp eq vp<%6>, ir<64>
- // EMIT branch-on-two-conds vp<%5>, vp<%7>
- // No successors
- // }
- // Successor(s): early.exit, middle.block
+ // for.body:
+ // EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+ // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
+ // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
+ // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
+ // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
+ // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
+ // Successor(s): for.inc
+ //
+ // for.inc:
+ // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+ // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
+ // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+ // EMIT vp<%4> = any-of ir<%3>
+ // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
+ // EMIT vp<%6> = or vp<%4>, vp<%5>
+ // EMIT branch-on-cond vp<%6>
+ // Successor(s): middle.block, for.body
//
// middle.block:
- // Successor(s): preheader
+ // Successor(s): ir-bb<exit>, scalar.ph
//
- // preheader:
+ // ir-bb<exit>:
// No successors
+ //
+ // scalar.ph:
// }
// Find the uncountable loop exit condition.
- auto *Region = Plan.getVectorLoopRegion();
VPValue *UncountableCondition = nullptr;
- if (!match(Region->getExitingBasicBlock()->getTerminator(),
- m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
- m_VPValue())))
+ if (!match(LatchVPBB->getTerminator(),
+ m_BranchOnCond(m_c_BinaryOr(
+ m_AnyOf(m_VPValue(UncountableCondition)), m_VPValue()))))
return std::nullopt;
SmallVector<VPValue *, 4> Worklist;
@@ -544,24 +550,32 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
if (match(V, m_ICmp(m_VPValue(Op1), m_VPValue(Op2)))) {
Worklist.push_back(Op1);
Worklist.push_back(Op2);
- Recipes.push_back(V->getDefiningRecipe());
- } else if (auto *Load = dyn_cast<VPWidenLoadRecipe>(V)) {
- // Reject masked loads for the time being; they make the exit condition
- // more complex.
- if (Load->isMasked())
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ } else if (match(V, m_VPInstruction<Instruction::Load>(m_VPValue(Op1)))) {
+ VPRecipeBase *GepR = Op1->getDefiningRecipe();
+ // Only matching base + single offset term for now.
+ if (GepR->getNumOperands() != 2)
return std::nullopt;
-
- VPValue *GEP = Load->getAddr();
- if (!match(GEP, m_GetElementPtr(m_LiveIn(), m_VPValue())))
+ // Matching a GEP with a loop-invariant base ptr.
+ if (!match(GepR, m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_VPValue())))
return std::nullopt;
-
- Recipes.push_back(Load);
- Recipes.push_back(GEP->getDefiningRecipe());
- GEPs.push_back(GEP->getDefiningRecipe());
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ Recipes.push_back(cast<VPInstruction>(GepR));
+ GEPs.push_back(cast<VPInstruction>(GepR));
+ } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
+ m_VPValue(Op1)))) {
+ Worklist.push_back(Op1);
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
} else
return std::nullopt;
}
+ // If we couldn't match anything, don't return the condition. It may be
+ // defined outside the loop.
+ if (Recipes.empty() || GEPs.empty())
+ return std::nullopt;
+
return UncountableCondition;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 14619aee6d27c..5fdd5ea4204e0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -80,9 +80,9 @@ unsigned getVFScaleFactor(VPRecipeBase *R);
/// \p GEPs.
LLVM_ABI_FOR_TEST
std::optional<VPValue *>
-getRecipesForUncountableExit(VPlan &Plan,
- SmallVectorImpl<VPRecipeBase *> &Recipes,
- SmallVectorImpl<VPRecipeBase *> &GEPs);
+getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
+ SmallVectorImpl<VPInstruction *> &GEPs,
+ VPBasicBlock *LatchVPBB);
/// Return a MemoryLocation for \p R with noalias metadata populated from
/// \p R, if the recipe is supported and std::nullopt otherwise. The pointer of
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index a4bbf6cbd9056..4c8e5aef2075b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -101,6 +101,17 @@ class VPlanTestIRBase : public testing::Test {
VPlanTransforms::createLoopRegions(*Plan);
return Plan;
}
+
+ VPlanPtr buildVPlan0(BasicBlock *LoopHeader) {
+ Function &F = *LoopHeader->getParent();
+ assert(!verifyFunction(F) && "input function must be valid");
+ doAnalysis(F);
+
+ Loop *L = LI->getLoopFor(LoopHeader);
+ PredicatedScalarEvolution PSE(*SE, *L);
+ return VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), {},
+ PSE);
+ }
};
class VPlanTestBase : public testing::Test {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
index 9494634f5f590..1ad9d2b1a49d6 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
@@ -7,7 +7,9 @@
//
//===----------------------------------------------------------------------===//
+#include "../lib/Transforms/Vectorize/VPRecipeBuilder.h"
#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
#include "../lib/Transforms/Vectorize/VPlanUtils.h"
#include "VPlanTestBase.h"
#include "llvm/ADT/SmallVector.h"
@@ -17,6 +19,59 @@ namespace llvm {
namespace {
class VPUncountableExitTest : public VPlanTestIRBase {};
+using namespace VPlanPatternMatch;
+
+static void combineExitConditions(VPlan &Plan) {
+ struct EarlyExitInfo {
+ VPBasicBlock *EarlyExitingVPBB;
+ VPIRBasicBlock *EarlyExitVPBB;
+ VPValue *CondToExit;
+ };
+
+ auto *MiddleVPBB = cast<VPBasicBlock>(
+ Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
+ auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
+
+ // Find the single early exit: a non-middle predecessor of an exit block.
+ VPBasicBlock *EarlyExitingVPBB = nullptr;
+ VPIRBasicBlock *EarlyExitVPBB = nullptr;
+ for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
+ for (VPBlockBase *Pred : ExitBlock->getPredecessors()) {
+ if (Pred != MiddleVPBB) {
+ EarlyExitingVPBB = cast<VPBasicBlock>(Pred);
+ EarlyExitVPBB = ExitBlock;
+ }
+ }
+ }
+ assert(EarlyExitingVPBB && "must have an early exit");
+
+ // Wrap the early exit condition in a MaskedCond.
+ VPValue *Cond;
+ [[maybe_unused]] bool Matched =
+ match(EarlyExitingVPBB->getTerminator(), m_BranchOnCond(m_VPValue(Cond)));
+ assert(Matched && "Terminator must be BranchOnCond");
+ VPBuilder EarlyExitBuilder(EarlyExitingVPBB->getTerminator());
+ if (EarlyExitingVPBB->getSuccessors()[0] != EarlyExitVPBB)
+ Cond = EarlyExitBuilder.createNot(Cond);
+ auto *MaskedCond =
+ EarlyExitBuilder.createNaryOp(VPInstruction::MaskedCond, {Cond});
+
+ // Combine the early exit with the latch exit on the latch terminator.
+ VPBuilder Builder(LatchVPBB->getTerminator());
+ auto *IsAnyExitTaken =
+ Builder.createNaryOp(VPInstruction::AnyOf, {MaskedCond});
+ auto *LatchBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+ auto *IsLatchExitTaken = Builder.createICmp(
+ CmpInst::ICMP_EQ, LatchBranch->getOperand(0), LatchBranch->getOperand(1));
+ LatchBranch->eraseFromParent();
+ Builder.setInsertPoint(LatchVPBB);
+ Builder.createNaryOp(VPInstruction::BranchOnCond,
+ {Builder.createOr(IsAnyExitTaken, IsLatchExitTaken)});
+
+ // Disconnect the early exit edge.
+ EarlyExitingVPBB->getTerminator()->eraseFromParent();
+ VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
+}
TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
const char *ModuleString =
@@ -33,9 +88,7 @@ TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
" %st.addr = getelementptr inbounds i16, ptr %array, i64 %iv\n"
" %data = load i16, ptr %st.addr, align 2\n"
" %inc = add nsw i16 %data, 1\n"
- // TODO: Uncomment store once more support is added for uncountable exits
- // in loops with stores.
- // " store i16 %inc, ptr %st.addr, align 2\n"
+ " store i16 %inc, ptr %st.addr, align 2\n"
" %uncountable.addr = getelementptr inbounds nuw i16, ptr %pred, i64 "
"%iv\n"
" %uncountable.val = load i16, ptr %uncountable.addr, align 2\n"
@@ -53,18 +106,21 @@ TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly);
- VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI);
- VPlanTransforms::optimize(*Plan);
+ VPlanPtr Plan = buildVPlan0(LoopHeader);
+ combineExitConditions(*Plan);
- SmallVector<VPRecipeBase *> Recipes;
- SmallVector<VPRecipeBase *> GEPs;
+ SmallVector<VPInstruction *> Recipes;
+ SmallVector<VPInstruction *> GEPs;
+
+ auto *MiddleVPBB = cast<VPBasicBlock>(
+ Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
+ auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(*Plan, Recipes, GEPs);
+ vputils::getRecipesForUncountableExit(Recipes, GEPs, LatchVPBB);
ASSERT_TRUE(UncountableCondition.has_value());
ASSERT_EQ(GEPs.size(), 1ull);
- ASSERT_EQ(Recipes.size(), 3ull);
+ ASSERT_EQ(Recipes.size(), 4ull);
}
TEST_F(VPUncountableExitTest, NoUncountableExit) {
@@ -89,15 +145,17 @@ TEST_F(VPUncountableExitTest, NoUncountableExit) {
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- auto Plan = buildVPlan(LoopHeader);
- VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI);
- VPlanTransforms::optimize(*Plan);
+ auto Plan = buildVPlan0(LoopHeader);
+
+ SmallVector<VPInstruction *> Recipes;
+ SmallVector<VPInstruction *> GEPs;
- SmallVector<VPRecipeBase *> Recipes;
- SmallVector<VPRecipeBase *> GEPs;
+ auto *MiddleVPBB = cast<VPBasicBlock>(
+ Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
+ auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(*Plan, Recipes, GEPs);
+ vputils::getRecipesForUncountableExit(Recipes, GEPs, LatchVPBB);
ASSERT_FALSE(UncountableCondition.has_value());
ASSERT_EQ(GEPs.size(), 0ull);
ASSERT_EQ(Recipes.size(), 0ull);
More information about the llvm-commits
mailing list