[llvm] [LV] Follow up to uncountable exit with side effects vectorization (PR #201589)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 06:19:01 PDT 2026
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/201589
>From cf69b146a5f7610078b5d432ce8f26282a98e338 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 3 Jun 2026 16:12:23 +0000
Subject: [PATCH 1/4] [LV] Follow up to uncountable exit with side effects
vectorization
Addressing post-commit comments on #178454.
---
.../Vectorize/LoopVectorizationLegality.cpp | 11 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 120 ++++++++++--------
.../AArch64/early_exit_with_stores.ll | 6 +-
.../RISCV/early_exit_with_stores.ll | 6 +-
.../VPlan/early_exit_with_stores_vplan.ll | 4 +-
.../LoopVectorize/early_exit_with_stores.ll | 8 +-
.../interleave_uncountable_exits.ll | 2 +-
...rized_conditional_ops_uncountable_exits.ll | 8 +-
.../tail_fold_uncountable_exits.ll | 2 +-
9 files changed, 87 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 214b5cff8a03a..f8fb9d6d6a98f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1946,16 +1946,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- // TODO: Remove this restriction once we're sure it's safe to do so.
- // Handling stores to invariant addresses will be slightly different
- // based on the vectorization style chosen. If we bail out to a scalar
- // tail before executing any lane that would take the uncountable exit,
- // then the store that occurs in the scalar loop would suffice.
- //
- // If we instead handle the lane taking the uncountable exit within the
- // vectorized loop, then we will have to ensure that we extract the
- // last active lane at that point in the loop instead of the last lane
- // of the vector before performing a scalar store.
+ // TODO: Remove this restriction, should be straightforward to support.
if (UncountableExitType != UncountableExitTrait::None &&
!LAI->getStoresToInvariantAddresses().empty()) {
LLVM_DEBUG(dbgs() << "LV: Cannot vectorize early exit loops with stores to "
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5df097628ba7f..85b4fd697f047 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4210,6 +4210,31 @@ struct EarlyExitInfo {
/// Update \p Plan to mask memory operations in the loop based on whether the
/// early exit is taken or not.
+//
+// We're currently expecting to find a loop with properties similar to the
+// following C code:
+//
+// #define N 10000
+// int cond[N];
+// int src[N];
+// int dst[N];
+//
+// void foo(int threshold) {
+// for (int i=0; i<N; ++i) {
+// if (cond[i] > threshold)
+// break;
+// dst[i] = src[i] + 42;
+// }
+// }
+//
+// The loop must have a single unconditional load contributing to the
+// uncountable exit comparison, and the other term must be loop-invariant.
+// There must also be a counted exit. Other memory operations in the loop can
+// take place before or after the uncountable exit, but must also be
+// unconditional. All potential accesses to the memory used for the load for the
+// exit condition must be guaranteed to be dereferenceable. Any stores within
+// the loop must not alias with any other memory operations.
+//
static bool handleUncountableExitsWithSideEffects(
VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
@@ -4241,15 +4266,20 @@ static bool handleUncountableExitsWithSideEffects(
return false;
// Find load contributing to condition.
- VPRecipeBase *CondLoad = nullptr;
- for (auto *Recipe : ConditionRecipes) {
- if (match(Recipe, m_VPInstruction<Instruction::Load>(m_VPValue()))) {
- // TODO: Support more than one load. Needs legality updates too.
- assert(CondLoad == nullptr && "Too many condition loads");
- CondLoad = Recipe;
- }
- }
- assert(CondLoad && "Couldn't find load");
+ // At the moment LoopVectorizationLegality only supports a single
+ // early-exit expression with a compare and a single load that must
+ // be unconditional.
+ // TODO: Support more than one load.
+ auto *Load =
+ find_singleton<VPInstruction>(ConditionRecipes, [](auto *I, bool _) {
+ return match(I, m_VPInstruction<Instruction::Load>(m_VPValue()))
+ ? I
+ : nullptr;
+ });
+ assert(Load && "Couldn't find load");
+ // TODO: Support conditional loads for uncountable exits.
+ assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
+ "Uncountable exit condition load is conditional.");
// Ensure that we are guaranteed to be able to dereference the memory used
// for determining the uncountable exit for the maximum possible number of
@@ -4259,7 +4289,6 @@ static bool handleUncountableExitsWithSideEffects(
// all possible addresses are dereferenceable.
{
SmallVector<const SCEVPredicate *, 4> Predicates;
- VPSingleDefRecipe *Load = cast<VPSingleDefRecipe>(CondLoad);
VPValue *Ptr = Load->getOperand(0);
const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, TheLoop);
const DataLayout &DL = Plan.getDataLayout();
@@ -4272,54 +4301,36 @@ static bool handleUncountableExitsWithSideEffects(
return false;
}
- // Check GEPs to see if we can link them to a widen IV recipe with a step of
- // 1; we're only interested in contiguous accesses for the condition load
- // right now.
- for (auto *GEP : GEPs) {
- VPValue *MaybeIV = nullptr;
- if (!match(GEP, m_VPInstruction<Instruction::GetElementPtr>(
- m_LiveIn(), m_VPValue(MaybeIV))))
- return false;
-
- auto *WIV = dyn_cast<VPWidenInductionRecipe>(MaybeIV);
- if (!WIV)
- return false;
-
- if (!match(WIV->getStartValue(), m_SpecificInt(0)) ||
- !match(WIV->getStepValue(), m_SpecificInt(1)))
- return false;
- }
-
- // Find an insertion point. Default to the end of the header but override
- // if we find a memory op that needs masking before the condition load.
- auto InsertIt = HeaderVPBB->end();
- VPRecipeBase *CondR = (*Cond)->getDefiningRecipe();
- bool CondMoveNeeded = CondR->getParent() != HeaderVPBB;
- for (VPRecipeBase &R : *HeaderVPBB) {
- if (&R == CondLoad)
- continue;
+ // Check for a single GEP for the condition load to see if we can link it to
+ // a widen IV recipe with a step of 1; we're only interested in contiguous
+ // accesses for the condition load right now.
+ auto *IV = cast<VPWidenInductionRecipe>(&HeaderVPBB->front());
+ if (!match(IV->getStartValue(), m_SpecificInt(0)) ||
+ !match(IV->getStepValue(), m_SpecificInt(1)))
+ return false;
+ if (GEPs.size() != 1)
+ return false;
+ if (!match(GEPs.front(), m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_Specific(IV))))
+ return false;
- if (R.mayReadOrWriteMemory()) {
- if (!VPDT.properlyDominates(CondR, &R)) {
- CondMoveNeeded = true;
- InsertIt = R.getIterator();
- }
- break;
- }
+ // We want to guarantee that the uncountable exit condition (and the mask
+ // we will generate from it) are available for all operations in the loop
+ // that need to be masked. If the condition recipes are not already the first
+ // recipes in the header after the last phi, move them there.
+ auto InsertIt = HeaderVPBB->getFirstNonPhi();
+ while (InsertIt != HeaderVPBB->end() &&
+ is_contained(ConditionRecipes, &*InsertIt)) {
+ erase(ConditionRecipes, &*InsertIt);
+ InsertIt++;
}
-
- // If another memory operation would take place before the comparison to
- // determine whether to exit early or the comparison doesn't take place in
- // the header, move the comparison (and supporting recipes).
- if (CondMoveNeeded)
- for (auto *Recipe : reverse(ConditionRecipes))
- Recipe->moveBefore(*HeaderVPBB, InsertIt);
+ for (auto *Recipe : reverse(ConditionRecipes))
+ Recipe->moveBefore(*HeaderVPBB, InsertIt);
// Create a mask to represent all lanes that fully execute in the vector loop,
// stopping short of any early exit.
VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
VPValue *FirstActive = MaskBuilder.createFirstActiveLane(*Cond);
- VPValue *IV = cast<VPSingleDefRecipe>(&HeaderVPBB->front());
Type *IVScalarTy = IV->getScalarType();
Type *FirstActiveTy = FirstActive->getScalarType();
VPValue *ALMMultiplier = Plan.getConstantInt(IVScalarTy, 1);
@@ -4333,7 +4344,7 @@ static bool handleUncountableExitsWithSideEffects(
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
for (VPRecipeBase &R : *VPBB)
- if (R.mayReadOrWriteMemory() && &R != CondLoad) {
+ if (R.mayReadOrWriteMemory() && &R != Load) {
// TODO: Handle conditional memory operations in the loop.
if (!VPDT.dominates(R.getParent(), LatchVPBB))
return false;
@@ -4360,6 +4371,11 @@ static bool handleUncountableExitsWithSideEffects(
if (range_size(Phis) != 1)
return false;
VPPhi *ContinueIV = cast<VPPhi>(Phis.begin());
+ // Make sure we're referring to the same IV.
+ assert(
+ match(ContinueIV->getOperand(0),
+ m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
+ "Continuing from different IV");
ContinueIV->setOperand(0, ExitIV);
return true;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 46a21f8a95079..bd202423a17ee 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -64,12 +64,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -343,12 +343,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
@@ -1141,12 +1141,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 2b73a995d73bb..65fb9684de8df 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -62,12 +62,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -341,12 +341,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
@@ -1184,12 +1184,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 00ef6968159d1..b1abf9c0833b4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -23,13 +23,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
@@ -197,13 +197,13 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index affe5e30c1146..c2fd759504759 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -56,12 +56,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -328,12 +328,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
@@ -1022,12 +1022,12 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -1184,12 +1184,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index 26f4ee4b6ead8..d1ef3a607a243 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -12,12 +12,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index 942212dfc63d1..c572f15086082 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -13,15 +13,15 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP26]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP26]]
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP35]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index 73fef71ce774f..58ff87764459a 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -15,12 +15,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
>From 7ac5948fc26b551388fba1f9e3f03f440f5d70d3 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 9 Jun 2026 12:23:59 +0000
Subject: [PATCH 2/4] Remove separate GEPs parameter, check based on ptr arg
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++-------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 6 +++---
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 +---
.../Vectorize/VPlanUncountableExitTest.cpp | 13 +++++--------
4 files changed, 13 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 85b4fd697f047..2767b2ceb0345 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4257,11 +4257,10 @@ static bool handleUncountableExitsWithSideEffects(
// We can abandon a VPlan entirely if we return false here, so we shouldn't
// crash if some earlier assumptions on scalar IR don't hold for the vplan
// version of the loop.
- SmallVector<VPInstruction *, 2> GEPs;
SmallVector<VPInstruction *, 8> ConditionRecipes;
std::optional<VPValue *> Cond =
- vputils::getRecipesForUncountableExit(ConditionRecipes, GEPs, LatchVPBB);
+ vputils::getRecipesForUncountableExit(ConditionRecipes, LatchVPBB);
if (!Cond)
return false;
@@ -4280,6 +4279,7 @@ static bool handleUncountableExitsWithSideEffects(
// TODO: Support conditional loads for uncountable exits.
assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
"Uncountable exit condition load is conditional.");
+ VPInstruction *Ptr = cast<VPInstruction>(Load->getOperand(0));
// Ensure that we are guaranteed to be able to dereference the memory used
// for determining the uncountable exit for the maximum possible number of
@@ -4289,7 +4289,6 @@ static bool handleUncountableExitsWithSideEffects(
// all possible addresses are dereferenceable.
{
SmallVector<const SCEVPredicate *, 4> Predicates;
- VPValue *Ptr = Load->getOperand(0);
const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, TheLoop);
const DataLayout &DL = Plan.getDataLayout();
APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getScalarType()),
@@ -4308,10 +4307,8 @@ static bool handleUncountableExitsWithSideEffects(
if (!match(IV->getStartValue(), m_SpecificInt(0)) ||
!match(IV->getStepValue(), m_SpecificInt(1)))
return false;
- if (GEPs.size() != 1)
- return false;
- if (!match(GEPs.front(), m_VPInstruction<Instruction::GetElementPtr>(
- m_LiveIn(), m_Specific(IV))))
+ if (!match(Ptr, m_VPInstruction<Instruction::GetElementPtr>(m_LiveIn(),
+ m_Specific(IV))))
return false;
// We want to guarantee that the uncountable exit condition (and the mask
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f7f2d4591a3ce..2ec80db540060 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -530,7 +530,6 @@ bool vputils::cannotHoistOrSinkRecipe(const VPRecipeBase &R, bool Sinking) {
std::optional<VPValue *>
vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
- SmallVectorImpl<VPInstruction *> &GEPs,
VPBasicBlock *LatchVPBB) {
// Given a plain CFG VPlan loop with countable latch exiting block
// \p LatchVPBB, we're looking to match the recipes contributing to the
@@ -620,7 +619,6 @@ vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
return std::nullopt;
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
Recipes.push_back(cast<VPInstruction>(GepR));
- GEPs.push_back(cast<VPInstruction>(GepR));
} else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
m_VPValue(Op1)))) {
Worklist.push_back(Op1);
@@ -631,7 +629,9 @@ vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// If we couldn't match anything, don't return the condition. It may be
// defined outside the loop.
- if (Recipes.empty() || GEPs.empty())
+ if (Recipes.empty() || none_of(Recipes, [](VPInstruction *I) {
+ return match(I, m_VPInstruction<Instruction::GetElementPtr>());
+ }))
return std::nullopt;
return UncountableCondition;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 60ab2632a089c..05259dce81902 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -104,12 +104,10 @@ template <typename Ty> Intrinsic::ID getIntrinsicID(const Ty *R) {
/// AnyOf if the recipes it depends on can be traced back to live-ins and
/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
/// generating the values for the comparison. The recipes are stored in
-/// \p Recipes, and recipes forming an address for a load are also added to
-/// \p GEPs.
+/// \p Recipes.
LLVM_ABI_FOR_TEST
std::optional<VPValue *>
getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
- SmallVectorImpl<VPInstruction *> &GEPs,
VPBasicBlock *LatchVPBB);
/// Return a MemoryLocation for \p R with noalias metadata populated from
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
index 3e10d5f17699d..aa15c304df92c 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
@@ -110,17 +110,16 @@ TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
VPlanPtr Plan = buildVPlan0(LoopHeader);
combineExitConditions(*Plan);
- SmallVector<VPInstruction *> Recipes;
- SmallVector<VPInstruction *> GEPs;
-
auto *MiddleVPBB = cast<VPBasicBlock>(
Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
+ SmallVector<VPInstruction *> Recipes;
std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(Recipes, GEPs, LatchVPBB);
+ vputils::getRecipesForUncountableExit(Recipes, LatchVPBB);
ASSERT_TRUE(UncountableCondition.has_value());
- ASSERT_EQ(GEPs.size(), 1ull);
+ ASSERT_TRUE(
+ match(Recipes.back(), m_VPInstruction<Instruction::GetElementPtr>()));
ASSERT_EQ(Recipes.size(), 4ull);
}
@@ -149,16 +148,14 @@ TEST_F(VPUncountableExitTest, NoUncountableExit) {
auto Plan = buildVPlan0(LoopHeader);
SmallVector<VPInstruction *> Recipes;
- SmallVector<VPInstruction *> GEPs;
auto *MiddleVPBB = cast<VPBasicBlock>(
Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(Recipes, GEPs, LatchVPBB);
+ vputils::getRecipesForUncountableExit(Recipes, LatchVPBB);
ASSERT_FALSE(UncountableCondition.has_value());
- ASSERT_EQ(GEPs.size(), 0ull);
ASSERT_EQ(Recipes.size(), 0ull);
}
>From 325585a217139f250a9230d5af4c47e4ca5c49de Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 9 Jun 2026 13:39:03 +0000
Subject: [PATCH 3/4] Use vplan instead of C for comment
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 ++++++++++---------
1 file changed, 31 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2767b2ceb0345..a6168fcb35f3a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4210,31 +4210,37 @@ struct EarlyExitInfo {
/// Update \p Plan to mask memory operations in the loop based on whether the
/// early exit is taken or not.
-//
-// We're currently expecting to find a loop with properties similar to the
-// following C code:
-//
-// #define N 10000
-// int cond[N];
-// int src[N];
-// int dst[N];
-//
-// void foo(int threshold) {
-// for (int i=0; i<N; ++i) {
-// if (cond[i] > threshold)
-// break;
-// dst[i] = src[i] + 42;
-// }
-// }
-//
-// The loop must have a single unconditional load contributing to the
-// uncountable exit comparison, and the other term must be loop-invariant.
-// There must also be a counted exit. Other memory operations in the loop can
-// take place before or after the uncountable exit, but must also be
-// unconditional. All potential accesses to the memory used for the load for the
-// exit condition must be guaranteed to be dereferenceable. Any stores within
-// the loop must not alias with any other memory operations.
-//
+///
+/// We're currently expecting to find a loop with properties similar to the
+/// following:
+///
+/// for.body:
+/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
+/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
+/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
+/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
+/// EMIT branch-on-cond ir<%cmp1>
+/// Successor(s): ir-bb<cleanup>, if.end
+///
+/// if.end:
+/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
+/// EMIT-SCALAR ir<%1> = load ir<%arrayidx3>
+/// EMIT ir<%add> = add nsw ir<%1>, ir<42>
+/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
+/// EMIT store ir<%add>, ir<%arrayidx5>
+/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
+/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
+/// EMIT branch-on-cond ir<%exitcond.not>
+/// Successor(s): middle.block, for.body
+///
+/// The loop must have a single unconditional load contributing to the
+/// uncountable exit comparison, and the other term must be loop-invariant.
+/// There must also be a counted exit. Other memory operations in the loop can
+/// take place before or after the uncountable exit, but must also be
+/// unconditional. All potential accesses to the memory used for the load for
+/// the exit condition must be guaranteed to be dereferenceable. Any stores
+/// within the loop must not alias with any other memory operations.
+///
static bool handleUncountableExitsWithSideEffects(
VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
>From af79f57044db3df1fdbd164143d0806a900de6c5 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 11 Jun 2026 13:13:37 +0000
Subject: [PATCH 4/4] Remove unnecessary unit test file, move helper to
Transforms
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 116 ++++++++++++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 109 ------------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 10 --
.../Transforms/Vectorize/CMakeLists.txt | 1 -
.../Transforms/Vectorize/VPlanTestBase.h | 11 --
.../Vectorize/VPlanUncountableExitTest.cpp | 163 ------------------
6 files changed, 115 insertions(+), 295 deletions(-)
delete mode 100644 llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a6168fcb35f3a..e539d14ea7dfe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4202,6 +4202,120 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
}
}
+/// Returns the VPValue representing the uncountable exit comparison used by
+/// AnyOf if the recipes it depends on can be traced back to live-ins and
+/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
+/// generating the values for the comparison. The recipes are stored in
+/// \p Recipes.
+static std::optional<VPValue *>
+getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
+ VPBasicBlock *LatchVPBB) {
+ // Given a plain CFG VPlan loop with countable latch exiting block
+ // \p LatchVPBB, we're looking to match the recipes contributing to the
+ // uncountable exit condition comparison (here, vp<%4>) back to either
+ // live-ins or the address nodes for the load used as part of the uncountable
+ // exit comparison so that we can either move them within the loop, or copy
+ // them to the preheader depending on the chosen method for dealing with
+ // stores in uncountable exit loops.
+ //
+ // Currently, the address of the load is restricted to a GEP with 2 operands
+ // and a live-in base address. This constraint may be relaxed later.
+ //
+ // VPlan ' for UF>=1' {
+ // Live-in vp<%0> = VF * UF
+ // Live-in vp<%1> = vector-trip-count
+ // Live-in ir<20> = original trip-count
+ //
+ // ir-bb<entry>:
+ // Successor(s): scalar.ph, vector.ph
+ //
+ // vector.ph:
+ // Successor(s): for.body
+ //
+ // for.body:
+ // EMIT vp<%2> = phi ir<0>, vp<%index.next>
+ // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
+ // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
+ // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
+ // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
+ // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
+ // Successor(s): for.inc
+ //
+ // for.inc:
+ // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+ // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
+ // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+ // EMIT vp<%4> = any-of ir<%3>
+ // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
+ // EMIT branch-on-two-conds vp<%4>, vp<%5>
+ // Successor(s): middle.block, middle.block, for.body
+ //
+ // middle.block:
+ // Successor(s): ir-bb<exit>, scalar.ph
+ //
+ // ir-bb<exit>:
+ // No successors
+ //
+ // scalar.ph:
+ // }
+
+ // Find the uncountable loop exit condition.
+ VPValue *UncountableCondition = nullptr;
+ if (!match(LatchVPBB->getTerminator(),
+ m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
+ m_VPValue())))
+ return std::nullopt;
+
+ SmallVector<VPValue *, 4> Worklist;
+ Worklist.push_back(UncountableCondition);
+ while (!Worklist.empty()) {
+ VPValue *V = Worklist.pop_back_val();
+
+ // Any value defined outside the loop does not need to be copied.
+ if (V->isDefinedOutsideLoopRegions())
+ continue;
+
+ // FIXME: Remove the single user restriction; it's here because we're
+ // starting with the simplest set of loops we can, and multiple
+ // users means needing to add PHI nodes in the transform.
+ if (V->getNumUsers() > 1)
+ return std::nullopt;
+
+ VPValue *Op1, *Op2;
+ // Walk back through recipes until we find at least one load from memory.
+ if (match(V, m_ICmp(m_VPValue(Op1), m_VPValue(Op2)))) {
+ Worklist.push_back(Op1);
+ Worklist.push_back(Op2);
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ } else if (match(V, m_VPInstruction<Instruction::Load>(m_VPValue(Op1)))) {
+ VPRecipeBase *GepR = Op1->getDefiningRecipe();
+ // Only matching base + single offset term for now.
+ if (GepR->getNumOperands() != 2)
+ return std::nullopt;
+ // Matching a GEP with a loop-invariant base ptr.
+ if (!match(GepR, m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_VPValue())))
+ return std::nullopt;
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ Recipes.push_back(cast<VPInstruction>(GepR));
+ } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
+ m_VPValue(Op1)))) {
+ Worklist.push_back(Op1);
+ Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ } else
+ return std::nullopt;
+ }
+
+ // If we couldn't match anything, don't return the condition. It may be
+ // defined outside the loop.
+ if (Recipes.empty() || none_of(Recipes, [](VPInstruction *I) {
+ return match(I, m_VPInstruction<Instruction::GetElementPtr>());
+ }))
+ return std::nullopt;
+
+ return UncountableCondition;
+}
+
struct EarlyExitInfo {
VPBasicBlock *EarlyExitingVPBB;
VPIRBasicBlock *EarlyExitVPBB;
@@ -4266,7 +4380,7 @@ static bool handleUncountableExitsWithSideEffects(
SmallVector<VPInstruction *, 8> ConditionRecipes;
std::optional<VPValue *> Cond =
- vputils::getRecipesForUncountableExit(ConditionRecipes, LatchVPBB);
+ getRecipesForUncountableExit(ConditionRecipes, LatchVPBB);
if (!Cond)
return false;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2ec80db540060..95cac13a05d48 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -528,115 +528,6 @@ bool vputils::cannotHoistOrSinkRecipe(const VPRecipeBase &R, bool Sinking) {
return RepR && RepR->getOpcode() == Instruction::Alloca;
}
-std::optional<VPValue *>
-vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
- VPBasicBlock *LatchVPBB) {
- // Given a plain CFG VPlan loop with countable latch exiting block
- // \p LatchVPBB, we're looking to match the recipes contributing to the
- // uncountable exit condition comparison (here, vp<%4>) back to either
- // live-ins or the address nodes for the load used as part of the uncountable
- // exit comparison so that we can either move them within the loop, or copy
- // them to the preheader depending on the chosen method for dealing with
- // stores in uncountable exit loops.
- //
- // Currently, the address of the load is restricted to a GEP with 2 operands
- // and a live-in base address. This constraint may be relaxed later.
- //
- // VPlan ' for UF>=1' {
- // Live-in vp<%0> = VF * UF
- // Live-in vp<%1> = vector-trip-count
- // Live-in ir<20> = original trip-count
- //
- // ir-bb<entry>:
- // Successor(s): scalar.ph, vector.ph
- //
- // vector.ph:
- // Successor(s): for.body
- //
- // for.body:
- // EMIT vp<%2> = phi ir<0>, vp<%index.next>
- // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
- // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
- // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
- // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
- // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
- // Successor(s): for.inc
- //
- // for.inc:
- // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
- // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
- // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
- // EMIT vp<%4> = any-of ir<%3>
- // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
- // EMIT branch-on-two-conds vp<%4>, vp<%5>
- // Successor(s): middle.block, middle.block, for.body
- //
- // middle.block:
- // Successor(s): ir-bb<exit>, scalar.ph
- //
- // ir-bb<exit>:
- // No successors
- //
- // scalar.ph:
- // }
-
- // Find the uncountable loop exit condition.
- VPValue *UncountableCondition = nullptr;
- if (!match(LatchVPBB->getTerminator(),
- m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
- m_VPValue())))
- return std::nullopt;
-
- SmallVector<VPValue *, 4> Worklist;
- Worklist.push_back(UncountableCondition);
- while (!Worklist.empty()) {
- VPValue *V = Worklist.pop_back_val();
-
- // Any value defined outside the loop does not need to be copied.
- if (V->isDefinedOutsideLoopRegions())
- continue;
-
- // FIXME: Remove the single user restriction; it's here because we're
- // starting with the simplest set of loops we can, and multiple
- // users means needing to add PHI nodes in the transform.
- if (V->getNumUsers() > 1)
- return std::nullopt;
-
- VPValue *Op1, *Op2;
- // Walk back through recipes until we find at least one load from memory.
- if (match(V, m_ICmp(m_VPValue(Op1), m_VPValue(Op2)))) {
- Worklist.push_back(Op1);
- Worklist.push_back(Op2);
- Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
- } else if (match(V, m_VPInstruction<Instruction::Load>(m_VPValue(Op1)))) {
- VPRecipeBase *GepR = Op1->getDefiningRecipe();
- // Only matching base + single offset term for now.
- if (GepR->getNumOperands() != 2)
- return std::nullopt;
- // Matching a GEP with a loop-invariant base ptr.
- if (!match(GepR, m_VPInstruction<Instruction::GetElementPtr>(
- m_LiveIn(), m_VPValue())))
- return std::nullopt;
- Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
- Recipes.push_back(cast<VPInstruction>(GepR));
- } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op1)))) {
- Worklist.push_back(Op1);
- Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
- } else
- return std::nullopt;
- }
-
- // If we couldn't match anything, don't return the condition. It may be
- // defined outside the loop.
- if (Recipes.empty() || none_of(Recipes, [](VPInstruction *I) {
- return match(I, m_VPInstruction<Instruction::GetElementPtr>());
- }))
- return std::nullopt;
-
- return UncountableCondition;
-}
-
VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
if (VPValue *AliasMask = findIncomingAliasMask(Plan)) {
assert(match(AliasMask->getSingleUser(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 05259dce81902..c1f99353c566d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -100,16 +100,6 @@ template <typename Ty> Intrinsic::ID getIntrinsicID(const Ty *R) {
return Intrinsic::not_intrinsic;
}
-/// Returns the VPValue representing the uncountable exit comparison used by
-/// AnyOf if the recipes it depends on can be traced back to live-ins and
-/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
-/// generating the values for the comparison. The recipes are stored in
-/// \p Recipes.
-LLVM_ABI_FOR_TEST
-std::optional<VPValue *>
-getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
- VPBasicBlock *LatchVPBB);
-
/// Return a MemoryLocation for \p R with noalias metadata populated from
/// \p R, if the recipe is supported and std::nullopt otherwise. The pointer of
/// the location is conservatively set to nullptr.
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 2251579634a81..712dad31296a3 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -14,6 +14,5 @@ add_llvm_unittest(VectorizeTests
VPPostDomFrontierTest.cpp
VPlanHCFGTest.cpp
VPlanPatternMatchTest.cpp
- VPlanUncountableExitTest.cpp
VPlanVerifierTest.cpp
)
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 8a443c8a062d0..01141b047f909 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -103,17 +103,6 @@ class VPlanTestIRBase : public testing::Test {
VPlanTransforms::createLoopRegions(*Plan, {});
return Plan;
}
-
- VPlanPtr buildVPlan0(BasicBlock *LoopHeader) {
- Function &F = *LoopHeader->getParent();
- assert(!verifyFunction(F) && "input function must be valid");
- doAnalysis(F);
-
- Loop *L = LI->getLoopFor(LoopHeader);
- PredicatedScalarEvolution PSE(*SE, *L);
- return VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64),
- PSE);
- }
};
class VPlanTestBase : public testing::Test {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
deleted file mode 100644
index aa15c304df92c..0000000000000
--- a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
+++ /dev/null
@@ -1,163 +0,0 @@
-//===- llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp ---===//
-//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "../lib/Transforms/Vectorize/VPRecipeBuilder.h"
-#include "../lib/Transforms/Vectorize/VPlan.h"
-#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
-#include "../lib/Transforms/Vectorize/VPlanUtils.h"
-#include "VPlanTestBase.h"
-#include "llvm/ADT/SmallVector.h"
-#include "gtest/gtest.h"
-
-namespace llvm {
-
-namespace {
-class VPUncountableExitTest : public VPlanTestIRBase {};
-using namespace VPlanPatternMatch;
-
-static void combineExitConditions(VPlan &Plan) {
- struct EarlyExitInfo {
- VPBasicBlock *EarlyExitingVPBB;
- VPIRBasicBlock *EarlyExitVPBB;
- VPValue *CondToExit;
- };
-
- auto *MiddleVPBB = cast<VPBasicBlock>(
- Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
- auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
-
- // Find the single early exit: a non-middle predecessor of an exit block.
- VPBasicBlock *EarlyExitingVPBB = nullptr;
- VPIRBasicBlock *EarlyExitVPBB = nullptr;
- for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
- for (VPBlockBase *Pred : ExitBlock->getPredecessors()) {
- if (Pred != MiddleVPBB) {
- EarlyExitingVPBB = cast<VPBasicBlock>(Pred);
- EarlyExitVPBB = ExitBlock;
- }
- }
- }
- assert(EarlyExitingVPBB && "must have an early exit");
-
- // Wrap the early exit condition in a MaskedCond.
- VPValue *Cond;
- [[maybe_unused]] bool Matched =
- match(EarlyExitingVPBB->getTerminator(), m_BranchOnCond(m_VPValue(Cond)));
- assert(Matched && "Terminator must be BranchOnCond");
- VPBuilder EarlyExitBuilder(EarlyExitingVPBB->getTerminator());
- if (EarlyExitingVPBB->getSuccessors()[0] != EarlyExitVPBB)
- Cond = EarlyExitBuilder.createNot(Cond);
- auto *MaskedCond =
- EarlyExitBuilder.createNaryOp(VPInstruction::MaskedCond, {Cond});
-
- // Combine the early exit with the latch exit on the latch terminator.
- VPBuilder Builder(LatchVPBB->getTerminator());
- auto *IsAnyExitTaken =
- Builder.createNaryOp(VPInstruction::AnyOf, {MaskedCond});
- auto *LatchBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
- assert(LatchBranch->getOpcode() == VPInstruction::BranchOnCond &&
- "Unexpected terminator");
- VPValue *IsLatchExitTaken = LatchBranch->getOperand(0);
- LatchBranch->eraseFromParent();
- Builder.setInsertPoint(LatchVPBB);
- Builder.createNaryOp(VPInstruction::BranchOnTwoConds,
- {IsAnyExitTaken, IsLatchExitTaken});
-
- // Disconnect the early exit edge.
- EarlyExitingVPBB->getTerminator()->eraseFromParent();
- VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
-}
-
-TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
- const char *ModuleString =
- "target datalayout = "
- "\"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
- "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:"
- "32:64-S128\"\n"
- "define void @f(ptr dereferenceable(40) align 2 %array, "
- "ptr dereferenceable(40) align 2 %pred) {\n"
- "entry:\n"
- " br label %for.body\n"
- "for.body:\n"
- " %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]\n"
- " %st.addr = getelementptr inbounds i16, ptr %array, i64 %iv\n"
- " %data = load i16, ptr %st.addr, align 2\n"
- " %inc = add nsw i16 %data, 1\n"
- " store i16 %inc, ptr %st.addr, align 2\n"
- " %uncountable.addr = getelementptr inbounds nuw i16, ptr %pred, i64 "
- "%iv\n"
- " %uncountable.val = load i16, ptr %uncountable.addr, align 2\n"
- " %uncountable.cond = icmp sgt i16 %uncountable.val, 500\n"
- " br i1 %uncountable.cond, label %exit, label %for.inc\n"
- "for.inc:\n"
- " %iv.next = add nuw nsw i64 %iv, 1\n"
- " %countable.cond = icmp eq i64 %iv.next, 20\n"
- " br i1 %countable.cond, label %exit, label %for.body\n"
- "exit:\n"
- " ret void\n"
- "}\n";
-
- Module &M = parseModule(ModuleString);
-
- Function *F = M.getFunction("f");
- BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- VPlanPtr Plan = buildVPlan0(LoopHeader);
- combineExitConditions(*Plan);
-
- auto *MiddleVPBB = cast<VPBasicBlock>(
- Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
- auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
-
- SmallVector<VPInstruction *> Recipes;
- std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(Recipes, LatchVPBB);
- ASSERT_TRUE(UncountableCondition.has_value());
- ASSERT_TRUE(
- match(Recipes.back(), m_VPInstruction<Instruction::GetElementPtr>()));
- ASSERT_EQ(Recipes.size(), 4ull);
-}
-
-TEST_F(VPUncountableExitTest, NoUncountableExit) {
- const char *ModuleString =
- "define void @f(ptr %array, ptr %pred) {\n"
- "entry:\n"
- " br label %for.body\n"
- "for.body:\n"
- " %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]\n"
- " %st.addr = getelementptr inbounds i16, ptr %array, i64 %iv\n"
- " %data = load i16, ptr %st.addr, align 2\n"
- " %inc = add nsw i16 %data, 1\n"
- " store i16 %inc, ptr %st.addr, align 2\n"
- " %iv.next = add nuw nsw i64 %iv, 1\n"
- " %countable.cond = icmp eq i64 %iv.next, 20\n"
- " br i1 %countable.cond, label %exit, label %for.body\n"
- "exit:\n"
- " ret void\n"
- "}\n";
-
- Module &M = parseModule(ModuleString);
-
- Function *F = M.getFunction("f");
- BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- auto Plan = buildVPlan0(LoopHeader);
-
- SmallVector<VPInstruction *> Recipes;
-
- auto *MiddleVPBB = cast<VPBasicBlock>(
- Plan->getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
- auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
-
- std::optional<VPValue *> UncountableCondition =
- vputils::getRecipesForUncountableExit(Recipes, LatchVPBB);
- ASSERT_FALSE(UncountableCondition.has_value());
- ASSERT_EQ(Recipes.size(), 0ull);
-}
-
-} // namespace
-} // namespace llvm
More information about the llvm-commits
mailing list