[llvm] [LV] Follow up to uncountable exit with side effects vectorization (PR #201589)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 4 06:51:05 PDT 2026
https://github.com/huntergr-arm created https://github.com/llvm/llvm-project/pull/201589
Addressing post-commit comments on #178454.
>From cf69b146a5f7610078b5d432ce8f26282a98e338 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 3 Jun 2026 16:12:23 +0000
Subject: [PATCH] [LV] Follow up to uncountable exit with side effects
vectorization
Addressing post-commit comments on #178454.
---
.../Vectorize/LoopVectorizationLegality.cpp | 11 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 120 ++++++++++--------
.../AArch64/early_exit_with_stores.ll | 6 +-
.../RISCV/early_exit_with_stores.ll | 6 +-
.../VPlan/early_exit_with_stores_vplan.ll | 4 +-
.../LoopVectorize/early_exit_with_stores.ll | 8 +-
.../interleave_uncountable_exits.ll | 2 +-
...rized_conditional_ops_uncountable_exits.ll | 8 +-
.../tail_fold_uncountable_exits.ll | 2 +-
9 files changed, 87 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 214b5cff8a03a..f8fb9d6d6a98f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1946,16 +1946,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- // TODO: Remove this restriction once we're sure it's safe to do so.
- // Handling stores to invariant addresses will be slightly different
- // based on the vectorization style chosen. If we bail out to a scalar
- // tail before executing any lane that would take the uncountable exit,
- // then the store that occurs in the scalar loop would suffice.
- //
- // If we instead handle the lane taking the uncountable exit within the
- // vectorized loop, then we will have to ensure that we extract the
- // last active lane at that point in the loop instead of the last lane
- // of the vector before performing a scalar store.
+ // TODO: Remove this restriction, should be straightforward to support.
if (UncountableExitType != UncountableExitTrait::None &&
!LAI->getStoresToInvariantAddresses().empty()) {
LLVM_DEBUG(dbgs() << "LV: Cannot vectorize early exit loops with stores to "
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5df097628ba7f..85b4fd697f047 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4210,6 +4210,31 @@ struct EarlyExitInfo {
/// Update \p Plan to mask memory operations in the loop based on whether the
/// early exit is taken or not.
+//
+// We're currently expecting to find a loop with properties similar to the
+// following C code:
+//
+// #define N 10000
+// int cond[N];
+// int src[N];
+// int dst[N];
+//
+// void foo(int threshold) {
+// for (int i=0; i<N; ++i) {
+// if (cond[i] > threshold)
+// break;
+// dst[i] = src[i] + 42;
+// }
+// }
+//
+// The loop must have a single unconditional load contributing to the
+// uncountable exit comparison, and the other term must be loop-invariant.
+// There must also be a counted exit. Other memory operations in the loop can
+// take place before or after the uncountable exit, but must also be
+// unconditional. All potential accesses to the memory used for the load for the
+// exit condition must be guaranteed to be dereferenceable. Any stores within
+// the loop must not alias with any other memory operations.
+//
static bool handleUncountableExitsWithSideEffects(
VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
@@ -4241,15 +4266,20 @@ static bool handleUncountableExitsWithSideEffects(
return false;
// Find load contributing to condition.
- VPRecipeBase *CondLoad = nullptr;
- for (auto *Recipe : ConditionRecipes) {
- if (match(Recipe, m_VPInstruction<Instruction::Load>(m_VPValue()))) {
- // TODO: Support more than one load. Needs legality updates too.
- assert(CondLoad == nullptr && "Too many condition loads");
- CondLoad = Recipe;
- }
- }
- assert(CondLoad && "Couldn't find load");
+ // At the moment LoopVectorizationLegality only supports a single
+ // early-exit expression with a compare and a single load that must
+ // be unconditional.
+ // TODO: Support more than one load.
+ auto *Load =
+ find_singleton<VPInstruction>(ConditionRecipes, [](auto *I, bool _) {
+ return match(I, m_VPInstruction<Instruction::Load>(m_VPValue()))
+ ? I
+ : nullptr;
+ });
+ assert(Load && "Couldn't find load");
+ // TODO: Support conditional loads for uncountable exits.
+ assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
+ "Uncountable exit condition load is conditional.");
// Ensure that we are guaranteed to be able to dereference the memory used
// for determining the uncountable exit for the maximum possible number of
@@ -4259,7 +4289,6 @@ static bool handleUncountableExitsWithSideEffects(
// all possible addresses are dereferenceable.
{
SmallVector<const SCEVPredicate *, 4> Predicates;
- VPSingleDefRecipe *Load = cast<VPSingleDefRecipe>(CondLoad);
VPValue *Ptr = Load->getOperand(0);
const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, TheLoop);
const DataLayout &DL = Plan.getDataLayout();
@@ -4272,54 +4301,36 @@ static bool handleUncountableExitsWithSideEffects(
return false;
}
- // Check GEPs to see if we can link them to a widen IV recipe with a step of
- // 1; we're only interested in contiguous accesses for the condition load
- // right now.
- for (auto *GEP : GEPs) {
- VPValue *MaybeIV = nullptr;
- if (!match(GEP, m_VPInstruction<Instruction::GetElementPtr>(
- m_LiveIn(), m_VPValue(MaybeIV))))
- return false;
-
- auto *WIV = dyn_cast<VPWidenInductionRecipe>(MaybeIV);
- if (!WIV)
- return false;
-
- if (!match(WIV->getStartValue(), m_SpecificInt(0)) ||
- !match(WIV->getStepValue(), m_SpecificInt(1)))
- return false;
- }
-
- // Find an insertion point. Default to the end of the header but override
- // if we find a memory op that needs masking before the condition load.
- auto InsertIt = HeaderVPBB->end();
- VPRecipeBase *CondR = (*Cond)->getDefiningRecipe();
- bool CondMoveNeeded = CondR->getParent() != HeaderVPBB;
- for (VPRecipeBase &R : *HeaderVPBB) {
- if (&R == CondLoad)
- continue;
+ // Check for a single GEP for the condition load to see if we can link it to
+ // a widen IV recipe with a step of 1; we're only interested in contiguous
+ // accesses for the condition load right now.
+ auto *IV = cast<VPWidenInductionRecipe>(&HeaderVPBB->front());
+ if (!match(IV->getStartValue(), m_SpecificInt(0)) ||
+ !match(IV->getStepValue(), m_SpecificInt(1)))
+ return false;
+ if (GEPs.size() != 1)
+ return false;
+ if (!match(GEPs.front(), m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_Specific(IV))))
+ return false;
- if (R.mayReadOrWriteMemory()) {
- if (!VPDT.properlyDominates(CondR, &R)) {
- CondMoveNeeded = true;
- InsertIt = R.getIterator();
- }
- break;
- }
+ // We want to guarantee that the uncountable exit condition (and the mask
+ // we will generate from it) are available for all operations in the loop
+ // that need to be masked. If the condition recipes are not already the first
+ // recipes in the header after the last phi, move them there.
+ auto InsertIt = HeaderVPBB->getFirstNonPhi();
+ while (InsertIt != HeaderVPBB->end() &&
+ is_contained(ConditionRecipes, &*InsertIt)) {
+ erase(ConditionRecipes, &*InsertIt);
+ InsertIt++;
}
-
- // If another memory operation would take place before the comparison to
- // determine whether to exit early or the comparison doesn't take place in
- // the header, move the comparison (and supporting recipes).
- if (CondMoveNeeded)
- for (auto *Recipe : reverse(ConditionRecipes))
- Recipe->moveBefore(*HeaderVPBB, InsertIt);
+ for (auto *Recipe : reverse(ConditionRecipes))
+ Recipe->moveBefore(*HeaderVPBB, InsertIt);
// Create a mask to represent all lanes that fully execute in the vector loop,
// stopping short of any early exit.
VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
VPValue *FirstActive = MaskBuilder.createFirstActiveLane(*Cond);
- VPValue *IV = cast<VPSingleDefRecipe>(&HeaderVPBB->front());
Type *IVScalarTy = IV->getScalarType();
Type *FirstActiveTy = FirstActive->getScalarType();
VPValue *ALMMultiplier = Plan.getConstantInt(IVScalarTy, 1);
@@ -4333,7 +4344,7 @@ static bool handleUncountableExitsWithSideEffects(
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
for (VPRecipeBase &R : *VPBB)
- if (R.mayReadOrWriteMemory() && &R != CondLoad) {
+ if (R.mayReadOrWriteMemory() && &R != Load) {
// TODO: Handle conditional memory operations in the loop.
if (!VPDT.dominates(R.getParent(), LatchVPBB))
return false;
@@ -4360,6 +4371,11 @@ static bool handleUncountableExitsWithSideEffects(
if (range_size(Phis) != 1)
return false;
VPPhi *ContinueIV = cast<VPPhi>(Phis.begin());
+ // Make sure we're referring to the same IV.
+ assert(
+ match(ContinueIV->getOperand(0),
+ m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
+ "Continuing from different IV");
ContinueIV->setOperand(0, ExitIV);
return true;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 46a21f8a95079..bd202423a17ee 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -64,12 +64,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -343,12 +343,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
@@ -1141,12 +1141,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 2b73a995d73bb..65fb9684de8df 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -62,12 +62,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -341,12 +341,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
@@ -1184,12 +1184,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 00ef6968159d1..b1abf9c0833b4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -23,13 +23,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
@@ -197,13 +197,13 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index affe5e30c1146..c2fd759504759 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -56,12 +56,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -328,12 +328,12 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
@@ -1022,12 +1022,12 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
@@ -1184,12 +1184,12 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index 26f4ee4b6ead8..d1ef3a607a243 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -12,12 +12,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index 942212dfc63d1..c572f15086082 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -13,15 +13,15 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP26]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP26]]
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP35]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index 73fef71ce774f..58ff87764459a 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -15,12 +15,12 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
More information about the llvm-commits
mailing list