[llvm-branch-commits] [llvm] [VPlan] Allow early exit loops with side effects with masking in the vector loop (PR #237254)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 02:16:19 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/237254
>From 7b44787c141a3c78570be51d12f084a8d32928a9 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 28 Oct 2026 22:33:08 +0800
Subject: [PATCH 1/2] Precommit tests
---
.../LoopVectorize/early_exit_with_stores.ll | 221 ++++++++++++++++++
1 file changed, 221 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index ad1269aa3c8523..139c9992cc38d7 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1555,3 +1555,224 @@ exit:
}
declare i64 @get_an_unknown_offset();
+
+define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @store_in_latch_after_exit(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.latch
+
+loop.latch:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
+; CHECK-LABEL: define void @conditional_store_after_two_exits(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]]
+; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
+; CHECK: [[LOOP_2]]:
+; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
+; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
+; CHECK: [[LOOP_3]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
+; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT_2]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv
+ %ee2.val = load i16, ptr %ee2.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.2
+
+loop.2:
+ %ee2.cond = icmp eq i16 %ee2.val, 0
+ br i1 %ee2.cond, label %exit.2, label %loop.3
+
+loop.3:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %st.cond = icmp slt i16 %data, 0
+ br i1 %st.cond, label %store, label %loop.latch
+
+store:
+ %inc = add i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ ret void
+
+exit.2:
+ ret void
+}
+
+define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define i16 @store_after_exit_live_out(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
+; CHECK: [[LOOP_STORE]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i16 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.store
+
+loop.store:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ %res = phi i16 [ -1, %loop ], [ %inc, %loop.latch ]
+ ret i16 %res
+}
>From 558bc4c7d167333e6f66531b392dbe634a3c9c32 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 30 Oct 2026 00:16:00 +0800
Subject: [PATCH 2/2] [VPlan] Allow early exit loops with side effects by
masking in the vector loop.
When all blocks with side effects are dominated by every early exit, we can vectorize the loop if we mask off any lanes past the exit. We can do this with control flow:
```mermaid
flowchart TD
header --> exiting
exiting -->| exit mask | sideeffects --> latch --> latch.split
exiting -->| !exit mask | latch.split
```
For every early exiting block we add a branch to the latch on `!mask.beforefirst(exitcond)`. This is implemented in UncountableExitStyle::ReadOnly which is now renamed to MaskedHandleExitInVectorLoop.
This is a complementary exit style to MaskedHandleExitInScalarLoop. If any of the side effects aren't dominated by the early exit it still falls back to the scalar loop style, which is why none of the existing early exit side effect tests changed. Masking in loop is profitable for RISC-V and I'm assuming it's also profitable for AArch64 too, but if not then we can add a TTI hook for a preferred exit style.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../Vectorize/VPlanConstruction.cpp | 4 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 119 ++++++++++-----
.../RISCV/early_exit_with_stores.ll | 15 +-
.../X86/early_exit_with_stores.ll | 18 +--
.../LoopVectorize/early_exit_with_stores.ll | 135 ++++++++----------
.../Vectorize/VPlanVerifierTest.cpp | 5 +-
8 files changed, 176 insertions(+), 132 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 128adbf60f1f22..8a4800a80a22f4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6268,7 +6268,8 @@ LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
return nullptr;
// TODO: Check target preference for style.
- UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly;
+ UncountableExitStyle EEStyle =
+ UncountableExitStyle::MaskedHandleExitInVectorLoop;
if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
EEStyle)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d27473af8a7ca5..d2354f58aba100 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -79,10 +79,11 @@ using VPlanPtr = std::unique_ptr<VPlan>;
/// Different methods of handling early exits.
///
enum class UncountableExitStyle {
- /// No side effects to worry about, so we can process any uncountable exits
- /// in the loop and branch either to the middle block if the trip count was
- /// reached, or an early exitblock to determine which exit was taken.
- ReadOnly,
+ /// Any side effects come after all the uncountable exits and are masked off,
+ /// so we can process any uncountable exits in the loop and branch either to
+ /// the middle block if the trip count was reached, or an early exitblock to
+ /// determine which exit was taken.
+ MaskedHandleExitInVectorLoop,
/// All memory operations other than the load(s) required to determine whether
/// an uncountable exit occurre will be masked based on that condition. If an
/// uncountable exit is taken, then all lanes before the exiting lane will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8a026aba613855..db47293ca8476d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1272,7 +1272,9 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
continue;
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
- assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
+ assert((!R.mayReadFromMemory() ||
+ (VPI && VPI->getOpcode() == Instruction::Store)) &&
+ "unexpected recipe reading memory");
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c75441a3ed2729..548ddc68f4f398 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3554,50 +3554,59 @@ static bool handleUncountableExitsWithSideEffects(
// Returns true if the loop contains stores, the only writes legality accepts in
// early-exit loops.
-static bool loopHasStores(VPBasicBlock *HeaderVPBB) {
- return any_of(vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) {
- return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
- (void)VPBB;
- if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
- return false;
- if (!R.mayWriteToMemory()) {
- assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
- R.isSafeToSpeculativelyExecute()) &&
- "legality only accepts recipes with side effects that are "
- "safe to speculate");
- return false;
- }
- assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
- "legality only accepts stores as writes in early-exit loops");
- return true;
- });
- });
+static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) {
+ return to_vector(make_filter_range(
+ vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) {
+ return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
+ (void)VPBB;
+ if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
+ return false;
+ if (!R.mayWriteToMemory()) {
+ assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
+ R.isSafeToSpeculativelyExecute()) &&
+ "legality only accepts recipes with side effects that are "
+ "safe to speculate");
+ return false;
+ }
+ assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
+ "legality only accepts stores as writes in early-exit loops");
+ return true;
+ });
+ }));
}
bool VPlanTransforms::handleUncountableEarlyExits(
VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
UncountableExitStyle Style) {
-#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
-#endif
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
- if (loopHasStores(HeaderVPBB))
- Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+ // If all side effects are dominated by the early exits, we can mask in the
+ // vector loop. Otherwise we need to handle side effects in the scalar loop.
+ // TODO: Legality only allows side effects that the scalar loop style can
+ // handle, relax it for side effects after all the early exits.
+ SmallVector<VPBasicBlock *> SideEffectVPBBs = blocksWithStores(HeaderVPBB);
+ if (!SideEffectVPBBs.empty()) {
+ auto EarlyExits = vputils::getEarlyExits(Plan, MiddleVPBB);
+ for (auto [EarlyExitingVPBB, _] : EarlyExits)
+ for (VPBasicBlock *VPBB : SideEffectVPBBs)
+ if (!VPDT.properlyDominates(EarlyExitingVPBB, VPBB))
+ Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+ }
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
- if (Style == UncountableExitStyle::ReadOnly &&
+ if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop &&
!areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
- reportVectorizationFailure(
- "Auto-vectorization of early exit loops with potentially "
- "faulting loads is not supported",
- "EarlyExitFaultingLoads", ORE, TheLoop);
- return false;
+ if (SideEffectVPBBs.empty()) {
+ reportVectorizationFailure(
+ "Auto-vectorization of early exit loops with potentially "
+ "faulting loads is not supported",
+ "EarlyExitFaultingLoads", ORE, TheLoop);
+ return false;
+ }
+ Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
}
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
@@ -3649,7 +3658,7 @@ bool VPlanTransforms::handleUncountableEarlyExits(
});
}
- if (Style == UncountableExitStyle::ReadOnly) {
+ if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop) {
// Lanes past the exit can be poison that the scalar loop never branched on,
// so freeze them to prevent branching on poison.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -3659,6 +3668,49 @@ bool VPlanTransforms::handleUncountableEarlyExits(
Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0)));
}
+ if (!SideEffectVPBBs.empty()) {
+ // Split the latch in case it has recipes with side effects.
+ LatchVPBB = LatchVPBB->splitAt(LatchVPBB->getTerminator()->getIterator());
+ LatchBuilder.setInsertPoint(LatchVPBB->getTerminator());
+
+ // Branch to the latch to skip any side effects past the exiting lane.
+ for (EarlyExitInfo &Exit : Exits) {
+ VPBasicBlock *EarlyExitingVPBB = Exit.EarlyExitingVPBB;
+ VPBuilder ExitingBuilder(EarlyExitingVPBB->getTerminator());
+ VPValue *ExitMask = ExitingBuilder.createWidenIntrinsic(
+ Intrinsic::mask_beforefirst,
+ ExitingBuilder.createFreeze(ExitingBuilder.createNaryOp(
+ VPInstruction::MaskedCond, Exit.CondToExit)),
+ Type::getInt1Ty(Plan.getContext()));
+ EarlyExitingVPBB->getTerminator()->setOperand(0, ExitMask);
+ VPBlockUtils::connectBlocks(EarlyExitingVPBB, LatchVPBB);
+ }
+
+ // The latch's exit condition may be poison on lanes past the exit, freeze
+ // it.
+ VPRecipeBase *LatchTerm = LatchVPBB->getTerminator();
+ LatchTerm->setOperand(
+ 0, VPBuilder(LatchTerm).createFreeze(LatchTerm->getOperand(0)));
+
+ // If we took the exiting branch then any uses in the latch and beyond are
+ // poison.
+ VPDT.recalculate(Plan);
+ for (VPBasicBlock *VPBB : {LatchVPBB, MiddleVPBB}) {
+ for (VPRecipeBase &R :
+ make_range(VPBB->getFirstNonPhi(), VPBB->end())) {
+ for (auto [I, Op] : enumerate(R.operands())) {
+ VPRecipeBase *DefR = Op->getDefiningRecipe();
+ if (!DefR || VPDT.dominates(DefR->getParent(), VPBB))
+ continue;
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {HeaderVPBB, Plan.getPoison(Op->getScalarType())}};
+ Defs[DefR->getParent()] = Op;
+ R.setOperand(I, vputils::reconstructSSA(LatchVPBB, Defs));
+ }
+ }
+ }
+ }
+
// An exit's condition is false on any path that didn't go through its
// exiting block.
for (EarlyExitInfo &Exit : Exits) {
@@ -3816,7 +3868,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(
ExitIRI->addIncoming(NewIncoming);
}
- EarlyExitingVPBB->getTerminator()->eraseFromParent();
+ if (EarlyExitingVPBB->getNumSuccessors() == 2)
+ EarlyExitingVPBB->getTerminator()->eraseFromParent();
VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 573ee73e76a439..8ba6d5f81679e5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1277,7 +1277,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1291,15 +1291,16 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK1]]:
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
index 3635a51a925dad..8ffc6ba186b301 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
@@ -12,7 +12,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -55,7 +55,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 3
-; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK: [[PRED_STORE_IF5]]:
; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], 3
; CHECK-NEXT: [[TMP23:%.*]] = shl i64 [[TMP22]], 1
@@ -69,13 +69,15 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP27]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
-; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
+; CHECK-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -169,7 +171,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 139c9992cc38d7..d4f5ef95e5da73 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1564,7 +1564,7 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1577,29 +1577,13 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
-; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1629,32 +1613,49 @@ exit:
define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
; CHECK-LABEL: define void @conditional_store_after_two_exits(
; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]]
-; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
-; CHECK: [[LOOP_2]]:
-; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
-; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
-; CHECK: [[LOOP_3]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
-; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
-; CHECK: [[STORE]]:
-; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP10]], <4 x i1> [[TMP9]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <4 x i16> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP14]], ptr align 2 [[TMP10]], <4 x i1> [[TMP13]])
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false)
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
; CHECK: [[EXIT_2]]:
@@ -1707,7 +1708,7 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1720,34 +1721,16 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
-; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
-; CHECK: [[LOOP_STORE]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[VECTOR_EARLY_EXIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[RES]]
;
entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index e006ed7e4bd124..aaf677b894d2a5 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -578,8 +578,9 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
// Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator
// (3 successors), without wrapping blocks in loop regions.
- auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly,
- /*CreateLoopRegions=*/false);
+ auto Plan =
+ buildVPlan(LoopHeader, UncountableExitStyle::MaskedHandleExitInVectorLoop,
+ /*CreateLoopRegions=*/false);
auto *MiddleVPBB =
cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]);
More information about the llvm-branch-commits
mailing list