[llvm] [LV] Derive max trip count for remainder loops through LCSSA phis (PR #206887)
Muhammed Shiyas N via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 05:23:47 PDT 2026
https://github.com/Shiyas-N updated https://github.com/llvm/llvm-project/pull/206887
>From 17ebcf4c3287c11e7d1472fb82b007d382f88053 Mon Sep 17 00:00:00 2001
From: Shiyas-N <muhammedshiyasn811 at gmail.com>
Date: Mon, 24 Aug 2026 12:08:38 +0000
Subject: [PATCH 1/2] [LV] Add baseline test for remainder loop vectorization
(NFC)
Add a precommit test for a strided main loop followed by a remainder loop, where SCEV cannot determine the remainder loop's trip count through its LCSSA phi. Captures current behavior as a baseline for the following patch.
---
.../X86/remainder-max-trip-count.ll | 177 ++++++++++++++++++
1 file changed, 177 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
diff --git a/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll b/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
new file mode 100644
index 0000000000000..43ebfbbc13a60
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 -S %s | FileCheck %s
+;
+; Test that the vectorizer derives the maximum trip count for a remainder loop
+; following a stride-8 main loop and clamps the vector factor to VF=4.
+
+define void @remainder_after_stride8(ptr noalias %src, i64 %n, ptr noalias %dst) {
+; CHECK-LABEL: @remainder_after_stride8(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-NEXT: br i1 [[CMP_NOT]], label [[LOOP_REMAINDER_PREHEADER:%.*]], label [[LOOP_MAIN_PREHEADER_PREHEADER:%.*]]
+; CHECK: loop.main.preheader.preheader:
+; CHECK-NEXT: br label [[LOOP_MAIN_PREHEADER:%.*]]
+; CHECK: loop.main.preheader:
+; CHECK-NEXT: [[START_MAIN:%.*]] = phi i64 [ [[START_NEXT:%.*]], [[LOOP_MAIN_LATCH:%.*]] ], [ 0, [[LOOP_MAIN_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC:%.*]], i64 [[START_MAIN]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[WIDE_LOAD]], splat (i32 32768)
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i32> [[TMP1]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST:%.*]], i64 [[START_MAIN]]
+; CHECK-NEXT: store <8 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LOOP_MAIN_LATCH]]
+; CHECK: loop.main.latch:
+; CHECK-NEXT: [[START_NEXT]] = add i64 [[START_MAIN]], 8
+; CHECK-NEXT: [[CMP_EXIT:%.*]] = icmp ugt i64 [[START_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_EXIT]], label [[LOOP_REMAINDER_LOOPEXIT:%.*]], label [[LOOP_MAIN_PREHEADER]]
+; CHECK: loop.remainder.loopexit:
+; CHECK-NEXT: [[START_LCSSA:%.*]] = phi i64 [ [[START_NEXT]], [[LOOP_MAIN_LATCH]] ]
+; CHECK-NEXT: br label [[LOOP_REMAINDER_PREHEADER]]
+; CHECK: loop.remainder.preheader:
+; CHECK-NEXT: [[START_REM:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[START_LCSSA]], [[LOOP_REMAINDER_LOOPEXIT]] ]
+; CHECK-NEXT: [[CMP_REM:%.*]] = icmp ult i64 [[START_REM]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_REM]], label [[ITER_CHECK:%.*]], label [[EXIT:%.*]]
+; CHECK: iter.check:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[N]], [[START_REM]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH3:%.*]]
+; CHECK: vector.ph3:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 32
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[START_REM]], [[N_VEC]]
+; CHECK-NEXT: br label [[VECTOR_BODY4:%.*]]
+; CHECK: vector.body4:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH3]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY4]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[START_REM]], [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 8
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 16
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 24
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i32> [[WIDE_LOAD5]], splat (i32 32768)
+; CHECK-NEXT: [[TMP12:%.*]] = add <8 x i32> [[WIDE_LOAD6]], splat (i32 32768)
+; CHECK-NEXT: [[TMP13:%.*]] = add <8 x i32> [[WIDE_LOAD7]], splat (i32 32768)
+; CHECK-NEXT: [[TMP14:%.*]] = add <8 x i32> [[WIDE_LOAD8]], splat (i32 32768)
+; CHECK-NEXT: [[TMP15:%.*]] = and <8 x i32> [[TMP11]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP16:%.*]] = and <8 x i32> [[TMP12]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP17:%.*]] = and <8 x i32> [[TMP13]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP18:%.*]] = and <8 x i32> [[TMP14]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 8
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 16
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 24
+; CHECK-NEXT: store <8 x i32> [[TMP15]], ptr [[TMP19]], align 4
+; CHECK-NEXT: store <8 x i32> [[TMP16]], ptr [[TMP20]], align 4
+; CHECK-NEXT: store <8 x i32> [[TMP17]], ptr [[TMP21]], align 4
+; CHECK-NEXT: store <8 x i32> [[TMP18]], ptr [[TMP22]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK9:%.*]], label [[VECTOR_BODY4]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block9:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK: vec.epilog.iter.check:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: vec.epilog.ph:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP4]], 8
+; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF10]]
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[START_REM]], [[N_VEC11]]
+; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK: vec.epilog.vector.body:
+; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[START_REM]], [[INDEX12]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP25]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i32>, ptr [[TMP26]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = add <8 x i32> [[WIDE_LOAD13]], splat (i32 32768)
+; CHECK-NEXT: [[TMP28:%.*]] = and <8 x i32> [[TMP27]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT: store <8 x i32> [[TMP28]], ptr [[TMP29]], align 4
+; CHECK-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX12]], 8
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC11]]
+; CHECK-NEXT: br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: vec.epilog.middle.block:
+; CHECK-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC11]]
+; CHECK-NEXT: br i1 [[CMP_N15]], label [[EXIT_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK: vec.epilog.scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL16:%.*]] = phi i64 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_REM]], [[ITER_CHECK]] ]
+; CHECK-NEXT: br label [[LOOP_REMAINDER:%.*]]
+; CHECK: loop.remainder:
+; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ [[IV2_NEXT:%.*]], [[LOOP_REMAINDER]] ], [ [[BC_RESUME_VAL16]], [[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV2]]
+; CHECK-NEXT: [[VAL_REM:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
+; CHECK-NEXT: [[ADD16:%.*]] = add i32 [[VAL_REM]], 32768
+; CHECK-NEXT: [[AND17:%.*]] = and i32 [[ADD16]], -65536
+; CHECK-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[IV2]]
+; CHECK-NEXT: store i32 [[AND17]], ptr [[ARRAYIDX18]], align 4
+; CHECK-NEXT: [[IV2_NEXT]] = add nuw i64 [[IV2]], 1
+; CHECK-NEXT: [[EXITCOND_REM:%.*]] = icmp eq i64 [[IV2_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_REM]], label [[EXIT_LOOPEXIT]], label [[LOOP_REMAINDER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp.not = icmp ult i64 %n, 8
+ br i1 %cmp.not, label %loop.remainder.preheader, label %loop.main.preheader
+
+loop.main.preheader:
+ %start.main = phi i64 [ %start.next, %loop.main.latch ], [ 0, %entry ]
+ br label %loop.main
+
+loop.main:
+ %iv1 = phi i64 [ 0, %loop.main.preheader ], [ %iv1.next, %loop.main ]
+ %idx = or disjoint i64 %iv1, %start.main
+ %arrayidx = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %idx
+ %val = load i32, ptr %arrayidx, align 4
+ %add5 = add i32 %val, 32768
+ %and = and i32 %add5, -65536
+ %arrayidx8 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %idx
+ store i32 %and, ptr %arrayidx8, align 4
+ %iv1.next = add nuw nsw i64 %iv1, 1
+ %exitcond = icmp eq i64 %iv1.next, 8
+ br i1 %exitcond, label %loop.main.latch, label %loop.main
+
+loop.main.latch:
+ %start.next = add i64 %start.main, 8
+ %cmp.exit = icmp ugt i64 %start.next, %n
+ br i1 %cmp.exit, label %loop.remainder.loopexit, label %loop.main.preheader
+
+loop.remainder.loopexit:
+ %start.lcssa = phi i64 [ %start.next, %loop.main.latch ]
+ br label %loop.remainder.preheader
+
+loop.remainder.preheader:
+ %start.rem = phi i64 [ 0, %entry ], [ %start.lcssa, %loop.remainder.loopexit ]
+ %cmp.rem = icmp ult i64 %start.rem, %n
+ br i1 %cmp.rem, label %loop.remainder, label %exit
+
+loop.remainder:
+ %iv2 = phi i64 [ %start.rem, %loop.remainder.preheader ], [ %iv2.next, %loop.remainder ]
+ %arrayidx15 = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv2
+ %val.rem = load i32, ptr %arrayidx15, align 4
+ %add16 = add i32 %val.rem, 32768
+ %and17 = and i32 %add16, -65536
+ %arrayidx18 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv2
+ store i32 %and17, ptr %arrayidx18, align 4
+ %iv2.next = add nuw i64 %iv2, 1
+ %exitcond.rem = icmp eq i64 %iv2.next, %n
+ br i1 %exitcond.rem, label %exit, label %loop.remainder
+
+exit:
+ ret void
+}
>From c3323e79f63d2e08f0a01f4af4ca4315fa70fc39 Mon Sep 17 00:00:00 2001
From: Shiyas-N <muhammedshiyasn811 at gmail.com>
Date: Mon, 24 Aug 2026 12:09:03 +0000
Subject: [PATCH 2/2] [LV] Derive max trip count for remainder loops through
LCSSA phis
When a remainder loop follows a strided main loop, SCEV fails to determine the remainder loop's max trip count because its starting value is an LCSSA phi.
Add getMaxTCForRemainderLoop() to derive the max trip count directly from the branch conditions guarding the LCSSA phi. This is used as a fallback in getMaxTCFromNonZeroRange(), allowing clampVFByMaxTripCount() to clamp the vector factor appropriately.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 129 ++++++++++++++++++
.../X86/remainder-max-trip-count.ll | 79 +++--------
2 files changed, 146 insertions(+), 62 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f1ca4061bfd9e..9ab709ae2ff72 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -445,6 +445,119 @@ static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
return ElementCount::getFixed(0);
}
+/// Derives the maximum trip count for the "remainder after a strided main loop"
+/// pattern by examining the predecessors of \p StartVal (an LCSSA phi).
+/// This acts as a fallback when SCEV cannot trace bounds through LCSSA phis.
+/// Returns the derived maximum trip count (Stride - 1), or 0 if unproven.
+static unsigned getMaxTCForRemainderLoop(Value *StartVal, Value *Bound) {
+ using namespace PatternMatch;
+
+ auto *StartPhi = dyn_cast<PHINode>(StartVal);
+ if (!StartPhi)
+ return 0;
+
+ BasicBlock *PhiBB = StartPhi->getParent();
+ unsigned NumIncoming = StartPhi->getNumIncomingValues();
+ if (NumIncoming == 0)
+ return 0;
+
+ unsigned MaxTCBound = 0;
+
+ for (unsigned i = 0; i < NumIncoming; ++i) {
+ Value *InVal = StartPhi->getIncomingValue(i);
+ BasicBlock *CheckBB = StartPhi->getIncomingBlock(i);
+ BasicBlock *LastBB = PhiBB;
+ Value *CheckVal = InVal;
+ unsigned BoundForThisPred = 0;
+
+ // Trace through single-value LCSSA phis (up to a small depth).
+ for (unsigned Depth = 0; Depth < 4; ++Depth) {
+ auto *LCSSAPhi = dyn_cast<PHINode>(CheckVal);
+ if (!LCSSAPhi || LCSSAPhi->getNumIncomingValues() != 1)
+ break;
+ LastBB = CheckBB;
+ CheckVal = LCSSAPhi->getIncomingValue(0);
+ CheckBB = LCSSAPhi->getIncomingBlock(0);
+ }
+
+ // Walk up single-predecessor blocks to find the bounding condition.
+ for (unsigned Depth = 0; Depth < 4; ++Depth) {
+ if (auto *CBI = dyn_cast<CondBrInst>(CheckBB->getTerminator())) {
+ auto *ICmp = dyn_cast<ICmpInst>(CBI->getCondition());
+ if (!ICmp)
+ break;
+
+ ICmpInst::Predicate Pred = ICmp->getPredicate();
+ Value *Op0 = ICmp->getOperand(0);
+ Value *Op1 = ICmp->getOperand(1);
+
+ if (CBI->getSuccessor(1) == LastBB)
+ Pred = ICmpInst::getInversePredicate(Pred);
+ else if (CBI->getSuccessor(0) != LastBB)
+ break;
+
+ uint64_t Stride = 0;
+
+ // Pattern 1: entry path (Bound < Stride) and InVal == 0
+ if (match(InVal, m_Zero())) {
+ if ((Pred == ICmpInst::ICMP_ULT && Op0 == Bound &&
+ match(Op1, m_ConstantInt(Stride))) ||
+ (Pred == ICmpInst::ICMP_UGT && Op1 == Bound &&
+ match(Op0, m_ConstantInt(Stride)))) {
+ if (Stride >= 2 && Stride <= 4096) {
+ BoundForThisPred = Stride - 1;
+ break;
+ }
+ }
+ }
+
+ // Pattern 2: loop exit path
+ // Form A: CheckVal is (start + Stride) and condition is (CheckVal >
+ // Bound)
+ if ((Pred == ICmpInst::ICMP_UGT && Op0 == CheckVal && Op1 == Bound &&
+ match(CheckVal, m_c_Add(m_Value(), m_ConstantInt(Stride)))) ||
+ (Pred == ICmpInst::ICMP_ULT && Op1 == CheckVal && Op0 == Bound &&
+ match(CheckVal, m_c_Add(m_Value(), m_ConstantInt(Stride))))) {
+ if (Stride >= 2 && Stride <= 4096) {
+ BoundForThisPred = Stride - 1;
+ break;
+ }
+ }
+
+ // Form B: condition is (CheckVal + Stride > Bound)
+ if ((Pred == ICmpInst::ICMP_UGT && Op1 == Bound &&
+ match(Op0,
+ m_c_Add(m_Specific(CheckVal), m_ConstantInt(Stride)))) ||
+ (Pred == ICmpInst::ICMP_ULT && Op0 == Bound &&
+ match(Op1,
+ m_c_Add(m_Specific(CheckVal), m_ConstantInt(Stride))))) {
+ if (Stride >= 2 && Stride <= 4096) {
+ BoundForThisPred = Stride - 1;
+ break;
+ }
+ }
+
+ break; // Found a conditional branch but didn't match.
+ }
+
+ // Walk backward to the unique predecessor block.
+ LastBB = CheckBB;
+ CheckBB = CheckBB->getUniquePredecessor();
+ if (!CheckBB)
+ break;
+ }
+
+ if (BoundForThisPred == 0)
+ return 0; // Predecessor could not be bounded.
+
+ MaxTCBound = std::max(MaxTCBound, BoundForThisPred);
+ }
+
+ LLVM_DEBUG(dbgs() << "LV: Derived max TC " << MaxTCBound
+ << " from remainder-of-stride pattern\n");
+ return MaxTCBound;
+}
+
/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
/// zero from the range. Only valid when not folding the tail, as the minimum
/// iteration count check guards against a zero trip count. Returns 0 if
@@ -460,6 +573,22 @@ static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
APInt MaxTCFromRange = TCRange.getUnsignedMax();
if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
return MaxTCFromRange.getZExtValue();
+
+ // Fallback: detect the "remainder of a strided loop" pattern.
+ // Use Loop::getBounds to extract the IV start value and loop bound,
+ // then check if the start value is an LCSSA phi from a strided loop.
+ if (auto Bounds = L->getBounds(*SE)) {
+ if (Bounds->getDirection() == Loop::LoopBounds::Direction::Increasing &&
+ Bounds->getStepValue() &&
+ PatternMatch::match(Bounds->getStepValue(), PatternMatch::m_One())) {
+ Value *StartVal = &Bounds->getInitialIVValue();
+ Value *BoundVal = &Bounds->getFinalIVValue();
+ if (L->isLoopInvariant(BoundVal))
+ if (unsigned MaxTC = getMaxTCForRemainderLoop(StartVal, BoundVal))
+ return MaxTC;
+ }
+ }
+
return 0;
}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll b/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
index 43ebfbbc13a60..e19fb6a57fd23 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/remainder-max-trip-count.ll
@@ -37,80 +37,35 @@ define void @remainder_after_stride8(ptr noalias %src, i64 %n, ptr noalias %dst)
; CHECK-NEXT: [[START_REM:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[START_LCSSA]], [[LOOP_REMAINDER_LOOPEXIT]] ]
; CHECK-NEXT: [[CMP_REM:%.*]] = icmp ult i64 [[START_REM]], [[N]]
; CHECK-NEXT: br i1 [[CMP_REM]], label [[ITER_CHECK:%.*]], label [[EXIT:%.*]]
-; CHECK: iter.check:
+; CHECK: loop.remainder.preheader1:
; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[N]], [[START_REM]]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
-; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP4]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH3:%.*]]
-; CHECK: vector.ph3:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 32
+; CHECK: vector.ph2:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[START_REM]], [[N_VEC]]
; CHECK-NEXT: br label [[VECTOR_BODY4:%.*]]
-; CHECK: vector.body4:
+; CHECK: vector.body3:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH3]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY4]] ]
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[START_REM]], [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 8
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 16
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 24
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i32>, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i32> [[WIDE_LOAD5]], splat (i32 32768)
-; CHECK-NEXT: [[TMP12:%.*]] = add <8 x i32> [[WIDE_LOAD6]], splat (i32 32768)
-; CHECK-NEXT: [[TMP13:%.*]] = add <8 x i32> [[WIDE_LOAD7]], splat (i32 32768)
-; CHECK-NEXT: [[TMP14:%.*]] = add <8 x i32> [[WIDE_LOAD8]], splat (i32 32768)
-; CHECK-NEXT: [[TMP15:%.*]] = and <8 x i32> [[TMP11]], splat (i32 -65536)
-; CHECK-NEXT: [[TMP16:%.*]] = and <8 x i32> [[TMP12]], splat (i32 -65536)
-; CHECK-NEXT: [[TMP17:%.*]] = and <8 x i32> [[TMP13]], splat (i32 -65536)
-; CHECK-NEXT: [[TMP18:%.*]] = and <8 x i32> [[TMP14]], splat (i32 -65536)
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[WIDE_LOAD4]], splat (i32 32768)
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i32> [[TMP8]], splat (i32 -65536)
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 8
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 16
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP19]], i64 24
-; CHECK-NEXT: store <8 x i32> [[TMP15]], ptr [[TMP19]], align 4
-; CHECK-NEXT: store <8 x i32> [[TMP16]], ptr [[TMP20]], align 4
-; CHECK-NEXT: store <8 x i32> [[TMP17]], ptr [[TMP21]], align 4
-; CHECK-NEXT: store <8 x i32> [[TMP18]], ptr [[TMP22]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP19]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK9:%.*]], label [[VECTOR_BODY4]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block9:
+; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK5:%.*]], label [[VECTOR_BODY4]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block5:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
-; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP4]], 8
-; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF10]]
-; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[START_REM]], [[N_VEC11]]
-; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[START_REM]], [[INDEX12]]
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP25]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i32>, ptr [[TMP26]], align 4
-; CHECK-NEXT: [[TMP27:%.*]] = add <8 x i32> [[WIDE_LOAD13]], splat (i32 32768)
-; CHECK-NEXT: [[TMP28:%.*]] = and <8 x i32> [[TMP27]], splat (i32 -65536)
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP25]]
-; CHECK-NEXT: store <8 x i32> [[TMP28]], ptr [[TMP29]], align 4
-; CHECK-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX12]], 8
-; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC11]]
-; CHECK-NEXT: br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC11]]
-; CHECK-NEXT: br i1 [[CMP_N15]], label [[EXIT_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
-; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL16:%.*]] = phi i64 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_REM]], [[ITER_CHECK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK5]] ], [ [[START_REM]], [[ITER_CHECK]] ]
; CHECK-NEXT: br label [[LOOP_REMAINDER:%.*]]
; CHECK: loop.remainder:
-; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ [[IV2_NEXT:%.*]], [[LOOP_REMAINDER]] ], [ [[BC_RESUME_VAL16]], [[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ [[IV2_NEXT:%.*]], [[LOOP_REMAINDER]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ]
; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV2]]
; CHECK-NEXT: [[VAL_REM:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
; CHECK-NEXT: [[ADD16:%.*]] = add i32 [[VAL_REM]], 32768
@@ -119,7 +74,7 @@ define void @remainder_after_stride8(ptr noalias %src, i64 %n, ptr noalias %dst)
; CHECK-NEXT: store i32 [[AND17]], ptr [[ARRAYIDX18]], align 4
; CHECK-NEXT: [[IV2_NEXT]] = add nuw i64 [[IV2]], 1
; CHECK-NEXT: [[EXITCOND_REM:%.*]] = icmp eq i64 [[IV2_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_REM]], label [[EXIT_LOOPEXIT]], label [[LOOP_REMAINDER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_REM]], label [[EXIT_LOOPEXIT]], label [[LOOP_REMAINDER]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: exit.loopexit:
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
More information about the llvm-commits
mailing list