[llvm] [LV] Derive max trip count for remainder loops through LCSSA phis (PR #206887)

Muhammed Shiyas N via llvm-commits llvm-commits at lists.llvm.org
Sun Jul 5 21:32:52 PDT 2026


https://github.com/Shiyas-N updated https://github.com/llvm/llvm-project/pull/206887

>From 7fde1541dad058cff1c265f0fb8cdc628cd33349 Mon Sep 17 00:00:00 2001
From: Shiyas-N <muhammedshiyasn811 at gmail.com>
Date: Tue, 30 Jun 2026 16:28:20 +0000
Subject: [PATCH 1/2] [LV] Add baseline test for remainder loop vectorization
 (NFC)

Add a precommit test for a strided main loop followed by a remainder
loop, where SCEV cannot determine the remainder loop's trip count
through its LCSSA phi. Captures current behavior (remainder loop gets
vectorized with runtime checks) as a baseline for the following patch.
---
 .../LoopVectorize/remainder-max-trip-count.ll | 163 ++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll

diff --git a/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll b/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll
new file mode 100644
index 0000000000000..8ce0624289cfb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s
+;
+; When a remainder loop follows a stride-8 main loop, the remainder loop
+; has a proven max trip count of 7 (it handles n%8 elements). SCEV fails
+; to derive this bound because it does not trace the LCSSA phi back to the
+; preceding loop's exit condition. The loop vectorizer should detect this
+; "remainder of stride" pattern and avoid vectorizing the remainder loop
+; since its trip count is too small to benefit from vectorization.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; The remainder loop (for.body14) should NOT be vectorized.
+
+define void @remainder_after_stride8(ptr %src, i64 %n, ptr %dst) {
+; CHECK-LABEL: @remainder_after_stride8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoaddr ptr [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST:%.*]] to i64
+; CHECK-NEXT:    [[CMP_NOT36:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-NEXT:    br i1 [[CMP_NOT36]], label [[FOR_COND12_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader.preheader:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ADD38:%.*]], 8
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp ugt i64 [[ADD]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[FOR_COND12_PREHEADER_LOOPEXIT:%.*]], label [[FOR_COND1_PREHEADER]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    [[ADD38]] = phi i64 [ [[ADD]], [[FOR_COND_LOOPEXIT:%.*]] ], [ 8, [[FOR_COND1_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[I_037:%.*]] = phi i64 [ [[ADD38]], [[FOR_COND_LOOPEXIT]] ], [ 0, [[FOR_COND1_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.cond12.preheader.loopexit:
+; CHECK-NEXT:    [[ADD38_LCSSA:%.*]] = phi i64 [ [[ADD38]], [[FOR_COND_LOOPEXIT]] ]
+; CHECK-NEXT:    br label [[FOR_COND12_PREHEADER]]
+; CHECK:       for.cond12.preheader:
+; CHECK-NEXT:    [[I_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD38_LCSSA]], [[FOR_COND12_PREHEADER_LOOPEXIT]] ]
+; CHECK-NEXT:    [[CMP1339:%.*]] = icmp ult i64 [[I_0_LCSSA]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP1339]], label [[FOR_BODY14_PREHEADER:%.*]], label [[FOR_END21:%.*]]
+; CHECK:       for.body14.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[N]], [[I_0_LCSSA]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 32
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[I_0_LCSSA]], [[N_VEC]]
+; CHECK-NEXT:    br label [[FOR_BODY14:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY14]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP4]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 32768)
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i32> [[WIDE_LOAD3]], splat (i32 32768)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <4 x i32> [[TMP6]], splat (i32 -65536)
+; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i32> [[TMP7]], splat (i32 -65536)
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP10]], i64 4
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY14]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END21_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ [[I_0_LCSSA]], [[FOR_BODY14_PREHEADER]] ], [ [[I_0_LCSSA]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY15:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[ADD4:%.*]] = or disjoint i64 [[INDVARS_IV]], [[I_037]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[ADD4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD5:%.*]] = add i32 [[TMP13]], 32768
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD5]], -65536
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[ADD4]]
+; CHECK-NEXT:    store i32 [[AND]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_LOOPEXIT]], label [[FOR_BODY3]]
+; CHECK:       for.body14:
+; CHECK-NEXT:    [[I_140:%.*]] = phi i64 [ [[INC20:%.*]], [[FOR_BODY15]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[I_140]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[ADD16:%.*]] = add i32 [[TMP14]], 32768
+; CHECK-NEXT:    [[AND17:%.*]] = and i32 [[ADD16]], -65536
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[I_140]]
+; CHECK-NEXT:    store i32 [[AND17]], ptr [[ARRAYIDX18]], align 4
+; CHECK-NEXT:    [[INC20]] = add nuw i64 [[I_140]], 1
+; CHECK-NEXT:    [[EXITCOND42_NOT:%.*]] = icmp eq i64 [[INC20]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND42_NOT]], label [[FOR_END21_LOOPEXIT]], label [[FOR_BODY15]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       for.end21.loopexit:
+; CHECK-NEXT:    br label [[FOR_END21]]
+; CHECK:       for.end21:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp.not36 = icmp ult i64 %n, 8
+  br i1 %cmp.not36, label %for.cond12.preheader, label %for.cond1.preheader.preheader
+
+for.cond1.preheader.preheader:
+  br label %for.cond1.preheader
+
+for.cond.loopexit:
+  %add = add i64 %add38, 8
+  %cmp.not = icmp ugt i64 %add, %n
+  br i1 %cmp.not, label %for.cond12.preheader.loopexit, label %for.cond1.preheader
+
+for.cond1.preheader:
+  %add38 = phi i64 [ %add, %for.cond.loopexit ], [ 8, %for.cond1.preheader.preheader ]
+  %i.037 = phi i64 [ %add38, %for.cond.loopexit ], [ 0, %for.cond1.preheader.preheader ]
+  br label %for.body3
+
+for.cond12.preheader.loopexit:
+  %add38.lcssa = phi i64 [ %add38, %for.cond.loopexit ]
+  br label %for.cond12.preheader
+
+for.cond12.preheader:
+  %i.0.lcssa = phi i64 [ 0, %entry ], [ %add38.lcssa, %for.cond12.preheader.loopexit ]
+  %cmp1339 = icmp ult i64 %i.0.lcssa, %n
+  br i1 %cmp1339, label %for.body14.preheader, label %for.end21
+
+for.body14.preheader:
+  br label %for.body14
+
+for.body3:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body3 ]
+  %add4 = or disjoint i64 %indvars.iv, %i.037
+  %arrayidx = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %add4
+  %0 = load i32, ptr %arrayidx, align 4
+  %add5 = add i32 %0, 32768
+  %and = and i32 %add5, -65536
+  %arrayidx8 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %add4
+  store i32 %and, ptr %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond.not, label %for.cond.loopexit, label %for.body3
+
+for.body14:
+  %i.140 = phi i64 [ %inc20, %for.body14 ], [ %i.0.lcssa, %for.body14.preheader ]
+  %arrayidx15 = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %i.140
+  %1 = load i32, ptr %arrayidx15, align 4
+  %add16 = add i32 %1, 32768
+  %and17 = and i32 %add16, -65536
+  %arrayidx18 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %i.140
+  store i32 %and17, ptr %arrayidx18, align 4
+  %inc20 = add nuw i64 %i.140, 1
+  %exitcond42.not = icmp eq i64 %inc20, %n
+  br i1 %exitcond42.not, label %for.end21.loopexit, label %for.body14
+
+for.end21.loopexit:
+  br label %for.end21
+
+for.end21:
+  ret void
+}

>From 877f345d2a8a1f08f5509c6b38077f9b362fc0db Mon Sep 17 00:00:00 2001
From: Shiyas-N <muhammedshiyasn811 at gmail.com>
Date: Tue, 30 Jun 2026 16:42:16 +0000
Subject: [PATCH 2/2] [LV] Derive max trip count for remainder loops through
 LCSSA phis

SCEV can't see a remainder loop's max trip count through its LCSSA phi,
causing unnecessary vectorization with runtime checks on loops that
provably run only a few iterations.

Add getMaxTCForRemainderLoop() to derive this bound by matching known
patterns at the phi's incoming branches, and use it in computeMaxVF to
skip vectorization when the overhead can't be amortized.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 117 ++++++++++++++++++
 .../LoopVectorize/remainder-max-trip-count.ll |  45 +------
 2 files changed, 121 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f1ca4061bfd9e..66a14ff4e78e3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -445,6 +445,96 @@ static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
   return ElementCount::getFixed(0);
 }
 
+/// Derives the maximum trip count for the "remainder after a strided main loop"
+/// pattern by examining the predecessors of \p StartVal (an LCSSA phi).
+/// This acts as a fallback when SCEV cannot trace bounds through LCSSA phis.
+/// Returns the derived maximum trip count (the stride), or 0 if unproven.
+static unsigned getMaxTCForRemainderLoop(Value *StartVal, Value *Bound) {
+  using namespace PatternMatch;
+
+  auto *StartPhi = dyn_cast<PHINode>(StartVal);
+  if (!StartPhi)
+    return 0;
+
+  BasicBlock *PhiBB = StartPhi->getParent();
+  unsigned NumIncoming = StartPhi->getNumIncomingValues();
+  if (NumIncoming == 0)
+    return 0;
+
+  unsigned MaxTCBound = 0;
+
+  for (unsigned i = 0; i < NumIncoming; ++i) {
+    Value *InVal = StartPhi->getIncomingValue(i);
+    BasicBlock *CheckBB = StartPhi->getIncomingBlock(i);
+    Value *CheckVal = InVal;
+    unsigned BoundForThisPred = 0;
+
+    // Trace through single-value LCSSA phis.
+    while (auto *LCSSAPhi = dyn_cast<PHINode>(CheckVal)) {
+      if (LCSSAPhi->getNumIncomingValues() != 1)
+        break;
+      CheckVal = LCSSAPhi->getIncomingValue(0);
+      CheckBB = LCSSAPhi->getIncomingBlock(0);
+    }
+
+    // Walk up single-predecessor blocks to find the bounding condition.
+    for (unsigned Depth = 0; Depth < 4; ++Depth) {
+      if (auto *CBI = dyn_cast<CondBrInst>(CheckBB->getTerminator())) {
+        auto *ICmp = dyn_cast<ICmpInst>(CBI->getCondition());
+        if (!ICmp)
+          break;
+
+        ICmpInst::Predicate Pred = ICmp->getPredicate();
+        Value *Op0 = ICmp->getOperand(0);
+        Value *Op1 = ICmp->getOperand(1);
+
+        // Invert the predicate if the false branch leads to PhiBB.
+        bool TrueBranchToPhiBB =
+            CBI->getSuccessor(0) == PhiBB ||
+            CBI->getSuccessor(0) == StartPhi->getIncomingBlock(i);
+        if (!TrueBranchToPhiBB)
+          Pred = ICmpInst::getInversePredicate(Pred);
+
+        uint64_t Stride = 0;
+
+        // Pattern 1: entry path (Bound < Stride) and InVal == 0
+        if (Pred == ICmpInst::ICMP_ULT && Op0 == Bound &&
+            match(InVal, m_Zero()) && match(Op1, m_ConstantInt(Stride))) {
+          if (Stride >= 2 && Stride <= 4096) {
+            BoundForThisPred = Stride;
+            break;
+          }
+        }
+
+        // Pattern 2: loop exit path (InVal + Stride > Bound)
+        if (Pred == ICmpInst::ICMP_UGT && Op1 == Bound &&
+            match(Op0, m_c_Add(m_Specific(CheckVal), m_ConstantInt(Stride)))) {
+          if (Stride >= 2 && Stride <= 4096) {
+            BoundForThisPred = Stride;
+            break;
+          }
+        }
+
+        break; // Found a conditional branch but didn't match.
+      }
+
+      // Walk backward to the unique predecessor block.
+      CheckBB = CheckBB->getUniquePredecessor();
+      if (!CheckBB)
+        break;
+    }
+
+    if (BoundForThisPred == 0)
+      return 0; // Predecessor could not be bounded.
+
+    MaxTCBound = std::max(MaxTCBound, BoundForThisPred);
+  }
+
+  LLVM_DEBUG(dbgs() << "LV: Derived max TC " << MaxTCBound
+                    << " from remainder-of-stride pattern\n");
+  return MaxTCBound;
+}
+
 /// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
 /// zero from the range. Only valid when not folding the tail, as the minimum
 /// iteration count check guards against a zero trip count. Returns 0 if
@@ -460,6 +550,18 @@ static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
   APInt MaxTCFromRange = TCRange.getUnsignedMax();
   if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
     return MaxTCFromRange.getZExtValue();
+
+  // Fallback: detect the "remainder of a strided loop" pattern.
+  // Use Loop::getBounds to extract the IV start value and loop bound,
+  // then check if the start value is an LCSSA phi from a strided loop.
+  if (auto Bounds = L->getBounds(*SE)) {
+    Value *StartVal = &Bounds->getInitialIVValue();
+    Value *BoundVal = &Bounds->getFinalIVValue();
+    if (L->isLoopInvariant(BoundVal))
+      if (unsigned MaxTC = getMaxTCForRemainderLoop(StartVal, BoundVal))
+        return MaxTC;
+  }
+
   return 0;
 }
 
@@ -2943,6 +3045,21 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
   if (TC != ElementCount::getFixed(MaxTC))
     LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
+
+  // If we derived a small max trip count (e.g. from a remainder-of-stride
+  // pattern) and runtime checks are required, reject vectorization since
+  // the overhead cannot be amortized over so few iterations.
+  if (MaxTC > 0 && MaxTC < TinyTripCountVectorThreshold &&
+      EpilogueLoweringStatus == CM_EpilogueAllowed &&
+      Config.runtimeChecksRequired()) {
+    reportVectorizationFailure(
+        "Max trip count too small for vectorization with runtime checks",
+        "Loop max trip count is too low to amortize vectorization overhead "
+        "with required runtime checks",
+        "TooSmallWithRuntimeChecks", ORE, TheLoop);
+    return FixedScalableVFPair::getNone();
+  }
+
   if (TC.isScalar()) {
     reportVectorizationFailure(
         "Single iteration (non) loop",
diff --git a/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll b/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll
index 8ce0624289cfb..a40348bf0f47a 100644
--- a/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/remainder-max-trip-count.ll
@@ -15,8 +15,6 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @remainder_after_stride8(ptr %src, i64 %n, ptr %dst) {
 ; CHECK-LABEL: @remainder_after_stride8(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoaddr ptr [[SRC:%.*]] to i64
-; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST:%.*]] to i64
 ; CHECK-NEXT:    [[CMP_NOT36:%.*]] = icmp ult i64 [[N:%.*]], 8
 ; CHECK-NEXT:    br i1 [[CMP_NOT36]], label [[FOR_COND12_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_PREHEADER:%.*]]
 ; CHECK:       for.cond1.preheader.preheader:
@@ -37,56 +35,21 @@ define void @remainder_after_stride8(ptr %src, i64 %n, ptr %dst) {
 ; CHECK-NEXT:    [[CMP1339:%.*]] = icmp ult i64 [[I_0_LCSSA]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP1339]], label [[FOR_BODY14_PREHEADER:%.*]], label [[FOR_END21:%.*]]
 ; CHECK:       for.body14.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[N]], [[I_0_LCSSA]]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC2]]
-; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 32
-; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[I_0_LCSSA]], [[N_VEC]]
-; CHECK-NEXT:    br label [[FOR_BODY14:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY14]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP4]], i64 4
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 32768)
-; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i32> [[WIDE_LOAD3]], splat (i32 32768)
-; CHECK-NEXT:    [[TMP8:%.*]] = and <4 x i32> [[TMP6]], splat (i32 -65536)
-; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i32> [[TMP7]], splat (i32 -65536)
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP10]], i64 4
-; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY14]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END21_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ [[I_0_LCSSA]], [[FOR_BODY14_PREHEADER]] ], [ [[I_0_LCSSA]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY15:%.*]]
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[ADD4:%.*]] = or disjoint i64 [[INDVARS_IV]], [[I_037]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[ADD4]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC:%.*]], i64 [[ADD4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD5:%.*]] = add i32 [[TMP13]], 32768
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD5]], -65536
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[ADD4]]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST:%.*]], i64 [[ADD4]]
 ; CHECK-NEXT:    store i32 [[AND]], ptr [[ARRAYIDX8]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_LOOPEXIT]], label [[FOR_BODY3]]
 ; CHECK:       for.body14:
-; CHECK-NEXT:    [[I_140:%.*]] = phi i64 [ [[INC20:%.*]], [[FOR_BODY15]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_140:%.*]] = phi i64 [ [[INC20:%.*]], [[FOR_BODY15]] ], [ [[I_0_LCSSA]], [[FOR_BODY14_PREHEADER]] ]
 ; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[I_140]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
 ; CHECK-NEXT:    [[ADD16:%.*]] = add i32 [[TMP14]], 32768
@@ -95,7 +58,7 @@ define void @remainder_after_stride8(ptr %src, i64 %n, ptr %dst) {
 ; CHECK-NEXT:    store i32 [[AND17]], ptr [[ARRAYIDX18]], align 4
 ; CHECK-NEXT:    [[INC20]] = add nuw i64 [[I_140]], 1
 ; CHECK-NEXT:    [[EXITCOND42_NOT:%.*]] = icmp eq i64 [[INC20]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND42_NOT]], label [[FOR_END21_LOOPEXIT]], label [[FOR_BODY15]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND42_NOT]], label [[FOR_END21_LOOPEXIT:%.*]], label [[FOR_BODY15]]
 ; CHECK:       for.end21.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END21]]
 ; CHECK:       for.end21:



More information about the llvm-commits mailing list