[llvm] [LAA] Avoid unprofitable unit-stride versioning (PR #225331)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 01:46:20 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Boyao Wang (BoyaoWang430)
<details>
<summary>Changes</summary>
LAA uses `Stride >= TripCount` to reject unprofitable unit-stride versioning. The same i32 value `N` determines both the stride and trip count here, but the induction variable is widened to i64. The stride is sign-extended while the trip count uses `zext(N)`, so LAA may still allow versioning even though `N == 1` implies a single iteration. This can leave the vector path restricted to one iteration and keep the general case scalar.
Apply loop guards to MaxBTC and evaluate it under a local `Stride == 1` predicate. Reject versioning when unit stride is impossible on loop entry or the rewritten count is zero.
Fixes https://github.com/llvm/llvm-project/issues/225061.
---
Patch is 61.31 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/225331.diff
6 Files Affected:
- (modified) llvm/lib/Analysis/LoopAccessAnalysis.cpp (+22-29)
- (modified) llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll (+7-5)
- (modified) llvm/test/Transforms/LoopVectorize/pr34681.ll (+503)
- (modified) llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll (+83-23)
- (modified) llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll (+75-43)
- (modified) llvm/test/Transforms/LoopVersioning/single-iteration.ll (+4-23)
``````````diff
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 437d25b3d56ede..32708f25dabe82 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -3235,9 +3235,8 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
return;
}
- // Avoid adding the "Stride == 1" predicate when we know that
- // Stride >= Trip-Count. Such a predicate will effectively optimize a single
- // or zero iteration loop, as Trip-Count <= Stride == 1.
+ // Avoid unit-stride versioning if the predicate cannot hold on loop entry or
+ // would leave at most one iteration.
//
// TODO: We are currently not making a very informed decision on when it is
// beneficial to apply stride versioning. It might make more sense that the
@@ -3248,39 +3247,33 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
// of various possible stride specializations, considering the alternatives
// of using gather/scatters (if available).
- const SCEV *MaxBTC = PSE->getSymbolicMaxBackedgeTakenCount();
-
- // Match the types so we can compare the stride and the MaxBTC.
- // The Stride can be positive/negative, so we sign extend Stride;
- // The backedgeTakenCount is non-negative, so we zero extend MaxBTC.
- const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
- uint64_t StrideTypeSizeBits = DL.getTypeSizeInBits(StrideExpr->getType());
- uint64_t BETypeSizeBits = DL.getTypeSizeInBits(MaxBTC->getType());
- const SCEV *CastedStride = StrideExpr;
- const SCEV *CastedBECount = MaxBTC;
ScalarEvolution *SE = PSE->getSE();
- if (BETypeSizeBits >= StrideTypeSizeBits)
- CastedStride = SE->getNoopOrSignExtend(StrideExpr, MaxBTC->getType());
- else
- CastedBECount = SE->getZeroExtendExpr(MaxBTC, StrideExpr->getType());
- const SCEV *StrideMinusBETaken = SE->getMinusSCEV(CastedStride, CastedBECount);
- // Since TripCount == BackEdgeTakenCount + 1, checking:
- // "Stride >= TripCount" is equivalent to checking:
- // Stride - MaxBTC> 0
- if (SE->isKnownPositive(StrideMinusBETaken)) {
- LLVM_DEBUG(
- dbgs() << "LAA: Stride>=TripCount; No point in versioning as the "
- "Stride==1 predicate will imply that the loop executes "
- "at most once.\n");
- return;
- }
- LLVM_DEBUG(dbgs() << "LAA: Found a strided access that we can version.\n");
+ const SCEV *MaxBTC = PSE->getSymbolicMaxBackedgeTakenCount();
+ if (!LoopGuards)
+ LoopGuards.emplace(ScalarEvolution::LoopGuards::collect(TheLoop, *SE));
+ MaxBTC = SE->applyLoopGuards(MaxBTC, *LoopGuards);
// Strip back off the integer cast, and check that our result is a
// SCEVUnknown as we expect.
const SCEV *StrideBase = StrideExpr;
if (const auto *C = dyn_cast<SCEVIntegralCastExpr>(StrideBase))
StrideBase = C->getOperand();
+
+ // Evaluate the guarded trip count under the unit-stride predicate instead of
+ // comparing the stride and trip count, which may use different integer
+ // extensions. Keep the predicate local: we have not decided to version the
+ // access yet.
+ const SCEV *One = SE->getOne(StrideBase->getType());
+ const SCEVPredicate *StrideIsOne = SE->getEqualPredicate(StrideBase, One);
+ if (SE->isLoopEntryGuardedByCond(TheLoop, ICmpInst::ICMP_NE, StrideBase,
+ One) ||
+ SE->rewriteUsingPredicate(MaxBTC, TheLoop, *StrideIsOne)->isZero()) {
+ LLVM_DEBUG(dbgs() << "LAA: No point in versioning as the unit-stride path "
+ "is unreachable or executes at most once.\n");
+ return;
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Found a strided access that we can version.\n");
assert(SE->isLoopInvariant(StrideBase, TheLoop) &&
"users of the map rely on the stride being loop invariant");
SymbolicStrides[Ptr] = cast<SCEVUnknown>(StrideBase);
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
index f4aff75582c027..51ba5e42bcf0d1 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
@@ -315,22 +315,24 @@ exit:
ret void
}
+; There is no benefit in versioning a single-iteration loop for unit stride.
define double @single_iteration_unknown_stride(i32 %x, ptr %y, i1 %cond) {
; CHECK-LABEL: 'single_iteration_unknown_stride'
; CHECK-NEXT: loop.body:
-; CHECK-NEXT: Memory dependences are safe
+; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT: Unsafe indirect dependence.
; CHECK-NEXT: Dependences:
+; CHECK-NEXT: IndirectUnsafe:
+; CHECK-NEXT: %load11 = load double, ptr %gep10, align 8 ->
+; CHECK-NEXT: store double %load11, ptr %y, align 8
+; CHECK-EMPTY:
; CHECK-NEXT: Run-time memory checks:
; CHECK-NEXT: Grouped accesses:
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: Equal predicate: %x == 1
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
-; CHECK-NEXT: [PSE] %gep10 = getelementptr double, ptr %gep8, i64 %mul:
-; CHECK-NEXT: {(8 + %y),+,(8 * (sext i32 %x to i64))<nsw>}<nw><%loop.body>
-; CHECK-NEXT: --> {(8 + %y),+,8}<nw><%loop.body>
;
entry:
br i1 %cond, label %noloop.exit, label %loop.ph
diff --git a/llvm/test/Transforms/LoopVectorize/pr34681.ll b/llvm/test/Transforms/LoopVectorize/pr34681.ll
index f3427f112bc5e3..fb33b6f1b96ef0 100644
--- a/llvm/test/Transforms/LoopVectorize/pr34681.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr34681.ll
@@ -199,3 +199,506 @@ for.end:
%tmp.0.lcssa = phi i32 [ 0, %entry ], [ %add4.lcssa, %for.end.loopexit ]
ret i32 %tmp.0.lcssa
}
+
+; The i64 trip count is the zero-extended i32 stride.
+; Unit stride leaves only one iteration.
+define i32 @same_stride_and_trip_count(i32 %N, ptr %B, i32 %j) {
+; CHECK-LABEL: define i32 @same_stride_and_trip_count(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[B:%.*]], i32 [[J:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ZERO:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[ZERO]], [[EXIT:label %.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[N64:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N64]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N64]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N64]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[J]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX3:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP15]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP17:%.*]] = zext <4 x i32> [[TMP16]] to <4 x i64>
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP17]], i64 1
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP17]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP17]], i64 3
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = load i16, ptr [[TMP19]], align 2
+; CHECK-NEXT: [[TMP27:%.*]] = load i16, ptr [[TMP21]], align 2
+; CHECK-NEXT: [[TMP28:%.*]] = load i16, ptr [[TMP23]], align 2
+; CHECK-NEXT: [[TMP29:%.*]] = load i16, ptr [[TMP25]], align 2
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i16> poison, i16 [[TMP26]], i64 0
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i16> [[TMP30]], i16 [[TMP27]], i64 1
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> [[TMP31]], i16 [[TMP28]], i64 2
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i16> [[TMP32]], i16 [[TMP29]], i64 3
+; CHECK-NEXT: [[TMP34:%.*]] = sext <4 x i16> [[TMP33]] to <4 x i32>
+; CHECK-NEXT: [[TMP35]] = add <4 x i32> [[VEC_PHI]], [[TMP34]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX3]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP35]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N64]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %zero = icmp eq i32 %N, 0
+ br i1 %zero, label %exit, label %preheader
+
+preheader:
+ %N64 = zext i32 %N to i64
+ br label %loop
+
+loop:
+ %k = phi i64 [ 0, %preheader ], [ %next, %loop ]
+ %acc = phi i32 [ 0, %preheader ], [ %sum, %loop ]
+ %k32 = trunc i64 %k to i32
+ %offset = mul i32 %k32, %N
+ %index = add i32 %offset, %j
+ %index64 = zext i32 %index to i64
+ %ptr = getelementptr inbounds i16, ptr %B, i64 %index64
+ %v = load i16, ptr %ptr, align 2
+ %v32 = sext i16 %v to i32
+ %sum = add i32 %acc, %v32
+ %next = add nuw i64 %k, 1
+ %done = icmp eq i64 %next, %N64
+ br i1 %done, label %loopexit, label %loop
+
+loopexit:
+ %result = phi i32 [ %sum, %loop ]
+ br label %exit
+
+exit:
+ %ret = phi i32 [ 0, %entry ], [ %result, %loopexit ]
+ ret i32 %ret
+}
+
+; The stride range excludes 1; the trip count is always four.
+define i32 @known_nonunit_stride(i32 range(i32 4, 128) %stride, ptr %B) {
+; CHECK-LABEL: define i32 @known_nonunit_stride(
+; CHECK-SAME: i32 range(i32 4, 128) [[STRIDE:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STRIDE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[TMP0]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP7]], align 2
+; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP9]], align 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i16> poison, i16 [[TMP10]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> [[TMP14]], i16 [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i64 3
+; CHECK-NEXT: [[TMP18:%.*]] = sext <4 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i32> zeroinitializer, [[TMP18]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP19]])
+; CHECK-NEXT: br label %[[LOOPEXIT:.*]]
+; CHECK: [[LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP20]]
+;
+entry:
+ br label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %k = phi i64 [ 0, %preheader ], [ %next, %loop ]
+ %acc = phi i32 [ 0, %preheader ], [ %sum, %loop ]
+ %k32 = trunc i64 %k to i32
+ %index = mul i32 %k32, %stride
+ %index64 = zext i32 %index to i64
+ %ptr = getelementptr inbounds i16, ptr %B, i64 %index64
+ %v = load i16, ptr %ptr, align 2
+ %v32 = sext i16 %v to i32
+ %sum = add i32 %acc, %v32
+ %next = add nuw i64 %k, 1
+ %done = icmp eq i64 %next, 4
+ br i1 %done, label %loopexit, label %loop
+
+loopexit:
+ %result = phi i32 [ %sum, %loop ]
+ br label %exit
+
+exit:
+ ret i32 %result
+}
+
+; Negative strides cannot be specialized for unit stride.
+define i32 @negative_stride(i32 range(i32 -128, 0) %stride, ptr %B) {
+; CHECK-LABEL: define i32 @negative_stride(
+; CHECK-SAME: i32 range(i32 -128, 0) [[STRIDE:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STRIDE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i32> [[TMP0]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP7]], align 2
+; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP9]], align 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i16> poison, i16 [[TMP10]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> [[TMP14]], i16 [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i64 3
+; CHECK-NEXT: [[TMP18:%.*]] = sext <4 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i32> zeroinitializer, [[TMP18]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP19]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP20]]
+;
+entry:
+ br label %loop
+
+loop:
+ %k = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %sum, %loop ]
+ %k32 = trunc i64 %k to i32
+ %index = mul i32 %k32, %stride
+ %index64 = sext i32 %index to i64
+ %ptr = getelementptr inbounds i16, ptr %B, i64 %index64
+ %v = load i16, ptr %ptr, align 2
+ %v32 = sext i16 %v to i32
+ %sum = add i32 %acc, %v32
+ %next = add nuw i64 %k, 1
+ %done = icmp eq i64 %next, 4
+ br i1 %done, label %exit, label %loop
+
+exit:
+ %result = phi i32 [ %sum, %loop ]
+ ret i32 %result
+}
+
+; The entry guard N >= 5 rules out unit stride.
+define i32 @stride_larger_than_trip_count(i64 %N, ptr %B) {
+; CHECK-LABEL: define i32 @stride_larger_than_trip_count(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SKIP:%.*]] = icmp ult i64 [[N]], 5
+; CHECK-NEXT: br i1 [[SKIP]], [[EXIT:label %.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[TRIP_COUNT:%.*]] = sub nuw i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TRIP_COUNT]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TRIP_COUNT]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[TMP6]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP8]], align 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i16> poison, i16 [[TMP9]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> [[TMP14]], i16 [[TMP10]], i64 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP11]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[TMP12]], i64 3
+; CHECK-NEXT: [[TMP17:%.*]] = sext <4 x i16> [[TMP16]] to <4 x i32>
+; CHECK-NEXT: [[TMP18]] = add <4 x i32> [[VEC_PHI]], [[TMP17]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHEC...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/225331
More information about the llvm-commits
mailing list