[llvm] [LAA] Enable diff checks for non-unit constant stride (PR #188826)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 07:53:48 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/188826
>From 3f13bc7427decd24d5fdba75ec976352770f7d76 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 25 Mar 2026 13:50:17 -0700
Subject: [PATCH 01/14] Add a test
---
.../runtime-checks-difference.ll | 32 +++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
index 16afd3044d180..00f566e0def59 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
@@ -619,5 +619,37 @@ exit:
ret void
}
+define void @constant_strided(ptr %p, ptr %p.out) {
+; CHECK-LABEL: define void @constant_strided(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[P_OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P_OUT]], i64 2040
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 2040
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P_OUT]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], [[SCALAR_PH:label %.*]], [[VECTOR_PH:label %.*]]
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+ %idx = mul nuw nsw i64 %iv, 2
+ %gep.ld = getelementptr inbounds i64, ptr %p, i64 %idx
+ %gep.st = getelementptr inbounds i64, ptr %p.out, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 4
+ %add = add i64 %ld, 1
+ store i64 %add, ptr %gep.st, align 4
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.mustprogress"}
>From 3e6654dd053b1937ef00946c377515961b43f3ad Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 26 Mar 2026 11:48:23 -0700
Subject: [PATCH 02/14] [LAA] Enable diff checks for non-unit constant stride
This can be extended to non-constant loop-invariant strides but that is
left to a separate change.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 5 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 12 +--
llvm/lib/Transforms/Utils/LoopUtils.cpp | 19 +++--
.../AArch64/interleave-with-runtime-checks.ll | 74 +++++--------------
.../LoopVectorize/RISCV/dead-ops-cost.ll | 70 +++++++-----------
.../runtime-checks-difference.ll | 9 +--
6 files changed, 72 insertions(+), 117 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..684550c302b48 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -494,11 +494,14 @@ struct PointerDiffInfo {
const SCEV *SrcStart;
const SCEV *SinkStart;
unsigned AccessSize;
+ unsigned AbsCommonStrideInBytes;
bool NeedsFreeze;
PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart,
- unsigned AccessSize, bool NeedsFreeze)
+ unsigned AccessSize, unsigned AbsCommonStrideInBytes,
+ bool NeedsFreeze)
: SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize),
+ AbsCommonStrideInBytes(AbsCommonStrideInBytes),
NeedsFreeze(NeedsFreeze) {}
};
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 2b9efd22131c6..65f3b29c07bd9 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -488,10 +488,12 @@ bool RuntimePointerChecking::tryToCreateDiffCheck(
unsigned AllocSize =
std::max(DL.getTypeAllocSize(SrcTy), DL.getTypeAllocSize(DstTy));
- // Only matching constant steps matching the AllocSize are supported at the
- // moment. This simplifies the difference computation. Can be extended in the
- // future.
- if (Step->getAPInt().abs() != AllocSize)
+ std::optional<uint64_t> AbsStep = Step->getAPInt().abs().tryZExtValue();
+ if (!AbsStep)
+ return false;
+
+ // Step must be at least AllocSize.
+ if (*AbsStep < AllocSize)
return false;
// When counting down, the dependence distance needs to be swapped.
@@ -529,7 +531,7 @@ bool RuntimePointerChecking::tryToCreateDiffCheck(
LLVM_DEBUG(dbgs() << "LAA: Creating diff runtime check for:\n"
<< "SrcStart: " << *SrcStartInt << '\n'
<< "SinkStartInt: " << *SinkStartInt << '\n');
- DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
+ DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize, *AbsStep,
Src->NeedsFreeze || Sink->NeedsFreeze);
return true;
}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 03b693974e5fe..8864d26c46f0a 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2271,12 +2271,17 @@ Value *llvm::addDiffRuntimeChecks(
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
- for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
+ for (const auto &[SrcStart, SinkStart, AccessSize, AbsCommonStrideInBytes,
+ NeedsFreeze] : Checks) {
Type *Ty = SinkStart->getType();
- // Compute VF * IC * AccessSize.
- auto *VFTimesICTimesSize =
+ // Compute the range of the accessed memory during one vector loop
+ // iteration. This is equal to VF*IC*Stride-(Stride-AccessSize).
+ auto *VectorIterAccessRange =
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, IC * AccessSize));
+ ConstantInt::get(Ty, IC * AbsCommonStrideInBytes));
+ VectorIterAccessRange = ChkBuilder.CreateSub(
+ VectorIterAccessRange,
+ ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize));
const SCEV *SinkStartRewritten = Rewriter.visit(SinkStart);
const SCEV *SrcStartRewritten = Rewriter.visit(SrcStart);
Value *Diff = Expander.expandCodeFor(
@@ -2284,13 +2289,13 @@ Value *llvm::addDiffRuntimeChecks(
// Check if the same compare has already been created earlier. In that case,
// there is no need to check it again.
- Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
+ Value *IsConflict = SeenCompares.lookup({Diff, VectorIterAccessRange});
if (IsConflict)
continue;
IsConflict =
- ChkBuilder.CreateICmpULT(Diff, VFTimesICTimesSize, "diff.check");
- SeenCompares.insert({{Diff, VFTimesICTimesSize}, IsConflict});
+ ChkBuilder.CreateICmpULT(Diff, VectorIterAccessRange, "diff.check");
+ SeenCompares.insert({{Diff, VectorIterAccessRange}, IsConflict});
if (NeedsFreeze)
IsConflict =
ChkBuilder.CreateFreeze(IsConflict, IsConflict->getName() + ".fr");
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
index 9f21ec1806703..491fc72908cb8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
@@ -1,69 +1,33 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
; RUN: opt -p loop-vectorize -S %s | FileCheck %s
target triple = "arm64-apple-macosx15.0.0"
+; TODO: A[2i] and A[2i+1] are independent and are put into GRP0/GRP1. However,
+; A[offset+...] creates another GRP2 and that results in
+; GRP0<->GRP1/GRP0<->GRP2/GRP1<->GRP2 run-time checks. The first one always
+; fails but it's unnecessary and shouldn't be performed.
define void @interleave_groups_separated_by_offset(ptr %A, i64 %offset) {
; CHECK-LABEL: define void @interleave_groups_separated_by_offset(
; CHECK-SAME: ptr [[A:%.*]], i64 [[OFFSET:%.*]]) {
-; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[VEC_EPILOG_SCALAR_PH1:.*]]:
; CHECK-NEXT: [[A_OFFSET:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET]]
-; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 1999
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 1
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 2000
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET]], 1999
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
-; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[A_OFFSET]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
-; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
-; CHECK-NEXT: [[BOUND07:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP3]]
-; CHECK-NEXT: [[BOUND18:%.*]] = icmp ult ptr [[A_OFFSET]], [[SCEVGEP2]]
-; CHECK-NEXT: [[FOUND_CONFLICT9:%.*]] = and i1 [[BOUND07]], [[BOUND18]]
-; CHECK-NEXT: [[CONFLICT_RDX10:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT9]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX10]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A]], i64 1984
-; CHECK-NEXT: [[IND_END17:%.*]] = getelementptr i8, ptr [[A_OFFSET]], i64 1984
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 32
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP11]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 992, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 1984
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A_OFFSET]], i64 1984
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX12]], 1
-; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP13]], align 1
-; CHECK-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX12]], 8
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT14]], 992
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[A]], %[[VEC_EPILOG_SCALAR_PH1]] ], [ [[PTR_IV_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[PTR_IV_OFFSET:%.*]] = phi ptr [ [[A_OFFSET]], %[[VEC_EPILOG_SCALAR_PH1]] ], [ [[PTR_IV_OFFSET_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[PTR_IV_1:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 1
+; CHECK-NEXT: store i8 0, ptr [[PTR_IV]], align 1
+; CHECK-NEXT: [[PTR_IV_OFFSET_NEXT]] = getelementptr i8, ptr [[PTR_IV_OFFSET]], i64 2
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[PTR_IV_OFFSET]], align 1
+; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 2
+; CHECK-NEXT: store i8 0, ptr [[PTR_IV_1]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 1000
+; CHECK-NEXT: br i1 [[EC]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: ret void
;
entry:
%A.offset = getelementptr i8, ptr %A, i64 %offset
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 791130f004e0e..f3d53c76fa2ed 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
; RUN: opt -p loop-vectorize -mtriple riscv64-linux-gnu -mattr=+v,+f -S %s | FileCheck %s
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
@@ -70,13 +70,16 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-LABEL: define i8 @dead_live_out_due_to_scalar_epilogue_required(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 1005
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 1005
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP12]], 3
+; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP13]], [[TMP14]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -91,13 +94,13 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i32> [[VEC_IND]] to <vscale x 16 x i64>
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 16 x i64> [[TMP9]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 16 x i64> [[TMP9]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1
@@ -114,7 +117,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: store i8 0, ptr [[GEP_DST]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 4
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV]], 1001
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[R:%.*]] = phi i8 [ [[L]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i8 [[R]]
@@ -178,48 +181,48 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
; CHECK: [[PRED_STORE_CONTINUE8]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
; CHECK: [[PRED_STORE_CONTINUE10]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
; CHECK: [[PRED_STORE_CONTINUE12]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
; CHECK: [[PRED_STORE_CONTINUE14]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
; CHECK: [[PRED_STORE_IF15]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_IF17]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META5]], !noalias [[META8]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META8]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -241,7 +244,7 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[EC:%.*]] = icmp slt i32 [[IV]], [[SUB]]
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: br label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: br label %[[RETURN:.*]]
; CHECK: [[RETURN]]:
@@ -304,7 +307,7 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -369,7 +372,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -398,24 +401,3 @@ exit:
attributes #0 = { "target-features"="+64bit,+v" }
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
-; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], !"LVerDomain"}
-; CHECK: [[META13]] = !{[[META14:![0-9]+]]}
-; CHECK: [[META14]] = distinct !{[[META14]], [[META12]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
index 00f566e0def59..e2695ff2fe6b7 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
@@ -623,13 +623,12 @@ define void @constant_strided(ptr %p, ptr %p.out) {
; CHECK-LABEL: define void @constant_strided(
; CHECK-SAME: ptr [[P:%.*]], ptr [[P_OUT:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P2:%.*]] = ptrtoaddr ptr [[P]] to i64
+; CHECK-NEXT: [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P_OUT]], i64 2040
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 2040
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P_OUT]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[P_OUT1]], [[P2]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP0]], 56
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], [[SCALAR_PH:label %.*]], [[VECTOR_PH:label %.*]]
;
entry:
>From 6726ce2858d30a730c7898f6256148aff0c5ebf0 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 15 Apr 2026 11:23:44 -0700
Subject: [PATCH 03/14] unsigned->uint64_t for AbsCommonStrideInBytes
---
llvm/include/llvm/Analysis/LoopAccessAnalysis.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 684550c302b48..938c007d84331 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -494,11 +494,11 @@ struct PointerDiffInfo {
const SCEV *SrcStart;
const SCEV *SinkStart;
unsigned AccessSize;
- unsigned AbsCommonStrideInBytes;
+ uint64_t AbsCommonStrideInBytes;
bool NeedsFreeze;
PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart,
- unsigned AccessSize, unsigned AbsCommonStrideInBytes,
+ unsigned AccessSize, uint64_t AbsCommonStrideInBytes,
bool NeedsFreeze)
: SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize),
AbsCommonStrideInBytes(AbsCommonStrideInBytes),
>From 83df28ae73818b7cdf7b02f413f229961cead35d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 29 Jun 2026 11:39:07 -0700
Subject: [PATCH 04/14] Rename VectorIterAccessRange -> VectorIterAccessSpan
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 4206e72ed5b5d..e5145c6d01dad 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2287,13 +2287,14 @@ Value *llvm::addDiffRuntimeChecks(
for (const auto &[SrcStart, SinkStart, AccessSize, AbsCommonStrideInBytes,
NeedsFreeze] : Checks) {
Type *Ty = SinkStart->getType();
- // Compute the range of the accessed memory during one vector loop
- // iteration. This is equal to VF*IC*Stride-(Stride-AccessSize).
- auto *VectorIterAccessRange =
+ // Compute the distance between first/last bytes of the accessed memory
+ // during one vector loop iteration. This is equal to
+ // VF*IC*Stride-(Stride-AccessSize).
+ auto *VectorIterAccessSpan =
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
ConstantInt::get(Ty, IC * AbsCommonStrideInBytes));
- VectorIterAccessRange = ChkBuilder.CreateSub(
- VectorIterAccessRange,
+ VectorIterAccessSpan = ChkBuilder.CreateSub(
+ VectorIterAccessSpan,
ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize));
const SCEV *SinkStartRewritten = Rewriter.visit(SinkStart);
const SCEV *SrcStartRewritten = Rewriter.visit(SrcStart);
@@ -2302,13 +2303,13 @@ Value *llvm::addDiffRuntimeChecks(
// Check if the same compare has already been created earlier. In that case,
// there is no need to check it again.
- Value *IsConflict = SeenCompares.lookup({Diff, VectorIterAccessRange});
+ Value *IsConflict = SeenCompares.lookup({Diff, VectorIterAccessSpan});
if (IsConflict)
continue;
IsConflict =
- ChkBuilder.CreateICmpULT(Diff, VectorIterAccessRange, "diff.check");
- SeenCompares.insert({{Diff, VectorIterAccessRange}, IsConflict});
+ ChkBuilder.CreateICmpULT(Diff, VectorIterAccessSpan, "diff.check");
+ SeenCompares.insert({{Diff, VectorIterAccessSpan}, IsConflict});
if (NeedsFreeze)
IsConflict =
ChkBuilder.CreateFreeze(IsConflict, IsConflict->getName() + ".fr");
>From 4bc5abef37a0fd6cd7cfcc202b2ffed0c3767d71 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 29 Jun 2026 13:29:42 -0700
Subject: [PATCH 05/14] Add a test for 2^64 IV step
---
.../runtime-checks-large_step.ll | 47 +++++++++++++++++++
1 file changed, 47 insertions(+)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-large_step.ll
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-large_step.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-large_step.ll
new file mode 100644
index 0000000000000..1889f0de35a73
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-large_step.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -disable-output -passes='print<access-info>' %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:128:128:128"
+
+define void @test_large_step(ptr %A, ptr %B, i128 %n) {
+; CHECK-LABEL: 'test_large_step'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.B = getelementptr inbounds i32, ptr %B, i128 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %B High: (4 + (73786976294838206464 * ((-18446744073709551616 + %n) /u 18446744073709551616)) + %B))
+; CHECK-NEXT: Member: {%B,+,73786976294838206464}<%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %A High: (4 + (73786976294838206464 * ((-18446744073709551616 + %n) /u 18446744073709551616)) + %A))
+; CHECK-NEXT: Member: {%A,+,73786976294838206464}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-NEXT: Equal predicate: (zext i64 (trunc i128 %n to i64) to i128) == 0
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+ %l = load i32, ptr %gep.A, align 4
+ %add = add i32 %l, 1
+ %gep.B = getelementptr inbounds i32, ptr %B, i128 %iv
+ store i32 %add, ptr %gep.B, align 4
+ %iv.next = add i128 %iv, u0x10000000000000000 ; 2^64
+ %ec = icmp eq i128 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From fa7c71439e3c80b8c3a813b662417fead807674d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 29 Jun 2026 15:02:38 -0700
Subject: [PATCH 06/14] Address potential `IC * AbsCommonStrideInBytes`
truncation
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 12 ++++-
...time-checks-diff-wrap-i32-address-space.ll | 47 +++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index e5145c6d01dad..1ca39cb2fac95 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2290,16 +2290,24 @@ Value *llvm::addDiffRuntimeChecks(
// Compute the distance between first/last bytes of the accessed memory
// during one vector loop iteration. This is equal to
// VF*IC*Stride-(Stride-AccessSize).
+ uint64_t ICTimesStride = IC * AbsCommonStrideInBytes;
+ if (!isUIntN(Ty->getScalarSizeInBits(), ICTimesStride)) {
+ // This is probably UB in the original IR, but let's be conservative:
+ Ty = Ty->getWithNewBitWidth(Ty->getScalarSizeInBits() * 2);
+ assert(isUIntN(Ty->getScalarSizeInBits(), ICTimesStride));
+ }
auto *VectorIterAccessSpan =
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, IC * AbsCommonStrideInBytes));
+ ConstantInt::get(Ty, ICTimesStride));
VectorIterAccessSpan = ChkBuilder.CreateSub(
VectorIterAccessSpan,
ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize));
const SCEV *SinkStartRewritten = Rewriter.visit(SinkStart);
const SCEV *SrcStartRewritten = Rewriter.visit(SrcStart);
Value *Diff = Expander.expandCodeFor(
- SE.getMinusSCEV(SinkStartRewritten, SrcStartRewritten), Ty, Loc);
+ SE.getNoopOrSignExtend(
+ SE.getMinusSCEV(SinkStartRewritten, SrcStartRewritten), Ty),
+ Ty, Loc);
// Check if the same compare has already been created earlier. In that case,
// there is no need to check it again.
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
new file mode 100644
index 0000000000000..06f77636ab055
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph\:" --version 5
+; RUN: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s
+
+; With IC=2, stride=2147483648 (2^31), we get IC * stride = 4294967296 (2^32)
+; which wraps to 0 when truncated to i32.
+
+target datalayout = "e-p:64:64-p1:32:32-i64:64-n32:64"
+
+define void @test_wrap_i32_address_space(ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %n) {
+; CHECK-LABEL: define void @test_wrap_i32_address_space(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr addrspace(1) [[B]] to i32
+; CHECK-NEXT: [[A1:%.*]] = ptrtoaddr ptr addrspace(1) [[A]] to i32
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A1]], [[B2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15032385537
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ ; Access with very large stride: 2^31 bytes = 2147483648
+ %iv.ext = zext i32 %iv to i64
+ %offset = mul i64 %iv.ext, 2147483648
+ ; I believe this produces a poison that becomes UB when dereferenced, but even
+ ; for UB input we can't `assert` and have produce something. Ensure that
+ ; DIFF_CHECK above (even if constant-foldable to false) is done in i64 that
+ ; can represent IC x ConstantStride.
+ %gep.a = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %offset
+ %l = load i8, ptr addrspace(1) %gep.a
+ %add = add i8 %l, 1
+ %gep.b = getelementptr inbounds i8, ptr addrspace(1) %b, i64 %offset
+ store i8 %add, ptr addrspace(1) %gep.b
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 94ba1d74b676a9d1b98dcedf348480443303e41e Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Jul 2026 12:04:39 -0700
Subject: [PATCH 07/14] Update tests after merging in `origin/main`
---
.../Transforms/LoopVectorize/RISCV/dead-ops-cost.ll | 12 +++++++-----
.../runtime-checks-diff-wrap-i32-address-space.ll | 3 ++-
.../LoopVectorize/runtime-checks-difference.ll | 3 ++-
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 38840d16158da..59911fbfe118e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -78,8 +78,10 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP5]], 4
; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP12]], 3
+; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[DST1]], [[SRC2]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP13]], 1
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP6]], [[TMP16]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -94,10 +96,10 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i32> [[VEC_IND]] to <vscale x 16 x i64>
-; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[INDEX]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP17]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP18]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP19]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 16 x i64> [[TMP9]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP2]], [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
index 06f77636ab055..1ef0c9f5fb8dc 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
@@ -17,7 +17,8 @@ define void @test_wrap_i32_address_space(ptr addrspace(1) %a, ptr addrspace(1) %
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A1]], [[B2]]
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15032385537
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 15032385536
; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
index 7fc7cb6fe6ab4..b72c9444bfd3c 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
@@ -630,7 +630,8 @@ define void @constant_strided(ptr %p, ptr %p.out) {
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[P_OUT1]], [[P2]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP0]], 56
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP1]], 55
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], [[SCALAR_PH:label %.*]], [[VECTOR_PH:label %.*]]
;
entry:
>From 88a6fc0227e51f0c662c3f5284a9c0627bb5b41d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Jul 2026 13:04:38 -0700
Subject: [PATCH 08/14] Fold "-1" in ThresholdMinusOne into another sub(_,
constant)
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 15 ++++++---------
.../LoopVectorize/RISCV/dead-ops-cost.ll | 3 +--
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 4714e69b098f6..04f63a4a185bf 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2294,18 +2294,14 @@ Value *llvm::addDiffRuntimeChecks(
assert(isUIntN(Ty->getScalarSizeInBits(), ICTimesStride));
}
- Value *One = ConstantInt::get(Ty, 1);
Value *&ThresholdMinusOne =
ThresholdCache[{Ty, IC, AbsCommonStrideInBytes}];
- if (!ThresholdMinusOne) {
- auto *VectorIterAccessSpan =
+ if (!ThresholdMinusOne)
+ ThresholdMinusOne = ChkBuilder.CreateSub(
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, ICTimesStride));
- VectorIterAccessSpan = ChkBuilder.CreateSub(
- VectorIterAccessSpan,
- ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize));
- ThresholdMinusOne = ChkBuilder.CreateSub(VectorIterAccessSpan, One);
- }
+ ConstantInt::get(Ty, ICTimesStride)),
+ ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize + 1));
+
Value *Diff = Expander.expandCodeFor(
SE.getNoopOrSignExtend(SE.getMinusSCEV(SinkStart, SrcStart), Ty), Ty,
Loc);
@@ -2319,6 +2315,7 @@ Value *llvm::addDiffRuntimeChecks(
// Use (Diff - 1) <u (VectorIterAccessSpanMinusOne - 1), equivalent to
// 0 < Diff <u VectorIterAccessSpanMinusOne, to exclude Diff == 0 (equal
// pointers are safe).
+ Value *One = ConstantInt::get(Ty, 1);
IsConflict = ChkBuilder.CreateICmpULT(ChkBuilder.CreateSub(Diff, One),
ThresholdMinusOne, "diff.check");
SeenCompares.insert({{Diff, ThresholdMinusOne}, IsConflict});
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 59911fbfe118e..5e6112096f0ce 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -77,8 +77,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP5]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP12]], 3
-; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP12]], 4
; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[DST1]], [[SRC2]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP13]], 1
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP6]], [[TMP16]]
>From 43cf47c7932deec06619a82ad9d69226cba81798 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 3 Aug 2026 15:28:13 -0700
Subject: [PATCH 09/14] Don't enable partial alias masking if there is strided
diff check required
---
.../Transforms/Vectorize/LoopVectorize.cpp | 12 ++
.../Transforms/Vectorize/VPlanLowering.cpp | 2 +
.../alias-mask-negative-tests.ll | 196 ++++++++++++++++++
3 files changed, 210 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de385ae0a4511..408b1c0acc64e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1183,6 +1183,18 @@ class LoopVectorizationCostModel {
if (!DiffChecks || DiffChecks->empty())
return;
+ // From LangRef for @llvm.loop.dependence.[war|raw].mask:
+ // > %elementSize is the size of the accessed elements in bytes. The
+ // > %intrinsic %returns poison if the distance between %addrA and %addrB
+ // > %is not a %multiple of %elementsize.
+ // There is no reason to expect the distance is a multiple of stride, so we
+ // shouldn't enable partial alias masking if there is a strided access
+ // pattern requiring run-time checks.
+ if (any_of(*DiffChecks, [](PointerDiffInfo Check) {
+ return Check.AccessSize != Check.AbsCommonStrideInBytes;
+ }))
+ return;
+
[[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
return any_of(CB->args(), [](Value const *Arg) {
return Arg->getType()->isPointerTy();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 72b6503157d68..63edb765337b3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -1005,6 +1005,8 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
VPValue *AliasMask = nullptr;
for (const PointerDiffInfo &Check : DiffChecks) {
+ assert(Check.AccessSize == Check.AbsCommonStrideInBytes &&
+ "Don't know how to handle that!");
VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SrcStart);
VPValue *Sink =
vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SinkStart);
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
index 629ec8bfc792b..c6a126b627b6a 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
@@ -135,3 +135,199 @@ for.body:
exit:
ret void
}
+
+; From LangRef:
+; > %elementSize is the size of the accessed elements in bytes. The intrinsic
+; > %returns poison if the distance between %addrA and %addrB is not a
+; > %multiple of %elementsize.
+; There is no reason to expect the distance is a multiple of 7, so we shouldn't
+; even attempt to use @llvm.loop.dependence.war.mask to vectorize this.
+define void @alias_mask_stride_7(ptr %src, ptr %dst, i64 %n) {
+; CHECK-LABEL: define void @alias_mask_stride_7(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 87
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 7)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT: store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT: store i32 [[TMP22]], ptr [[TMP23]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+
+ %iv.x.stride = mul nuw nsw i64 %iv, 7
+ %src.gep = getelementptr inbounds i32, ptr %src, i64 %iv.x.stride
+ %val = load i32, ptr %src.gep, align 4
+ %dst.gep = getelementptr inbounds i32, ptr %dst, i64 %iv.x.stride
+ store i32 %val, ptr %dst.gep, align 4
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Similar to above, but with a more "reasonable" stride. Still, there is no
+; guarantee distance (in i32 elements) is divisible by two, so we aren't
+; vectorizing this with @llvm.loop.dependence.war.mask. The test is mostly to
+; prevent accidental unguarded vectorization using it.
+define void @alias_mask_stride_2(ptr %src, ptr %dst, i64 %n) {
+; CHECK-LABEL: define void @alias_mask_stride_2(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 27
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT: store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT: store i32 [[TMP22]], ptr [[TMP23]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+
+ %iv.x.stride = mul nuw nsw i64 %iv, 2
+ %src.gep = getelementptr inbounds i32, ptr %src, i64 %iv.x.stride
+ %val = load i32, ptr %src.gep, align 4
+ %dst.gep = getelementptr inbounds i32, ptr %dst, i64 %iv.x.stride
+ store i32 %val, ptr %dst.gep, align 4
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %for.body
+
+exit:
+ ret void
+}
>From 20376c12d636ebe0a1ef7a71e2a0df6146dd0ba7 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 3 Aug 2026 16:07:37 -0700
Subject: [PATCH 10/14] AccessSize must be part of the key
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 5 +-
.../runtime-checks-difference-scalable.ll | 128 ++++++++++++++++++
2 files changed, 131 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-checks-difference-scalable.ll
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index cb8125a0cfec7..2dd7808dd8da3 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2295,7 +2295,8 @@ Value *llvm::addDiffRuntimeChecks(
// Cache of (VF*IC*Stride-(Stride-AccessSize)) - 1, shared across checks with
// matching type/IC/Stride to avoid emitting duplicate runtime computations.
DenseMap<
- std::tuple<Type *, unsigned /*IC*/, unsigned /*AbsCommonStrideInBytes*/>,
+ std::tuple<Type *, unsigned /*IC*/, unsigned /*AbsCommonStrideInBytes*/,
+ unsigned /*AccessSize*/>,
Value *>
ThresholdCache;
for (const auto &[SrcStart, SinkStart, AccessSize, AbsCommonStrideInBytes,
@@ -2314,7 +2315,7 @@ Value *llvm::addDiffRuntimeChecks(
}
Value *&ThresholdMinusOne =
- ThresholdCache[{Ty, IC, AbsCommonStrideInBytes}];
+ ThresholdCache[{Ty, IC, AbsCommonStrideInBytes, AccessSize}];
if (!ThresholdMinusOne)
ThresholdMinusOne = ChkBuilder.CreateSub(
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-scalable.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-scalable.ll
new file mode 100644
index 0000000000000..322ecfca69f53
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-scalable.ll
@@ -0,0 +1,128 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph" --version 5
+; RUN: opt %s -passes=loop-vectorize -hoist-runtime-checks=false -scalable-vectorization=on -force-target-supports-scalable-vectors -force-target-supports-gather-scatter-ops -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Make sure that %p0*/%p1* checks re-use as much arithmetic as possible.
+;
+;Alias scope metadata is used to reduce the number of generating checks by
+; declaring %p0* not aliasing %p1* and vice versa.
+define void @constant_strided_two_checks(ptr %p0, ptr %p0.out, ptr %p1, ptr %p1.out, i64 %n) {
+; CHECK-LABEL: define void @constant_strided_two_checks(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P0_OUT:%.*]], ptr [[P1:%.*]], ptr [[P1_OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P15:%.*]] = ptrtoaddr ptr [[P1]] to i64
+; CHECK-NEXT: [[P1_OUT1:%.*]] = ptrtoaddr ptr [[P1_OUT]] to i64
+; CHECK-NEXT: [[P03:%.*]] = ptrtoaddr ptr [[P0]] to i64
+; CHECK-NEXT: [[P0_OUT2:%.*]] = ptrtoaddr ptr [[P0_OUT]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 9
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[P0_OUT2]], [[P03]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT: [[DIFF_CHECK8:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[P1_OUT1]], [[P15]]
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK8]], [[DIFF_CHECK10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], [[SCALAR_PH]], [[VECTOR_PH:label %.*]]
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+ %idx = mul nuw nsw i64 %iv, 2
+
+ %gep0.ld = getelementptr inbounds i64, ptr %p0, i64 %idx
+ %gep0.st = getelementptr inbounds i64, ptr %p0.out, i64 %idx
+ %ld0 = load i64, ptr %gep0.ld, align 4, !alias.scope !3, !noalias !4
+ %add0 = add i64 %ld0, 1
+ store i64 %add0, ptr %gep0.st, align 4, !alias.scope !3, !noalias !4
+
+ %gep1.ld = getelementptr inbounds i64, ptr %p1, i64 %idx
+ %gep1.st = getelementptr inbounds i64, ptr %p1.out, i64 %idx
+ %ld1 = load i64, ptr %gep1.ld, align 4, !alias.scope !4, !noalias !3
+ %add1 = add i64 %ld1, 1
+ store i64 %add1, ptr %gep1.st, align 4, !alias.scope !4, !noalias !3
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above, but one access group is using different access size while
+; keeping the strides the same, so that that "threshold" computation cannot be
+; re-used.
+define void @constant_strided_two_checks_different_access_size(ptr %p0, ptr %p0.out, ptr %p1, ptr %p1.out, i64 %n) {
+; CHECK-LABEL: define void @constant_strided_two_checks_different_access_size(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P0_OUT:%.*]], ptr [[P1:%.*]], ptr [[P1_OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P14:%.*]] = ptrtoaddr ptr [[P1]] to i64
+; CHECK-NEXT: [[P1_OUT3:%.*]] = ptrtoaddr ptr [[P1_OUT]] to i64
+; CHECK-NEXT: [[P02:%.*]] = ptrtoaddr ptr [[P0]] to i64
+; CHECK-NEXT: [[P0_OUT1:%.*]] = ptrtoaddr ptr [[P0_OUT]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 9
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[P0_OUT1]], [[P02]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP3]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 13
+; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[P1_OUT3]], [[P14]]
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; CHECK-NEXT: [[DIFF_CHECK5:%.*]] = icmp ult i64 [[TMP11]], [[TMP9]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK5]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], [[SCALAR_PH]], [[VECTOR_PH:label %.*]]
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+ %idx = mul nuw nsw i64 %iv, 2
+
+ %gep0.ld = getelementptr inbounds i64, ptr %p0, i64 %idx
+ %gep0.st = getelementptr inbounds i64, ptr %p0.out, i64 %idx
+ %ld0 = load i64, ptr %gep0.ld, align 4, !alias.scope !3, !noalias !4
+ %add0 = add i64 %ld0, 1
+ store i64 %add0, ptr %gep0.st, align 4, !alias.scope !3, !noalias !4
+
+ ; geps are intentionally i64 to keep byte stride same as group 0 above.
+ %gep1.ld = getelementptr inbounds i64, ptr %p1, i64 %idx
+ %gep1.st = getelementptr inbounds i64, ptr %p1.out, i64 %idx
+ %ld1 = load i32, ptr %gep1.ld, align 4, !alias.scope !4, !noalias !3
+ %add1 = add i32 %ld1, 1
+ store i32 %add1, ptr %gep1.st, align 4, !alias.scope !4, !noalias !3
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+; Alias scope domain.
+!0 = !{!0}
+
+; Two alias scopes.
+!1 = !{!1, !0}
+!2 = !{!2, !0}
+
+; And finally scope lists.
+!3 = !{!1}
+!4 = !{!2}
>From 11a7090bb2359c14cb75e244bae668ed3964a387 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 3 Aug 2026 16:31:40 -0700
Subject: [PATCH 11/14] Add test showing how we incorrectly truncate
`ThresholdMinusOne`
AI-generated, then reviewed/modified manually.
---
...ime-checks-diff-wrap-i128-address-space.ll | 48 +++++++++++++++++++
...time-checks-diff-wrap-i32-address-space.ll | 17 +++----
2 files changed, 57 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
new file mode 100644
index 0000000000000..46dbb85249d1d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph\:" --version 6
+; RUN: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:128:128:128:128-i64:64-n32:64"
+
+; With IC=2 and stride=0xFFFFFFFFFFFFFFFF (2^64-1), IC * stride is 2^65-2 which
+; would wrap in uint64_t type..
+;
+; FIXME: ThresholdMinusOne 55340232221128654841 (0x2,FFFF,FFFF,FFFF,FFF9) is
+; wrong. It should be 4 (VF) * 2 (IC) * (2^64 - 1) (Stride) - ((2^64 - 1)
+; (Stride) - 1 (AccessSize) + 1) = 129127208515966861305
+; (0x6,FFFF,FFFF,FFFF,FFF9). We miscalculate it by truncating IC * stride to
+; uint64_t before extending it to i128.
+define void @test_wrap_i128_address_space(ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %n) {
+; CHECK-LABEL: define void @test_wrap_i128_address_space(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr addrspace(1) [[A]] to i128
+; CHECK-NEXT: [[B1:%.*]] = ptrtoaddr ptr addrspace(1) [[B]] to i128
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i128 [[B1]], [[A2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i128 [[TMP0]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i128 [[TMP1]], 55340232221128654841
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+
+ %iv.ext = zext i32 %iv to i128
+ %iv.x.stride = mul i128 %iv.ext, u0xFFFFFFFFFFFFFFFF
+ %gep.a = getelementptr inbounds i8, ptr addrspace(1) %a, i128 %iv.x.stride
+ %ld = load i8, ptr addrspace(1) %gep.a
+ %gep.b = getelementptr inbounds i8, ptr addrspace(1) %b, i128 %iv.x.stride
+ store i8 %ld, ptr addrspace(1) %gep.b
+
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
index 1ef0c9f5fb8dc..f9f8ee1ebf251 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
@@ -1,8 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph\:" --version 5
; RUN: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s
-; With IC=2, stride=2147483648 (2^31), we get IC * stride = 4294967296 (2^32)
-; which wraps to 0 when truncated to i32.
+; With IC=2 and stride=0x80000000 (2^31), IC * stride is 2^32, which wraps
+; to 0 when truncated to i32. The generated check needs i33 to represent its
+; threshold.
target datalayout = "e-p:64:64-p1:32:32-i64:64-n32:64"
@@ -19,21 +20,21 @@ define void @test_wrap_i32_address_space(ptr addrspace(1) %a, ptr addrspace(1) %
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 15032385536
-; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
;
entry:
br label %loop
loop:
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
- ; Access with very large stride: 2^31 bytes = 2147483648
+ ; Access with very large stride: 2^31 bytes.
%iv.ext = zext i32 %iv to i64
- %offset = mul i64 %iv.ext, 2147483648
+ %offset = mul i64 %iv.ext, u0x80000000
; I believe this produces a poison that becomes UB when dereferenced, but even
; for UB input we can't `assert` and have produce something. Ensure that
- ; DIFF_CHECK above (even if constant-foldable to false) is done in i64 that
- ; can represent IC x ConstantStride.
+ ; DIFF_CHECK above (even if constant-foldable to false) is done in a type
+ ; that can represent IC x ConstantStride.
%gep.a = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %offset
%l = load i8, ptr addrspace(1) %gep.a
%add = add i8 %l, 1
>From 705e252cd7d4b79d87fde40cef3ba562e21f721a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 4 Aug 2026 07:47:56 -0700
Subject: [PATCH 12/14] Fix overflow in IC*Stride calculation
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 23 +++++++++++--------
...ime-checks-diff-wrap-i128-address-space.ll | 8 +------
2 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 2dd7808dd8da3..d3a1d197450ce 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2295,8 +2295,8 @@ Value *llvm::addDiffRuntimeChecks(
// Cache of (VF*IC*Stride-(Stride-AccessSize)) - 1, shared across checks with
// matching type/IC/Stride to avoid emitting duplicate runtime computations.
DenseMap<
- std::tuple<Type *, unsigned /*IC*/, unsigned /*AbsCommonStrideInBytes*/,
- unsigned /*AccessSize*/>,
+ std::tuple<Type *, unsigned /*IC*/,
+ uint64_t /*AbsCommonStrideInBytes*/, unsigned /*AccessSize*/>,
Value *>
ThresholdCache;
for (const auto &[SrcStart, SinkStart, AccessSize, AbsCommonStrideInBytes,
@@ -2304,22 +2304,27 @@ Value *llvm::addDiffRuntimeChecks(
assert(IC * AccessSize > 0 &&
"Threshold must be non-zero to use diff-check");
Type *Ty = SinkStart->getType();
+
// Compute the distance between first/last bytes of the accessed memory
// during one vector loop iteration. This is equal to
// VF*IC*Stride-(Stride-AccessSize).
- uint64_t ICTimesStride = IC * AbsCommonStrideInBytes;
- if (!isUIntN(Ty->getScalarSizeInBits(), ICTimesStride)) {
- // This is probably UB in the original IR, but let's be conservative:
- Ty = Ty->getWithNewBitWidth(Ty->getScalarSizeInBits() * 2);
- assert(isUIntN(Ty->getScalarSizeInBits(), ICTimesStride));
- }
+ static_assert((sizeof(IC) + sizeof(AbsCommonStrideInBytes)) * 8 <= 128,
+ "APInt below would overflow!");
+ APInt ICTimesStride(128, IC);
+ ICTimesStride *= APInt(128, AbsCommonStrideInBytes);
+ if (ICTimesStride.getActiveBits() > Ty->getScalarSizeInBits())
+ Ty = Ty->getWithNewBitWidth(
+ NextPowerOf2(ICTimesStride.getActiveBits() - 1));
+
Value *&ThresholdMinusOne =
ThresholdCache[{Ty, IC, AbsCommonStrideInBytes, AccessSize}];
if (!ThresholdMinusOne)
ThresholdMinusOne = ChkBuilder.CreateSub(
ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, ICTimesStride)),
+ ConstantInt::get(
+ Ty, ICTimesStride.zextOrTrunc(
+ Ty->getScalarSizeInBits()))),
ConstantInt::get(Ty, AbsCommonStrideInBytes - AccessSize + 1));
Value *Diff = Expander.expandCodeFor(
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
index 46dbb85249d1d..4160e9027587d 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
@@ -5,12 +5,6 @@ target datalayout = "e-p:64:64-p1:128:128:128:128-i64:64-n32:64"
; With IC=2 and stride=0xFFFFFFFFFFFFFFFF (2^64-1), IC * stride is 2^65-2 which
; would wrap in uint64_t type..
-;
-; FIXME: ThresholdMinusOne 55340232221128654841 (0x2,FFFF,FFFF,FFFF,FFF9) is
-; wrong. It should be 4 (VF) * 2 (IC) * (2^64 - 1) (Stride) - ((2^64 - 1)
-; (Stride) - 1 (AccessSize) + 1) = 129127208515966861305
-; (0x6,FFFF,FFFF,FFFF,FFF9). We miscalculate it by truncating IC * stride to
-; uint64_t before extending it to i128.
define void @test_wrap_i128_address_space(ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %n) {
; CHECK-LABEL: define void @test_wrap_i128_address_space(
; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[N:%.*]]) {
@@ -22,7 +16,7 @@ define void @test_wrap_i128_address_space(ptr addrspace(1) %a, ptr addrspace(1)
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = sub i128 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP1:%.*]] = sub i128 [[TMP0]], 1
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i128 [[TMP1]], 55340232221128654841
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i128 [[TMP1]], 129127208515966861305
; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
;
>From 29b3512b0820f115724cea0d17737de79d07c444 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 4 Aug 2026 07:50:23 -0700
Subject: [PATCH 13/14] Fix typo in a comment
---
.../runtime-checks-diff-wrap-i32-address-space.ll | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
index f9f8ee1ebf251..48807565f1926 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
@@ -32,9 +32,9 @@ loop:
%iv.ext = zext i32 %iv to i64
%offset = mul i64 %iv.ext, u0x80000000
; I believe this produces a poison that becomes UB when dereferenced, but even
- ; for UB input we can't `assert` and have produce something. Ensure that
- ; DIFF_CHECK above (even if constant-foldable to false) is done in a type
- ; that can represent IC x ConstantStride.
+ ; for UB input we can't `assert` and have to produce something. Ensure that
+ ; DIFF_CHECK above (even if constant-foldable to false) is done in a type that
+ ; can represent IC x ConstantStride.
%gep.a = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %offset
%l = load i8, ptr addrspace(1) %gep.a
%add = add i8 %l, 1
>From c0fa50ea931fd89614ed2702334eee78cf626ca6 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 4 Aug 2026 07:52:29 -0700
Subject: [PATCH 14/14] Rename test files/move comment to the test function
(instead of module-level)
---
...ss-space.ll => runtime-checks-diff-i128-address-space.ll} | 0
...ess-space.ll => runtime-checks-diff-i32-address-space.ll} | 5 ++---
2 files changed, 2 insertions(+), 3 deletions(-)
rename llvm/test/Transforms/LoopVectorize/{runtime-checks-diff-wrap-i128-address-space.ll => runtime-checks-diff-i128-address-space.ll} (100%)
rename llvm/test/Transforms/LoopVectorize/{runtime-checks-diff-wrap-i32-address-space.ll => runtime-checks-diff-i32-address-space.ll} (99%)
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-i128-address-space.ll
similarity index 100%
rename from llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i128-address-space.ll
rename to llvm/test/Transforms/LoopVectorize/runtime-checks-diff-i128-address-space.ll
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-i32-address-space.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
rename to llvm/test/Transforms/LoopVectorize/runtime-checks-diff-i32-address-space.ll
index 48807565f1926..54f62c4f3c678 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-wrap-i32-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-diff-i32-address-space.ll
@@ -1,12 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph\:" --version 5
; RUN: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s
+target datalayout = "e-p:64:64-p1:32:32-i64:64-n32:64"
+
; With IC=2 and stride=0x80000000 (2^31), IC * stride is 2^32, which wraps
; to 0 when truncated to i32. The generated check needs i33 to represent its
; threshold.
-
-target datalayout = "e-p:64:64-p1:32:32-i64:64-n32:64"
-
define void @test_wrap_i32_address_space(ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %n) {
; CHECK-LABEL: define void @test_wrap_i32_address_space(
; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[N:%.*]]) {
More information about the llvm-commits
mailing list