[llvm] [VectorUtils] Collect predicates separately in collectConstStrideAccesses. (PR #200807)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 06:05:37 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/200807
None
>From 75a18f597335e7cb6de00ad12b1410766896789f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 21 May 2026 17:25:04 +0100
Subject: [PATCH] [VectorUtils] Collect predicates separately in
collectConstStrideAccesses.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 14 +++-
llvm/include/llvm/Analysis/VectorUtils.h | 6 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 65 ++++++++++++++-----
llvm/lib/Analysis/ScalarEvolution.cpp | 21 +++++-
llvm/lib/Analysis/VectorUtils.cpp | 21 +++++-
.../AArch64/discarded-interleave-group.ll | 42 ++++++++----
6 files changed, 133 insertions(+), 36 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..7528debf84367 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -889,6 +889,11 @@ replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
/// The \p Assume parameter indicates if we are allowed to make additional
/// run-time assumptions.
///
+/// If \p Predicates is non-null, the no-wrap SCEV predicates that would
+/// otherwise be added to \p PSE to enable the stride analysis are instead
+/// appended to it, leaving \p PSE unchanged. This allows the caller to discard
+/// the predicates if the result turns out to be unused.
+///
/// Note that the analysis results are defined if-and-only-if the original
/// memory access was defined. If that access was dead, or UB, then the
/// result of this function is undefined.
@@ -897,7 +902,14 @@ getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp, const DominatorTree &DT,
const DenseMap<Value *, const SCEV *> &StridesMap =
DenseMap<Value *, const SCEV *>(),
- bool Assume = false, bool ShouldCheckWrap = true);
+ bool ShouldCheckWrap = true,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
+
+LLVM_ABI std::optional<int64_t>
+getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
+ const Loop *Lp, const DominatorTree &DT,
+ const DenseMap<Value *, const SCEV *> &StridesMap, bool Assume,
+ bool ShouldCheckWrap = true);
/// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
/// compatible and it is possible to calculate the distance between them. This
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 3fd4de926683c..9ae21fc5554a8 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -820,8 +820,12 @@ class InterleavedAccessInfo {
delete Group;
}
- /// Collect all the accesses with a constant stride in program order.
+ /// Collect all the accesses with a constant stride in program order. Any
+ /// SCEV predicates needed to compute the strides are appended to \p
+ /// Predicates rather than being added to PSE, so they can be discarded if no
+ /// interleave group is formed.
void collectConstStrideAccesses(
+ SmallVectorImpl<const SCEVPredicate *> &Predicates,
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
const DenseMap<Value *, const SCEV *> &Strides);
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 60e15b2a5bd82..99e9e429f5168 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1023,15 +1023,17 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
/// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use
/// informating from the IR pointer value to determine no-wrap.
-static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
- Value *Ptr, Type *AccessTy, const Loop *L, bool Assume,
- const DominatorTree &DT,
- std::optional<int64_t> Stride = std::nullopt) {
+static bool
+isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr,
+ Type *AccessTy, const Loop *L, const DominatorTree &DT,
+ std::optional<int64_t> Stride = std::nullopt,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr) {
// FIXME: This should probably only return true for NUW.
if (any(AR->getNoWrapFlags(SCEV::NoWrapMask)))
return true;
- if (Ptr && PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
+ if (Ptr && isa<SCEVAddRecExpr>(PSE.getSCEV(Ptr)) &&
+ PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
return true;
// An nusw getelementptr that is an AddRec cannot wrap. If it would wrap,
@@ -1067,8 +1069,12 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
return true;
}
- if (Ptr && Assume) {
- PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
+ if (Ptr && Predicates) {
+ ScalarEvolution *SE = PSE.getSE();
+ auto Flags = SCEVWrapPredicate::clearFlags(
+ SCEVWrapPredicate::IncrementNUSW,
+ SCEVWrapPredicate::getImpliedFlags(AR, *SE));
+ Predicates->push_back(SE->getWrapPredicate(AR, Flags));
LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
<< "LAA: Pointer: " << *Ptr << "\n"
<< "LAA: SCEV: " << *AR << "\n"
@@ -1079,6 +1085,19 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
return false;
}
+static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
+ Value *Ptr, Type *AccessTy, const Loop *L, bool Assume,
+ const DominatorTree &DT,
+ std::optional<int64_t> Stride = std::nullopt) {
+
+ SmallVector<const SCEVPredicate *> Predicates;
+ bool NoWrap = isNoWrap(PSE, AR, Ptr, AccessTy, L, DT, Stride,
+ Assume ? &Predicates : nullptr);
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
+ return NoWrap;
+}
+
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop,
function_ref<void(Value *)> AddPointer) {
SmallPtrSet<Value *, 8> Visited;
@@ -1647,11 +1666,10 @@ void AccessAnalysis::buildDependenceSets() {
}
/// Check whether the access through \p Ptr has a constant stride.
-std::optional<int64_t>
-llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
- const Loop *Lp, const DominatorTree &DT,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- bool Assume, bool ShouldCheckWrap) {
+std::optional<int64_t> llvm::getPtrStride(
+ PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp,
+ const DominatorTree &DT, const DenseMap<Value *, const SCEV *> &StridesMap,
+ bool ShouldCheckWrap, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
if (PSE.getSE()->isLoopInvariant(PtrScev, Lp))
return 0;
@@ -1659,8 +1677,10 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr");
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
- if (Assume && !AR)
- AR = PSE.getAsAddRec(Ptr);
+ if (Predicates && !AR) {
+ AR = PSE.getSE()->convertSCEVToAddRecWithPredicates(PtrScev, Lp,
+ *Predicates);
+ }
if (!AR) {
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not an AddRecExpr pointer " << *Ptr
@@ -1673,7 +1693,7 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
if (!ShouldCheckWrap || !Stride)
return Stride;
- if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, Assume, DT, Stride))
+ if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, DT, Stride, Predicates))
return Stride;
LLVM_DEBUG(
@@ -1682,6 +1702,21 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
return std::nullopt;
}
+/// Check whether the access through \p Ptr has a constant stride.
+std::optional<int64_t>
+llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
+ const Loop *Lp, const DominatorTree &DT,
+ const DenseMap<Value *, const SCEV *> &StridesMap,
+ bool Assume, bool ShouldCheckWrap) {
+ SmallVector<const SCEVPredicate *> Predicates;
+ std::optional<int64_t> Stride =
+ getPtrStride(PSE, AccessTy, Ptr, Lp, DT, StridesMap, ShouldCheckWrap,
+ Assume ? &Predicates : nullptr);
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
+ return Stride;
+}
+
std::optional<int64_t> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
Type *ElemTyB, Value *PtrB,
const DataLayout &DL,
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index ca92aa2745b6e..60560fdfacb8f 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -15452,8 +15452,25 @@ bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
return this->implies(I, SE);
});
- return any_of(Preds,
- [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); });
+ if (any_of(Preds,
+ [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
+ return true;
+
+ // A wrap predicate may be implied by a warp predicate in Preds after applying
+ // equal predicates.
+ const auto *NWrap = dyn_cast<SCEVWrapPredicate>(N);
+ if (!NWrap)
+ return false;
+ const Loop *L = NWrap->getExpr()->getLoop();
+ return any_of(Preds, [&](const SCEVPredicate *I) {
+ const auto *IWrap = dyn_cast<SCEVWrapPredicate>(I);
+ if (!IWrap)
+ return false;
+ const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
+ SE.rewriteUsingPredicate(IWrap->getExpr(), L, *this));
+ return RewrittenAR &&
+ SE.getWrapPredicate(RewrittenAR, IWrap->getFlags())->implies(N, SE);
+ });
}
void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 402bef65c2f13..fbeb40480805c 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1358,8 +1358,9 @@ bool InterleavedAccessInfo::isStrided(int Stride) {
}
void InterleavedAccessInfo::collectConstStrideAccesses(
+ SmallVectorImpl<const SCEVPredicate *> &Predicates,
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value*, const SCEV*> &Strides) {
+ const DenseMap<Value *, const SCEV *> &Strides) {
auto &DL = TheLoop->getHeader()->getDataLayout();
// Since it's desired that the load/store instructions be maintained in
@@ -1391,7 +1392,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses(
// even without the transformation. The wrapping checks are therefore
// deferred until after we've formed the interleaved groups.
int64_t Stride = getPtrStride(PSE, ElementTy, Ptr, TheLoop, *DT, Strides,
- /*Assume=*/true, /*ShouldCheckWrap=*/false)
+ /*ShouldCheckWrap=*/false, &Predicates)
.value_or(0);
const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
@@ -1441,9 +1442,14 @@ void InterleavedAccessInfo::analyzeInterleaving(
LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");
const auto &Strides = LAI->getSymbolicStrides();
+ // Collect the SCEV predicates needed by the stride analysis below. They are
+ // only added to PSE if at least one interleave group is formed, so they can
+ // be discarded otherwise.
+ SmallVector<const SCEVPredicate *> Predicates;
+
// Holds all accesses with a constant stride.
MapVector<Instruction *, StrideDescriptor> AccessStrideInfo;
- collectConstStrideAccesses(AccessStrideInfo, Strides);
+ collectConstStrideAccesses(Predicates, AccessStrideInfo, Strides);
if (AccessStrideInfo.empty())
return;
@@ -1639,6 +1645,15 @@ void InterleavedAccessInfo::analyzeInterleaving(
} // Iteration over A accesses.
} // Iteration over B accesses.
+ // The predicates collected while forming the candidate groups above are
+ // needed by the wrap-check below (which uses Assume=false and therefore
+ // relies on the no-wrap assumptions already being available in PSE) and by
+ // later passes. Commit them now if any candidate group was formed; otherwise
+ // they are unused and discarded.
+ if (!LoadGroups.empty() || !StoreGroups.empty())
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
+
auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup<Instruction> *Group,
int Index,
const char *FirstOrLast) -> bool {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
index 4d21309776b40..1c79629ac1754 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
@@ -3,28 +3,42 @@
target triple = "arm64-apple-macosx"
-; FIXME: should vectorize without predicates.
define void @urem_lookup(ptr noalias %src, ptr noalias %dst, ptr noalias %tbl, i64 %N) #0 {
; CHECK-LABEL: define void @urem_lookup(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[TBL:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[LOOP]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load float, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[CLAMPED:%.*]] = urem i64 [[IV]], 4
-; CHECK-NEXT: [[GEP_TBL:%.*]] = getelementptr inbounds float, ptr [[TBL]], i64 [[CLAMPED]]
-; CHECK-NEXT: [[T:%.*]] = load float, ptr [[GEP_TBL]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul float [[X]], [[T]]
-; CHECK-NEXT: [[M2:%.*]] = fadd float [[M1]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = fmul float [[M2]], [[T]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[GEP_SRC]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = urem <vscale x 4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TBL]], <vscale x 4 x i64> [[TMP4]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd <vscale x 4 x float> [[TMP6]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <vscale x 4 x float> [[TMP7]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: store float [[M3]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP8]], ptr align 4 [[GEP_DST]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], [[TMP1]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list