[llvm] [VectorUtils] Collect predicates separately in collectConstStrideAccesses. (PR #200807)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 13:01:47 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/200807
>From 5be9ec6416ddb9bf30904f582a05bf63f4ea03c9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 21 May 2026 17:25:04 +0100
Subject: [PATCH] [VectorUtils] Collect predicates separately in
collectConstStrideAccesses.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 20 ++++++-
llvm/include/llvm/Analysis/VectorUtils.h | 6 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 58 ++++++++++++++-----
llvm/lib/Analysis/ScalarEvolution.cpp | 21 ++++++-
llvm/lib/Analysis/VectorUtils.cpp | 21 ++++++-
...untime-checks-after-dependence-analysis.ll | 48 +++++++--------
.../AArch64/discarded-interleave-group.ll | 42 +++++++++-----
7 files changed, 153 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..f51d8af1b56c9 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -886,8 +886,12 @@ replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
///
/// If necessary this method will version the stride of the pointer according
/// to \p PtrToStride and therefore add further predicates to \p PSE.
-/// The \p Assume parameter indicates if we are allowed to make additional
-/// run-time assumptions.
+///
+/// If \p Predicates is non-null, the no-wrap SCEV predicates needed to enable
+/// the stride analysis are appended to it rather than added to \p PSE, letting
+/// the caller discard them if the result turns out to be unused. (Stride
+/// versioning may still add predicates to \p PSE directly.) If \p Predicates is
+/// null, no such no-wrap predicates are collected.
///
/// Note that the analysis results are defined if-and-only-if the original
/// memory access was defined. If that access was dead, or UB, then the
@@ -897,7 +901,17 @@ getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp, const DominatorTree &DT,
const DenseMap<Value *, const SCEV *> &StridesMap =
DenseMap<Value *, const SCEV *>(),
- bool Assume = false, bool ShouldCheckWrap = true);
+ bool ShouldCheckWrap = true,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
+
+/// Overload of \ref getPtrStride that adds the no-wrap predicates directly to
+/// \p PSE. The \p Assume parameter indicates whether such additional run-time
+/// assumptions are allowed.
+LLVM_ABI std::optional<int64_t>
+getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
+ const Loop *Lp, const DominatorTree &DT,
+ const DenseMap<Value *, const SCEV *> &StridesMap, bool Assume,
+ bool ShouldCheckWrap = true);
/// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
/// compatible and it is possible to calculate the distance between them. This
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index b863a307b143e..b483466979786 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -805,8 +805,12 @@ class InterleavedAccessInfo {
delete Group;
}
- /// Collect all the accesses with a constant stride in program order.
+ /// Collect all the accesses with a constant stride in program order. Any
+ /// SCEV predicates needed to compute the strides are appended to \p
+ /// Predicates rather than being added to PSE, so they can be discarded if no
+ /// interleave group is formed.
void collectConstStrideAccesses(
+ SmallVectorImpl<const SCEVPredicate *> &Predicates,
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
const DenseMap<Value *, const SCEV *> &Strides);
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 74f0fa8a954da..d6c2bdc73fbeb 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1022,15 +1022,17 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
/// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use
/// informating from the IR pointer value to determine no-wrap.
-static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
- Value *Ptr, Type *AccessTy, const Loop *L, bool Assume,
- const DominatorTree &DT,
- std::optional<int64_t> Stride = std::nullopt) {
+static bool
+isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr,
+ Type *AccessTy, const Loop *L, const DominatorTree &DT,
+ std::optional<int64_t> Stride = std::nullopt,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr) {
// FIXME: This should probably only return true for NUW.
if (any(AR->getNoWrapFlags(SCEV::NoWrapMask)))
return true;
- if (Ptr && PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
+ if (Ptr && isa<SCEVAddRecExpr>(PSE.getSCEV(Ptr)) &&
+ PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
return true;
// An nusw getelementptr that is an AddRec cannot wrap. If it would wrap,
@@ -1066,8 +1068,12 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
return true;
}
- if (Ptr && Assume) {
- PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
+ if (Ptr && Predicates) {
+ ScalarEvolution *SE = PSE.getSE();
+ SCEVWrapPredicate::IncrementWrapFlags Flags = SCEVWrapPredicate::clearFlags(
+ SCEVWrapPredicate::IncrementNUSW,
+ SCEVWrapPredicate::getImpliedFlags(AR, *SE));
+ Predicates->push_back(SE->getWrapPredicate(AR, Flags));
LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
<< "LAA: Pointer: " << *Ptr << "\n"
<< "LAA: SCEV: " << *AR << "\n"
@@ -1295,6 +1301,7 @@ bool AccessAnalysis::createCheckForAccess(
/// Check whether all pointers can participate in a runtime bounds check. They
/// must either be invariant or non-wrapping affine AddRecs.
+ SmallVector<const SCEVPredicate *> Predicates;
for (auto &P : RTCheckPtrs) {
// The bounds for loop-invariant pointer is trivial.
if (SE->isLoopInvariant(P.getPointer(), TheLoop))
@@ -1315,9 +1322,12 @@ bool AccessAnalysis::createCheckForAccess(
}
if (!isNoWrap(PSE, AR, RTCheckPtrs.size() == 1 ? Ptr : nullptr, AccessTy,
- TheLoop, Assume, DT))
+ TheLoop, DT, /*Stride=*/std::nullopt,
+ Assume ? &Predicates : nullptr))
return false;
}
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
for (const auto &[PtrExpr, NeedsFreeze] : RTCheckPtrs) {
// The id of the dependence set.
@@ -1646,11 +1656,10 @@ void AccessAnalysis::buildDependenceSets() {
}
/// Check whether the access through \p Ptr has a constant stride.
-std::optional<int64_t>
-llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
- const Loop *Lp, const DominatorTree &DT,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- bool Assume, bool ShouldCheckWrap) {
+std::optional<int64_t> llvm::getPtrStride(
+ PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp,
+ const DominatorTree &DT, const DenseMap<Value *, const SCEV *> &StridesMap,
+ bool ShouldCheckWrap, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
if (PSE.getSE()->isLoopInvariant(PtrScev, Lp))
return 0;
@@ -1658,8 +1667,10 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr");
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
- if (Assume && !AR)
- AR = PSE.getAsAddRec(Ptr);
+ if (Predicates && !AR) {
+ AR = PSE.getSE()->convertSCEVToAddRecWithPredicates(PtrScev, Lp,
+ *Predicates);
+ }
if (!AR) {
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not an AddRecExpr pointer " << *Ptr
@@ -1672,7 +1683,7 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
if (!ShouldCheckWrap || !Stride)
return Stride;
- if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, Assume, DT, Stride))
+ if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, DT, Stride, Predicates))
return Stride;
LLVM_DEBUG(
@@ -1681,6 +1692,21 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
return std::nullopt;
}
+/// Check whether the access through \p Ptr has a constant stride.
+std::optional<int64_t>
+llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
+ const Loop *Lp, const DominatorTree &DT,
+ const DenseMap<Value *, const SCEV *> &StridesMap,
+ bool Assume, bool ShouldCheckWrap) {
+ SmallVector<const SCEVPredicate *> Predicates;
+ std::optional<int64_t> Stride =
+ getPtrStride(PSE, AccessTy, Ptr, Lp, DT, StridesMap, ShouldCheckWrap,
+ Assume ? &Predicates : nullptr);
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
+ return Stride;
+}
+
std::optional<int64_t> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
Type *ElemTyB, Value *PtrB,
const DataLayout &DL,
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index c93838ffdc50d..e1207d2d625bb 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -15448,8 +15448,25 @@ bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
return this->implies(I, SE);
});
- return any_of(Preds,
- [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); });
+ if (any_of(Preds,
+ [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
+ return true;
+
+ // A wrap predicate may be implied by a wrap predicate in Preds after applying
+ // equal predicates.
+ const auto *NWrap = dyn_cast<SCEVWrapPredicate>(N);
+ if (!NWrap)
+ return false;
+ const Loop *L = NWrap->getExpr()->getLoop();
+ return any_of(Preds, [&](const SCEVPredicate *I) {
+ const auto *IWrap = dyn_cast<SCEVWrapPredicate>(I);
+ if (!IWrap)
+ return false;
+ const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
+ SE.rewriteUsingPredicate(IWrap->getExpr(), L, *this));
+ return RewrittenAR &&
+ SE.getWrapPredicate(RewrittenAR, IWrap->getFlags())->implies(N, SE);
+ });
}
void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index ecded79e990f3..28e728bb4c5ac 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1308,8 +1308,9 @@ bool InterleavedAccessInfo::isStrided(int Stride) {
}
void InterleavedAccessInfo::collectConstStrideAccesses(
+ SmallVectorImpl<const SCEVPredicate *> &Predicates,
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value*, const SCEV*> &Strides) {
+ const DenseMap<Value *, const SCEV *> &Strides) {
auto &DL = TheLoop->getHeader()->getDataLayout();
// Since it's desired that the load/store instructions be maintained in
@@ -1341,7 +1342,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses(
// even without the transformation. The wrapping checks are therefore
// deferred until after we've formed the interleaved groups.
int64_t Stride = getPtrStride(PSE, ElementTy, Ptr, TheLoop, *DT, Strides,
- /*Assume=*/true, /*ShouldCheckWrap=*/false)
+ /*ShouldCheckWrap=*/false, &Predicates)
.value_or(0);
const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
@@ -1391,9 +1392,14 @@ void InterleavedAccessInfo::analyzeInterleaving(
LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");
const auto &Strides = LAI->getSymbolicStrides();
+ // Collect the SCEV predicates needed by the stride analysis below. They are
+ // only added to PSE if at least one interleave group is formed, so they can
+ // be discarded otherwise.
+ SmallVector<const SCEVPredicate *> Predicates;
+
// Holds all accesses with a constant stride.
MapVector<Instruction *, StrideDescriptor> AccessStrideInfo;
- collectConstStrideAccesses(AccessStrideInfo, Strides);
+ collectConstStrideAccesses(Predicates, AccessStrideInfo, Strides);
if (AccessStrideInfo.empty())
return;
@@ -1589,6 +1595,15 @@ void InterleavedAccessInfo::analyzeInterleaving(
} // Iteration over A accesses.
} // Iteration over B accesses.
+ // The predicates collected while forming the candidate groups above are
+ // needed by the wrap-check below (which uses Assume=false and therefore
+ // relies on the no-wrap assumptions already being available in PSE) and by
+ // later passes. Commit them now if any candidate group was formed; otherwise
+ // they are unused and discarded.
+ if (!LoadGroups.empty() || !StoreGroups.empty())
+ for (const SCEVPredicate *P : Predicates)
+ PSE.addPredicate(*P);
+
auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup<Instruction> *Group,
int Index,
const char *FirstOrLast) -> bool {
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 63abd4ef70d63..88b16d0f32534 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -72,27 +72,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_st
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b))
-; CHECK-NEXT: Member: {%b,+,5}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b))
+; CHECK-NEXT: Member: {%b,+,5}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -265,27 +265,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_may_wrap(ptr %a, pt
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b))
-; CHECK-NEXT: Member: {%b,+,8}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b))
+; CHECK-NEXT: Member: {%b,+,8}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -325,29 +325,29 @@ define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
; CHECK-NEXT: Run-time memory checks:
; CHECK-NEXT: Check 0:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT: ptr %A
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: ptr %A
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: ptr %A
+; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %A High: (4 + %A))
+; CHECK-NEXT: Member: %A
+; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + %A),+,8}<%loop>
-; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: (100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A) High: (96 + (16 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop>
-; CHECK-NEXT: Group GRP2:
-; CHECK-NEXT: (Low: %A High: (4 + %A))
-; CHECK-NEXT: Member: %A
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
index 4d21309776b40..1c79629ac1754 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
@@ -3,28 +3,42 @@
target triple = "arm64-apple-macosx"
-; FIXME: should vectorize without predicates.
define void @urem_lookup(ptr noalias %src, ptr noalias %dst, ptr noalias %tbl, i64 %N) #0 {
; CHECK-LABEL: define void @urem_lookup(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[TBL:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[LOOP]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load float, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[CLAMPED:%.*]] = urem i64 [[IV]], 4
-; CHECK-NEXT: [[GEP_TBL:%.*]] = getelementptr inbounds float, ptr [[TBL]], i64 [[CLAMPED]]
-; CHECK-NEXT: [[T:%.*]] = load float, ptr [[GEP_TBL]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul float [[X]], [[T]]
-; CHECK-NEXT: [[M2:%.*]] = fadd float [[M1]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = fmul float [[M2]], [[T]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[GEP_SRC]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = urem <vscale x 4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TBL]], <vscale x 4 x i64> [[TMP4]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd <vscale x 4 x float> [[TMP6]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <vscale x 4 x float> [[TMP7]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: store float [[M3]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP8]], ptr align 4 [[GEP_DST]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], [[TMP1]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list