[llvm] [LAA] Add stencil group merging to reduce runtime pointer checks (PR #187252)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 02:31:52 PDT 2026
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/187252
>From 456765db1a43be737ca31e18d8f21aca8c2125cc Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 12:17:21 +0000
Subject: [PATCH 01/33] [LAA] Add stencil group merging to reduce runtime
pointer checks
Add a post-processing pass in RuntimePointerChecking that merges
checking groups whose pointers share a base object and differ only
by linear combinations of loop-invariant strides. This reduces the
number of runtime overlap checks for stencil-like access patterns,
enabling vectorization of loops that would otherwise be rejected
due to exceeding the runtime-check threshold.
The primary motivation is the Einstein Toolkit / Cactus CCZ4
numerical relativity stencil code, where 3D stencils with two
runtime strides (cdj, cdk) produce thousands of runtime checks.
The optimization:
- Decomposes member offsets into constant + sum(coeff * stride)
- Computes a bounding box over the coefficient space
- Applies a cost model: only merges when checks_after < checks_before
- Adds stride > 0 predicates for non-provably-positive strides
- Rejects predicated accesses and write groups conservatively
Gated behind -enable-stencil-runtime-check-merge (default: off).
Also activates automatically when check count exceeds
-stencil-merge-check-threshold (default: 128).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 8 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 380 ++++-
.../affine-group-merging.ll | 1406 +++++++++++++++++
3 files changed, 1791 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 792b8ba9cabc5..4c76fa2e98ebe 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -603,7 +603,8 @@ class RuntimePointerChecking {
/// Generate the checks and store it. This also performs the grouping
/// of pointers to reduce the number of memchecks necessary.
- LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands);
+ LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands,
+ PredicatedScalarEvolution &PSE, Loop &L);
/// Returns the checks that generateChecks created. They can be used to ensure
/// no read/write accesses overlap across all loop iterations.
@@ -673,6 +674,11 @@ class RuntimePointerChecking {
/// and re-compute it.
void groupChecks(MemoryDepChecker::DepCandidates &DepCands);
+ /// Attempt to merge checking groups that share a base pointer and differ
+ /// by stencil functions of loop-invariant strides. This reduces runtime
+ /// checks for multi-dimensional stencil-like access patterns.
+ void mergeStencilGroups(PredicatedScalarEvolution &PSE, Loop &L);
+
/// Generate the checks and return them.
SmallVector<RuntimePointerCheck, 4> generateChecks();
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 437d25b3d56ed..d4338090f7fe1 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
@@ -100,6 +101,18 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable merging of runtime check groups with stencil stride "
+ "patterns (default = false)"),
+ cl::init(false));
+
+static cl::opt<unsigned> StencilMergeCheckThreshold(
+ "stencil-merge-check-threshold", cl::Hidden,
+ cl::desc("Trigger stencil group merging when runtime check count "
+ "exceeds this threshold (default = 128)"),
+ cl::init(128));
+
/// Maximum SIMD width.
const unsigned VectorizerParams::MaxVectorWidth = 64;
@@ -628,9 +641,11 @@ SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() {
}
void RuntimePointerChecking::generateChecks(
- MemoryDepChecker::DepCandidates &DepCands) {
+ MemoryDepChecker::DepCandidates &DepCands, PredicatedScalarEvolution &PSE,
+ Loop &L) {
assert(Checks.empty() && "Checks is not empty");
groupChecks(DepCands);
+ mergeStencilGroups(PSE, L);
Checks = generateChecks();
}
@@ -813,6 +828,367 @@ void RuntimePointerChecking::groupChecks(
}
}
+/// Result of decomposing a SCEV expression into stencil offset form:
+/// Offset = Constant + sum(Coefficients[stride] * stride)
+/// where each stride is a loop-invariant SCEV expression.
+struct StencilDecomposition {
+ int64_t Constant = 0;
+ /// Map from loop-invariant stride SCEV to its integer coefficient.
+ SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+};
+
+/// Try to decompose \p Expr into a stencil offset function of loop-invariant
+/// strides: C + a1*s1 + a2*s2 + ...
+/// Relies on SCEV's canonical form: AddExpr operands are flattened,
+/// MulExpr has the constant operand first.
+/// Returns std::nullopt if the expression contains non-stencil terms.
+static std::optional<StencilDecomposition>
+decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
+ StencilDecomposition D;
+
+ // Collect top-level additive terms.
+ SmallVector<const SCEV *, 4> Terms;
+ if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
+ Terms.append(Add->operands().begin(), Add->operands().end());
+ else
+ Terms.push_back(Expr);
+
+ for (const SCEV *Term : Terms) {
+ if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ D.Constant += C->getAPInt().getSExtValue();
+ } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
+ // SCEV canonical form: constant is operand 0 in a MulExpr.
+ if (Mul->getNumOperands() != 2)
+ return std::nullopt;
+ auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
+ if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
+ return std::nullopt;
+ D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ } else if (SE.isLoopInvariant(Term, &L)) {
+ D.Coefficients[Term] += 1;
+ } else {
+ return std::nullopt;
+ }
+ }
+ return D;
+}
+
+void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
+ Loop &L) {
+ LLVM_DEBUG(dbgs() << "LAA: Attempting stencil group merging on "
+ << CheckingGroups.size() << " groups\n");
+
+ if (CheckingGroups.size() < 2)
+ return;
+
+ // Count the total checks from current grouping.
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks, proceeding with stencil merge\n");
+
+ // Group CheckingGroups by (DependencySetId, AliasSetId) pair.
+ // DependencySetId alone is not unique: it resets per alias set, so
+ // pointers in different alias sets can share the same DependencySetId.
+ // Use MapVector for deterministic iteration order across platforms.
+ using DepAliasKey = std::pair<unsigned, unsigned>;
+ MapVector<DepAliasKey, SmallVector<unsigned, 4>> DepSetToGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ const auto &P = Pointers[CheckingGroups[I].Members[0]];
+ DepSetToGroups[{P.DependencySetId, P.AliasSetId}].push_back(I);
+ }
+
+ SmallDenseSet<unsigned, 4> MergedGroupIndices;
+ SmallVector<RuntimeCheckingPtrGroup, 2> NewMergedGroups;
+ // Track strides that already have committed predicates (across all DepSets).
+ SmallDenseSet<const SCEV *, 4> CommittedStridePredicates;
+
+ for (auto &[DepAliasKey, GroupIndices] : DepSetToGroups) {
+ [[maybe_unused]] auto [DepId, ASId] = DepAliasKey;
+ if (GroupIndices.size() < 2)
+ continue;
+
+ // Collect all member pointers across these groups.
+ SmallVector<unsigned, 8> AllMembers;
+ for (unsigned GI : GroupIndices)
+ AllMembers.append(CheckingGroups[GI].Members.begin(),
+ CheckingGroups[GI].Members.end());
+
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ bool HasPredicatedAccess = false;
+ for (unsigned Idx : AllMembers) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ if (auto *I = dyn_cast<Instruction>(PtrVal)) {
+ if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT())) {
+ HasPredicatedAccess = true;
+ break;
+ }
+ }
+ }
+ if (HasPredicatedAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
+ // Only merge read-only groups. Stencil patterns read an array at
+ // multiple offsets and write to a different array (different DepSet).
+ // Mixing reads and writes within a merged group complicates the cost
+ // model and doesn't match known stencil patterns.
+ bool HasWriteAccess = false;
+ for (unsigned Idx : AllMembers) {
+ if (Pointers[Idx].IsWritePtr) {
+ HasWriteAccess = true;
+ break;
+ }
+ }
+ if (HasWriteAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with write access\n");
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
+ << ") with " << AllMembers.size() << " members, base: "
+ << *Pointers[AllMembers[0]].Start << "\n");
+
+ // Use the first member as the reference for decomposition. All offsets
+ // are computed relative to BaseLow, and MergedHigh is built from
+ // BaseHigh (see merged-bounds computation below).
+ const SCEV *BaseLow = Pointers[AllMembers[0]].Start;
+ const SCEV *BaseHigh = Pointers[AllMembers[0]].End;
+
+ const SCEV *BaseStep = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[AllMembers[0]].Expr))
+ if (AR->getLoop() == &L)
+ BaseStep = AR->getStepRecurrence(*SE);
+
+ if (!BaseStep)
+ continue;
+
+ // Verify all members have the same access range (End - Start).
+ // MergedHigh = BaseHigh + max_offsets is only correct when every
+ // member's range equals BaseHigh - BaseLow. We check by subtracting
+ // ranges and testing for zero, which lets SCEV simplify algebraically
+ // even when the individual range SCEVs aren't pointer-identical.
+ const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ bool RangeMismatch = false;
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different access range, skipping DepSet\n");
+ RangeMismatch = true;
+ break;
+ }
+ }
+ if (RangeMismatch)
+ continue;
+
+ bool DecompOk = true;
+
+ // Verify all members have the same recurrence step w.r.t. the analyzed
+ // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
+ // correct when every member's High-Low range equals BaseHigh - BaseLow.
+ // Different steps (e.g., 8 vs 16) produce different ranges.
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+
+ if (Step != BaseStep) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different step, skipping DepSet\n");
+ DecompOk = false;
+ break;
+ }
+ }
+ if (!DecompOk)
+ continue;
+ SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
+ int64_t CMin = 0, CMax = 0;
+ SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+
+ for (unsigned Idx : AllMembers) {
+ const SCEV *PtrStart = Pointers[Idx].Start;
+
+ const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset)) {
+ DecompOk = false;
+ break;
+ }
+ auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
+ if (!DLow) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " NOT decomposable: " << *LowOffset << "\n");
+ DecompOk = false;
+ break;
+ }
+
+ CMin = std::min(CMin, DLow->Constant);
+ CMax = std::max(CMax, DLow->Constant);
+
+ for (const auto &[Stride, Coeff] : DLow->Coefficients) {
+ auto MinIt = MinCoeff.find(Stride);
+ if (MinIt == MinCoeff.end()) {
+ MinCoeff[Stride] = Coeff;
+ MaxCoeff[Stride] = Coeff;
+ } else {
+ MinIt->second = std::min(MinIt->second, Coeff);
+ MaxCoeff[Stride] = std::max(MaxCoeff[Stride], Coeff);
+ }
+
+ if (!SE->isKnownPositive(Stride))
+ LocalStridesNeedingPreds.insert(Stride);
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
+ << ", strides=" << DLow->Coefficients.size() << "\n");
+ }
+
+ if (!DecompOk)
+ continue;
+
+ // The base member (offset = 0) implicitly has coefficient 0 for every
+ // stride. Members that lack a particular stride term also have an
+ // implicit coefficient of 0 for that stride. Clamp so that the
+ // bounding-box always includes coefficient 0, which is guaranteed to
+ // exist (at least from the base member).
+ for (auto &[Stride, MinC] : MinCoeff)
+ MinC = std::min(MinC, (int64_t)0);
+ for (auto &[Stride, MaxC] : MaxCoeff)
+ MaxC = std::max(MaxC, (int64_t)0);
+
+ // MergedLow = BaseLow + CMin + sum(MinCoeff[s] * s)
+ const SCEV *MergedLow = BaseLow;
+ if (CMin != 0)
+ MergedLow =
+ SE->getAddExpr(MergedLow, SE->getConstant(BaseLow->getType(), CMin,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MinC] : MinCoeff) {
+ if (MinC != 0)
+ MergedLow = SE->getAddExpr(
+ MergedLow, SE->getMulExpr(SE->getConstant(Stride->getType(), MinC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ // MergedHigh = BaseHigh + CMax + sum(MaxCoeff[s] * s)
+ // BaseHigh = BaseLow + Range, so this gives max(Low_j) + Range.
+ const SCEV *MergedHigh = BaseHigh;
+ if (CMax != 0)
+ MergedHigh =
+ SE->getAddExpr(MergedHigh, SE->getConstant(BaseHigh->getType(), CMax,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MaxC] : MaxCoeff) {
+ if (MaxC != 0)
+ MergedHigh = SE->getAddExpr(
+ MergedHigh, SE->getMulExpr(SE->getConstant(Stride->getType(), MaxC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
+ << ", High=" << *MergedHigh << "\n");
+
+ // Local cost model.
+ // G = number of groups in this DepSet, C = number of external groups
+ // that need checking against any member of this DepSet.
+ // Before merge: G * C checks. After merge: C + predicates.
+ unsigned G = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned C = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ // Check if this external group needs checking against any member
+ // of our DepSet.
+ for (unsigned GI : GroupIndices) {
+ if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++C;
+ break;
+ }
+ }
+ }
+
+ // Only count predicates we haven't already committed as cost.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ unsigned ChecksBefore = G * C;
+ unsigned ChecksAfter = C + NewPredicates;
+
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: G=" << G << ", C=" << C
+ << ", predicates=" << NewPredicates << ", checks "
+ << ChecksBefore << "->" << ChecksAfter);
+
+ if (ChecksAfter >= ChecksBefore) {
+ LLVM_DEBUG(dbgs() << " (skipping, not beneficial)\n");
+ continue;
+ }
+ LLVM_DEBUG(dbgs() << " (merging, net saving " << ChecksBefore - ChecksAfter
+ << ")\n");
+
+ // Build the merged group (after deciding to merge).
+ RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
+ CandidateGroup.Low = MergedLow;
+ CandidateGroup.High = MergedHigh;
+ for (unsigned I = 1; I < AllMembers.size(); ++I)
+ CandidateGroup.Members.push_back(AllMembers[I]);
+ for (unsigned GI : GroupIndices)
+ CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
+
+ // COMMIT: add stride predicates (skip already-committed ones).
+ for (const SCEV *Stride : LocalStridesNeedingPreds) {
+ if (!CommittedStridePredicates.insert(Stride).second)
+ continue;
+ const SCEV *Zero = SE->getZero(Stride->getType());
+ PSE.addPredicate(
+ *SE->getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+
+ for (unsigned GI : GroupIndices)
+ MergedGroupIndices.insert(GI);
+ NewMergedGroups.push_back(std::move(CandidateGroup));
+ }
+
+ // Rebuild CheckingGroups if we merged anything.
+ if (!NewMergedGroups.empty()) {
+ SmallVector<RuntimeCheckingPtrGroup, 2> FinalGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ if (!MergedGroupIndices.contains(I))
+ FinalGroups.push_back(std::move(CheckingGroups[I]));
+ FinalGroups.append(std::make_move_iterator(NewMergedGroups.begin()),
+ std::make_move_iterator(NewMergedGroups.end()));
+ CheckingGroups = std::move(FinalGroups);
+
+ LLVM_DEBUG(dbgs() << "LAA: After stencil merging: " << CheckingGroups.size()
+ << " groups\n");
+ }
+}
+
bool RuntimePointerChecking::arePointersInSamePartition(
const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
unsigned PtrIdx2) {
@@ -1609,7 +1985,7 @@ bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
}
if (MayNeedRTCheck && (CanDoRT || AllowPartial))
- RtCheck.generateChecks(DepCands);
+ RtCheck.generateChecks(DepCands, PSE, *TheLoop);
LLVM_DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
<< " pointer comparisons.\n");
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
new file mode 100644
index 0000000000000..fb6b116a5b300
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
@@ -0,0 +1,1406 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+;; Test 1: Basic stencil merge with one runtime stride.
+;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
+;; relative to base = %a + 8*iv. Store to %out.
+;; With merge: all 6 loads form 1 merged group with bounds spanning [-3*cdj, +3*cdj]
+;; relative to base, producing 1 runtime check and 1 stride predicate (cdj > 0).
+;; Without merge: 6 separate groups (each load alone), 6 checks, no predicates.
+define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_single_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_single_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 2: Two runtime strides with constant offsets.
+;; 5 loads from %vol at offsets: +8+cdj-cdk, -16-2*cdj, +24+3*cdj+2*cdk,
+;; -8+2*cdj-3*cdk, +32-4*cdj+cdk. Store to %out.
+;; With merge: 1 merged group with bounds spanning the bounding box over
+;; both cdj and cdk coefficients, 1 check, 2 stride predicates.
+;; Without merge: 5 separate groups, 5 checks, no predicates.
+define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64 %cdk) {
+; MERGE-LABEL: 'stencil_merge_two_strides'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-4 * %cdj) + (-3 * %cdk) + %vol) High: ((2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; MERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdk sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_two_strides'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (64 + (-4 * %cdj) + %cdk + %vol) High: ((8 * %n) + (-4 * %cdj) + %cdk + %vol))
+; NOMERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + (-3 * %cdk) + %vol) High: (-40 + (2 * %cdj) + (8 * %n) + (-3 * %cdk) + %vol))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (56 + (2 * %cdk) + (3 * %cdj) + %vol) High: (-8 + (2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; NOMERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %vol) High: (-48 + (8 * %n) + (-2 * %cdj) + %vol))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (40 + (-1 * %cdk) + %cdj + %vol) High: (-24 + (8 * %n) + (-1 * %cdk) + %cdj + %vol))
+; NOMERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %vol, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; +8 + cdj - cdk
+ %off0a = add nsw i64 8, %cdj
+ %negcdk = sub nsw i64 0, %cdk
+ %off0 = add nsw i64 %off0a, %negcdk
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %v0 = load double, ptr %p0, align 8
+
+ ; -16 - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %off1 = add nsw i64 -16, %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %v1 = load double, ptr %p1, align 8
+
+ ; +24 + 3*cdj + 2*cdk
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %pos2cdk = mul nsw i64 %cdk, 2
+ %off2a = add nsw i64 24, %pos3cdj
+ %off2 = add nsw i64 %off2a, %pos2cdk
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %v2 = load double, ptr %p2, align 8
+
+ ; -8 + 2*cdj - 3*cdk
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %neg3cdk = mul nsw i64 %cdk, -3
+ %off3a = add nsw i64 -8, %pos2cdj
+ %off3 = add nsw i64 %off3a, %neg3cdk
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %v3 = load double, ptr %p3, align 8
+
+ ; +32 - 4*cdj + cdk
+ %neg4cdj = mul nsw i64 %cdj, -4
+ %off4a = add nsw i64 32, %neg4cdj
+ %off4 = add nsw i64 %off4a, %cdk
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %v4 = load double, ptr %p4, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s3, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 3: Constant offsets only — existing grouping handles this.
+;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
+;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
+;; Both with and without flag: 1 check, 2 groups, no predicates.
+define void @constant_offsets_only(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'constant_offsets_only'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'constant_offsets_only'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s2, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 4: Cost model rejection — only 2 loads with runtime stride.
+;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
+;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
+define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'cost_model_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'cost_model_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %v1 = load double, ptr %base, align 8
+
+ %s = fadd double %v0, %v1
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 5: Invariant + strided accesses to same base -> different steps.
+;; A strided store {%a,+,8} and an invariant store to %a have different
+;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Same result with and without flag: 2 checks, 3 separate groups.
+define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'different_steps_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Unknown:
+; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; MERGE-EMPTY:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %a
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'different_steps_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Unknown:
+; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %a
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; GRP0: invariant store to %a (single address, step=0):
+; GRP1: strided store to %a (step=8, different range from GRP0):
+; GRP2: load from %out:
+entry:
+ %cmp = icmp sgt i64 %n, 2
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+
+ ; Strided store: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ store double 1.0, ptr %gep.a, align 8
+
+ ; Invariant store to %a (step = 0, different range)
+ store double 2.0, ptr %a, align 8
+
+ ; Read from %out
+ %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+ %v = load double, ptr %gep.out, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 6: Coefficient clamping in merged bounds.
+;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. Members that lack a stride
+;; term have an implicit coefficient of 0, which must be included in the
+;; min/max computation via clamping.
+define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'coefficient_clamping_regression'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'coefficient_clamping_regression'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base + 0 (no stride offset -- this is the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; load at base - cdj
+ %negcdj = sub nsw i64 0, %cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; load at base - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 7: Predicated accesses are rejected from stencil merging.
+;; Models the dilateKernel pattern (llvm-test-suite MicroBenchmarks/ImageProcessing/Dilate):
+;; 3 loads at {-cdj, 0, +cdj} where the -cdj and +cdj loads are conditional.
+;; Predicated loads have overapproximated SCEV bounds; merging would widen
+;; them further, causing false runtime overlap detection.
+;; Both modes: 3 checks (groups stay separate), no predicates.
+define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c1, i1 %c2) {
+; MERGE-LABEL: 'predicated_access_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 3 checks: each temp group separately vs %out (not merged due to predication):
+; 4 groups: 1 for %out, 3 separate temp groups:
+entry:
+ %cmp = icmp sgt i64 %n, 0
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Unconditional load at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Conditional load at base + cdj (predicated block)
+ br i1 %c1, label %if.then1, label %if.end1
+
+if.then1:
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+ br label %if.end1
+
+if.end1:
+ %m1 = phi double [ %v1, %if.then1 ], [ %v0, %loop ]
+
+ ; Conditional load at base - cdj (predicated block)
+ %negcdj = sub nsw i64 0, %cdj
+ br i1 %c2, label %if.then2, label %if.end2
+
+if.then2:
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+ br label %if.end2
+
+if.end2:
+ %m2 = phi double [ %v2, %if.then2 ], [ %m1, %if.end1 ]
+
+ %s = fadd double %m1, %m2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 8: Write pointer in a group prevents stencil merging.
+;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
+;; All three share the same base %a (same DepSet). The write prevents merging.
+;; Both modes: groups stay separate.
+define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'write_in_group_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'write_in_group_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 5 checks: write group vs all others, %out vs the two read groups:
+; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
+; Both modes produce identical output (write prevents merging).
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Read at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Read at base + cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; Write at base + 2*cdj (same DepSet as reads — prevents merging)
+ %cdj2 = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+ %s = fadd double %v0, %v1
+ store double %s, ptr %p2, align 8
+
+ ; Store to %out (different DepSet)
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 9: Mixed constant offsets + runtime stride.
+;; 4 constant-offset loads at {-16, -8, +8, +16} + 2 runtime-stride loads
+;; at {-cdj, +cdj}. Store to %out.
+;; Without merge: standard grouping merges the 4 constant-offset loads into
+;; 1 group but the 2 runtime-stride loads remain separate -> 3 checks.
+;; With merge: all 6 loads form 1 merged group -> 1 check + 1 predicate.
+;; Merged Low includes -cdj, Merged High includes +cdj:
+define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; -16 (constant)
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ ; -8 (constant)
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ ; +8 (constant)
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ ; +16 (constant)
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ ; -cdj (runtime)
+ %negcdj = sub nsw i64 0, %cdj
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v4 = load double, ptr %p4, align 8
+
+ ; +cdj (runtime)
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 10: Shared stride predicate deduplication across two DepSets.
+;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
+;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
+;; should be emitted, not 2.
+;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 1 predicate.
+;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
+define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'shared_stride_predicate_dedup'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; MERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'shared_stride_predicate_dedup'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + %a) High: (-24 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + %b) High: (-24 + (8 * %n) + %b))
+; NOMERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + (-1 * %cdj) + %b))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+
+ ; Reads from %a at {-cdj, 0, +cdj}
+ %basea = getelementptr inbounds double, ptr %a, i64 %iv
+ %basea.i8 = bitcast ptr %basea to ptr
+ %negcdj = sub nsw i64 0, %cdj
+ %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %va0 = load double, ptr %pa0, align 8
+ %va1 = load double, ptr %basea, align 8
+ %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %va2 = load double, ptr %pa2, align 8
+
+ ; Reads from %b at {-cdj, 0, +cdj}
+ %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+ %baseb.i8 = bitcast ptr %baseb to ptr
+ %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %vb0 = load double, ptr %pb0, align 8
+ %vb1 = load double, ptr %baseb, align 8
+ %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %vb2 = load double, ptr %pb2, align 8
+
+ %s0 = fadd double %va0, %va1
+ %s1 = fadd double %s0, %va2
+ %s2 = fadd double %s1, %vb0
+ %s3 = fadd double %s2, %vb1
+ %s4 = fadd double %s3, %vb2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 11: Known-positive stride skips predicate emission.
+;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
+;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
+;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
+;; Without merge: 3 separate groups + 1 %out, 3 checks, no predicates.
+define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %cdj_in) {
+; MERGE-LABEL: 'known_positive_stride_no_predicate'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; MERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'known_positive_stride_no_predicate'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (1 smax %cdj_in) + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; NOMERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (-1 * (1 smax %cdj_in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base - stride
+ %negcdj = sub nsw i64 0, %cdj
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v0 = load double, ptr %p0, align 8
+
+ ; load at base
+ %v1 = load double, ptr %base, align 8
+
+ ; load at base + stride
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
>From 7dd1e0916ecfa3d96f7e3e5dbb6b853a2887e319 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 15:13:42 +0000
Subject: [PATCH 02/33] Use determenistic structure Rename test file
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 4 ++--
...affine-group-merging.ll => runtime-check-group-merging.ll} | 0
2 files changed, 2 insertions(+), 2 deletions(-)
rename llvm/test/Analysis/LoopAccessAnalysis/{affine-group-merging.ll => runtime-check-group-merging.ll} (100%)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index d4338090f7fe1..eaf3c329e1d9b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -834,7 +834,7 @@ void RuntimePointerChecking::groupChecks(
struct StencilDecomposition {
int64_t Constant = 0;
/// Map from loop-invariant stride SCEV to its integer coefficient.
- SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+ SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
};
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
@@ -1022,7 +1022,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
- SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+ SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
for (unsigned Idx : AllMembers) {
const SCEV *PtrStart = Pointers[Idx].Start;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
similarity index 100%
rename from llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
>From d6e6dffd38c9d18033bc23b82b41f759f52380f9 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 09:22:37 +0100
Subject: [PATCH 03/33] Address some comments (nit related and about the flag)
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 193 ++++++++----------
.../runtime-check-group-merging.ll | 174 ++++++++--------
2 files changed, 175 insertions(+), 192 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index eaf3c329e1d9b..914e0d223fe80 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -101,15 +101,15 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable merging of runtime check groups with stencil stride "
- "patterns (default = false)"),
+static cl::opt<bool> ForceStencilMerge(
+ "force-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Force merging of runtime check groups with stencil stride "
+ "patterns regardless of check count (default = false)"),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
- cl::desc("Trigger stencil group merging when runtime check count "
+ cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128)"),
cl::init(128));
@@ -839,8 +839,8 @@ struct StencilDecomposition {
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
-/// Relies on SCEV's canonical form: AddExpr operands are flattened,
-/// MulExpr has the constant operand first.
+/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
+/// MulExpr has the constant operand first when present.
/// Returns std::nullopt if the expression contains non-stencil terms.
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
@@ -849,21 +849,20 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
- Terms.append(Add->operands().begin(), Add->operands().end());
+ append_range(Terms, Add->operands());
else
Terms.push_back(Expr);
+ const SCEVConstant *C;
+ const SCEV *Stride;
for (const SCEV *Term : Terms) {
- if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ if (match(Term, m_SCEVConstant(C))) {
D.Constant += C->getAPInt().getSExtValue();
- } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
- // SCEV canonical form: constant is operand 0 in a MulExpr.
- if (Mul->getNumOperands() != 2)
+ } else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
+ // Canonical 2-operand pattern (constant * loop-invariant).
+ if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
- if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
- return std::nullopt;
- D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ D.Coefficients[Stride] += C->getAPInt().getSExtValue();
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
@@ -881,20 +880,30 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
- // Count the total checks from current grouping.
- unsigned TotalChecks = 0;
- for (unsigned I = 0; I < CheckingGroups.size(); ++I)
- for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
- if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
- ++TotalChecks;
-
- if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks <= threshold, skipping stencil merge\n");
- return;
+ // Stencil merging runs when either:
+ // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
+ // - the current check count exceeds the auto-trigger threshold, where the
+ // vectorizer would otherwise reject the loop for having too many runtime
+ // checks. In that case the merge can only improve things: at worst we
+ // decline to merge and behave as before.
+ if (!ForceStencilMerge) {
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(
+ dbgs() << "LAA: " << TotalChecks
+ << " checks > threshold, proceeding with stencil merge\n");
+ } else {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge forced via flag\n");
}
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks, proceeding with stencil merge\n");
// Group CheckingGroups by (DependencySetId, AliasSetId) pair.
// DependencySetId alone is not unique: it resets per alias set, so
@@ -920,47 +929,34 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Collect all member pointers across these groups.
SmallVector<unsigned, 8> AllMembers;
for (unsigned GI : GroupIndices)
- AllMembers.append(CheckingGroups[GI].Members.begin(),
- CheckingGroups[GI].Members.end());
-
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
- bool HasPredicatedAccess = false;
- for (unsigned Idx : AllMembers) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- if (auto *I = dyn_cast<Instruction>(PtrVal)) {
- if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT())) {
- HasPredicatedAccess = true;
- break;
- }
- }
- }
- if (HasPredicatedAccess) {
- LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
- << ") with predicated access\n");
- continue;
- }
+ append_range(AllMembers, CheckingGroups[GI].Members);
// Only merge read-only groups. Stencil patterns read an array at
// multiple offsets and write to a different array (different DepSet).
// Mixing reads and writes within a merged group complicates the cost
// model and doesn't match known stencil patterns.
- bool HasWriteAccess = false;
- for (unsigned Idx : AllMembers) {
- if (Pointers[Idx].IsWritePtr) {
- HasWriteAccess = true;
- break;
- }
- }
- if (HasWriteAccess) {
+ if (any_of(AllMembers,
+ [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with write access\n");
continue;
}
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ auto *I = dyn_cast<Instruction>(PtrVal);
+ return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
<< ") with " << AllMembers.size() << " members, base: "
<< *Pointers[AllMembers[0]].Start << "\n");
@@ -985,59 +981,49 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// ranges and testing for zero, which lets SCEV simplify algebraically
// even when the individual range SCEVs aren't pointer-identical.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
- bool RangeMismatch = false;
- for (unsigned Idx : AllMembers) {
- const SCEV *Range =
- SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different access range, skipping DepSet\n");
- RangeMismatch = true;
- break;
- }
- }
- if (RangeMismatch)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ return Range != BaseRange &&
+ !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
+ "skipping DepSet\n");
continue;
-
- bool DecompOk = true;
+ }
// Verify all members have the same recurrence step w.r.t. the analyzed
// loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
// correct when every member's High-Low range equals BaseHigh - BaseLow.
// Different steps (e.g., 8 vs 16) produce different ranges.
- for (unsigned Idx : AllMembers) {
- const SCEV *Step = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
- if (AR->getLoop() == &L)
- Step = AR->getStepRecurrence(*SE);
-
- if (Step != BaseStep) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different step, skipping DepSet\n");
- DecompOk = false;
- break;
- }
- }
- if (!DecompOk)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+ return Step != BaseStep;
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
+ "skipping DepSet\n");
continue;
+ }
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
- for (unsigned Idx : AllMembers) {
- const SCEV *PtrStart = Pointers[Idx].Start;
-
- const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
- if (isa<SCEVCouldNotCompute>(LowOffset)) {
- DecompOk = false;
- break;
- }
+ // Decompose one member's offset (relative to BaseLow) and fold its
+ // constant and per-stride coefficients into the running bounding box.
+ // Returns false if the offset is not in stencil form (so the whole
+ // DepSet is skipped).
+ const auto AccumulateOffset = [&](unsigned Idx) -> bool {
+ const SCEV *LowOffset = SE->getMinusSCEV(Pointers[Idx].Start, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset))
+ return false;
auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
if (!DLow) {
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< " NOT decomposable: " << *LowOffset << "\n");
- DecompOk = false;
- break;
+ return false;
}
CMin = std::min(CMin, DLow->Constant);
@@ -1059,9 +1045,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
- }
+ return true;
+ };
- if (!DecompOk)
+ if (!all_of(AllMembers, AccumulateOffset))
continue;
// The base member (offset = 0) implicitly has coefficient 0 for every
@@ -1153,8 +1140,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
- for (unsigned I = 1; I < AllMembers.size(); ++I)
- CandidateGroup.Members.push_back(AllMembers[I]);
+ append_range(CandidateGroup.Members, drop_begin(AllMembers));
for (unsigned GI : GroupIndices)
CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
@@ -1169,8 +1155,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< *Stride << "\n");
}
- for (unsigned GI : GroupIndices)
- MergedGroupIndices.insert(GI);
+ MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
NewMergedGroups.push_back(std::move(CandidateGroup));
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index fb6b116a5b300..c6b9507d84752 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -18,12 +18,12 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -52,32 +52,32 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 5:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -105,6 +105,7 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -112,29 +113,28 @@ entry:
loop:
%iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
%neg3cdj = mul nsw i64 %cdj, -3
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
%v0 = load double, ptr %p0, align 8
%neg2cdj = mul nsw i64 %cdj, -2
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v1 = load double, ptr %p1, align 8
%negcdj = sub nsw i64 0, %cdj
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v2 = load double, ptr %p2, align 8
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v3 = load double, ptr %p3, align 8
%pos2cdj = mul nsw i64 %cdj, 2
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
%v4 = load double, ptr %p4, align 8
%pos3cdj = mul nsw i64 %cdj, 3
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -172,11 +172,11 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -205,27 +205,27 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -250,6 +250,7 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 8
br i1 %cmp, label %loop, label %exit
@@ -257,19 +258,18 @@ entry:
loop:
%iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %vol, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; +8 + cdj - cdk
%off0a = add nsw i64 8, %cdj
%negcdk = sub nsw i64 0, %cdk
%off0 = add nsw i64 %off0a, %negcdk
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
%v0 = load double, ptr %p0, align 8
; -16 - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
%off1 = add nsw i64 -16, %neg2cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
%v1 = load double, ptr %p1, align 8
; +24 + 3*cdj + 2*cdk
@@ -277,7 +277,7 @@ loop:
%pos2cdk = mul nsw i64 %cdk, 2
%off2a = add nsw i64 24, %pos3cdj
%off2 = add nsw i64 %off2a, %pos2cdk
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
%v2 = load double, ptr %p2, align 8
; -8 + 2*cdj - 3*cdk
@@ -285,14 +285,14 @@ loop:
%neg3cdk = mul nsw i64 %cdk, -3
%off3a = add nsw i64 -8, %pos2cdj
%off3 = add nsw i64 %off3a, %neg3cdk
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
%v3 = load double, ptr %p3, align 8
; +32 - 4*cdj + cdk
%neg4cdj = mul nsw i64 %cdj, -4
%off4a = add nsw i64 32, %neg4cdj
%off4 = add nsw i64 %off4a, %cdk
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
%v4 = load double, ptr %p4, align 8
%s0 = fadd double %v0, %v1
@@ -431,7 +431,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -462,7 +462,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -486,9 +486,8 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v0 = load double, ptr %p0, align 8
%v1 = load double, ptr %base, align 8
@@ -629,8 +628,8 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
@@ -657,12 +656,12 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -686,6 +685,7 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -693,19 +693,18 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base + 0 (no stride offset -- this is the base member)
%v0 = load double, ptr %base, align 8
; load at base - cdj
%negcdj = sub nsw i64 0, %cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v1 = load double, ptr %p1, align 8
; load at base - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
@@ -1019,12 +1018,12 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1053,20 +1052,20 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1088,6 +1087,7 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -1095,31 +1095,30 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; -16 (constant)
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
%v0 = load double, ptr %p0, align 8
; -8 (constant)
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
%v1 = load double, ptr %p1, align 8
; +8 (constant)
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
%v2 = load double, ptr %p2, align 8
; +16 (constant)
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
%v3 = load double, ptr %p3, align 8
; -cdj (runtime)
%negcdj = sub nsw i64 0, %cdj
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v4 = load double, ptr %p4, align 8
; +cdj (runtime)
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -1157,16 +1156,16 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; MERGE-NEXT: Check 1:
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
-; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1197,7 +1196,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1207,12 +1206,12 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1222,7 +1221,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1250,6 +1249,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -1259,21 +1259,19 @@ loop:
; Reads from %a at {-cdj, 0, +cdj}
%basea = getelementptr inbounds double, ptr %a, i64 %iv
- %basea.i8 = bitcast ptr %basea to ptr
%negcdj = sub nsw i64 0, %cdj
- %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
%va0 = load double, ptr %pa0, align 8
%va1 = load double, ptr %basea, align 8
- %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
%va2 = load double, ptr %pa2, align 8
; Reads from %b at {-cdj, 0, +cdj}
%baseb = getelementptr inbounds double, ptr %b, i64 %iv
- %baseb.i8 = bitcast ptr %baseb to ptr
- %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
%vb0 = load double, ptr %pb0, align 8
%vb1 = load double, ptr %baseb, align 8
- %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
%vb2 = load double, ptr %pb2, align 8
%s0 = fadd double %va0, %va1
@@ -1310,9 +1308,9 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1337,7 +1335,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1347,7 +1345,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1366,6 +1364,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
%cmp = icmp sgt i64 %n, 4
@@ -1374,18 +1373,17 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base - stride
%negcdj = sub nsw i64 0, %cdj
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v0 = load double, ptr %p0, align 8
; load at base
%v1 = load double, ptr %base, align 8
; load at base + stride
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
>From 51288900c3ca8e7473be29e853a468bd88772445 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 13:36:21 +0100
Subject: [PATCH 04/33] Address int64_t comment
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 914e0d223fe80..c824ed6e0368a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -841,11 +841,20 @@ struct StencilDecomposition {
/// strides: C + a1*s1 + a2*s2 + ...
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
-/// Returns std::nullopt if the expression contains non-stencil terms.
+/// Returns std::nullopt if the expression contains non-stencil terms or any
+/// SCEV constant doesn't fit in int64_t (we commit to the signed
+/// interpretation; values that need more than 64 significant bits are
+/// out of scope).
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
+ if (V.getSignificantBits() > 64)
+ return std::nullopt;
+ return V.getSExtValue();
+ };
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -857,12 +866,18 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- D.Constant += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- D.Coefficients[Stride] += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Coefficients[Stride] += *V;
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
>From 2470787c4e385462a2ede635b42ce1505a813aa4 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:35:25 +0100
Subject: [PATCH 05/33] Add 128-addrspace test and replaced lambda with
existing trySExtValue
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 10 +-
.../runtime-check-group-merging-i128.ll | 117 ++++++++++++++++++
2 files changed, 119 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index c824ed6e0368a..7f403a29a7d88 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -849,12 +849,6 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
- auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
- if (V.getSignificantBits() > 64)
- return std::nullopt;
- return V.getSExtValue();
- };
-
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -866,7 +860,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Constant += *V;
@@ -874,7 +868,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
new file mode 100644
index 0000000000000..dfb56233e91d5
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+target datalayout = "e-p:128:128"
+
+define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj) {
+; MERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %ivx = zext i64 %iv to i128
+
+ %o0 = mul i128 %cdj, 1
+ %t0 = add i128 %o0, %ivx
+ %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+ %v0 = load i8, ptr %p0
+
+ %o1 = mul i128 %cdj, 2
+ %t1 = add i128 %o1, %ivx
+ %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+ %v1 = load i8, ptr %p1
+
+ %o2 = mul i128 %cdj, 36893488147419103232 ; 2^65, needs 67 significant bits
+ %t2 = add i128 %o2, %ivx
+ %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+ %v2 = load i8, ptr %p2
+
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+ store i8 %s, ptr %op
+
+ %iv.next = add i64 %iv, 1
+ %c = icmp slt i64 %iv.next, %n
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
>From fae864b9b3c8a2f7b2754e86ac351820e3bf8b03 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:50:11 +0100
Subject: [PATCH 06/33] Add -enable-stencil-runtime-check-merge switch
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 7f403a29a7d88..820755b589487 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -101,16 +101,26 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable stencil-pattern merging of runtime memory checks "
+ "(default = false, intended to be flipped to true in a "
+ "follow-up patch once the algorithm has settled)"),
+ cl::init(false));
+
static cl::opt<bool> ForceStencilMerge(
"force-stencil-runtime-check-merge", cl::Hidden,
cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false)"),
+ "patterns regardless of check count (default = false). "
+ "Implies enabling the merge regardless of "
+ "-enable-stencil-runtime-check-merge."),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
- "exceeds this threshold (default = 128)"),
+ "exceeds this threshold (default = 128). Only used when "
+ "-enable-stencil-runtime-check-merge is true."),
cl::init(128));
/// Maximum SIMD width.
@@ -890,12 +900,17 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
- // - the current check count exceeds the auto-trigger threshold, where the
+ // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
+ // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
if (!ForceStencilMerge) {
+ if (!EnableStencilMerge) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
+ return;
+ }
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 9a2fc349dfc0044259758f7b4a10973314c10b3b Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:20:36 +0000
Subject: [PATCH 07/33] Fix test
---
.../runtime-check-group-merging.ll | 79 +++++++++----------
1 file changed, 36 insertions(+), 43 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index c6b9507d84752..822be3812b3a6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -507,41 +507,38 @@ exit:
}
-;; Test 5: Invariant + strided accesses to same base -> different steps.
-;; A strided store {%a,+,8} and an invariant store to %a have different
-;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Test 5: Invariant + strided reads from same base -> different access ranges.
+;; A strided read {%a,+,8} and an invariant read from %a have different
+;; access ranges (8*n vs 8), so merging must NOT combine them.
+;; Both reads are in the same DepSet (same underlying object, both reads),
+;; so they pass the write-access guard and reach the access-range check.
;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; MERGE-LABEL: 'different_steps_no_merge'
; MERGE-NEXT: loop:
-; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Memory dependences are safe with run-time checks
; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Unknown:
-; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; MERGE-EMPTY:
; MERGE-NEXT: Run-time memory checks:
; MERGE-NEXT: Check 0:
; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: ptr %a
-; MERGE-NEXT: Against group GRP2:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: Against group GRP1:
; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %a
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: %a
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
@@ -550,43 +547,38 @@ define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
;
; NOMERGE-LABEL: 'different_steps_no_merge'
; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Unknown:
-; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; NOMERGE-EMPTY:
; NOMERGE-NEXT: Run-time memory checks:
; NOMERGE-NEXT: Check 0:
; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: ptr %a
-; NOMERGE-NEXT: Against group GRP2:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: Against group GRP1:
; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %a
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
;
-; GRP0: invariant store to %a (single address, step=0):
-; GRP1: strided store to %a (step=8, different range from GRP0):
-; GRP2: load from %out:
+; GRP0: store to %out (write, different DepSet):
+; GRP1: strided read from %a (step=8):
+; GRP2: invariant read from %a (step=0, different from GRP1):
entry:
%cmp = icmp sgt i64 %n, 2
br i1 %cmp, label %loop, label %exit
@@ -594,16 +586,17 @@ entry:
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- ; Strided store: {%a,+,8}
- %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
- store double 1.0, ptr %gep.a, align 8
+ ; Invariant read from %a (step = 0, different from strided)
+ %v.inv = load double, ptr %a, align 8
- ; Invariant store to %a (step = 0, different range)
- store double 2.0, ptr %a, align 8
+ ; Strided read: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ %v.strided = load double, ptr %gep.a, align 8
- ; Read from %out
+ ; Store to %out (different DepSet, triggers runtime checks)
+ %sum = fadd double %v.strided, %v.inv
%gep.out = getelementptr inbounds double, ptr %out, i64 %iv
- %v = load double, ptr %gep.out, align 8
+ store double %sum, ptr %gep.out, align 8
%iv.next = add nuw nsw i64 %iv, 1
%cond = icmp slt i64 %iv.next, %n
>From cccea18ddf620ec2a3e1a73ac98ad9857c450c56 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:40:06 +0000
Subject: [PATCH 08/33] Refactor flags
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 46 ++++++++++++------------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 820755b589487..cf85b32068333 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -101,26 +101,27 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable stencil-pattern merging of runtime memory checks "
- "(default = false, intended to be flipped to true in a "
- "follow-up patch once the algorithm has settled)"),
- cl::init(false));
-
-static cl::opt<bool> ForceStencilMerge(
- "force-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false). "
- "Implies enabling the merge regardless of "
- "-enable-stencil-runtime-check-merge."),
- cl::init(false));
+enum class StencilMergePolicy { Off, Auto, Force };
+
+static cl::opt<StencilMergePolicy> StencilMerge(
+ "stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Control stencil-pattern merging of runtime memory checks"),
+ cl::init(StencilMergePolicy::Off),
+ cl::values(
+ clEnumValN(StencilMergePolicy::Off, "off",
+ "Disable stencil merge (default)"),
+ clEnumValN(StencilMergePolicy::Auto, "auto",
+ "Enable stencil merge when runtime check count exceeds "
+ "the threshold"),
+ clEnumValN(StencilMergePolicy::Force, "force",
+ "Always attempt stencil merge regardless of check "
+ "count")));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128). Only used when "
- "-enable-stencil-runtime-check-merge is true."),
+ "-stencil-runtime-check-merge is auto."),
cl::init(128));
/// Maximum SIMD width.
@@ -900,17 +901,18 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
- // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
+ // - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
// current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
- if (!ForceStencilMerge) {
- if (!EnableStencilMerge) {
- LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
- return;
- }
+ if (StencilMerge == StencilMergePolicy::Off) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled\n");
+ return;
+ }
+
+ if (StencilMerge == StencilMergePolicy::Auto) {
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 69c523c93a2707602b8ea17e45f12622e0d4d761 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 3 Jun 2026 15:29:11 +0100
Subject: [PATCH 09/33] Fix predicated-access check, turn some conditions into
asserts, add tests
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 52 ++-
.../runtime-check-group-merging-auto.ll | 150 +++++++++
.../runtime-check-group-merging-i128.ll | 20 +-
.../runtime-check-group-merging.ll | 314 +++++++++++++++++-
4 files changed, 506 insertions(+), 30 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index cf85b32068333..98a4713b9d94c 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -860,6 +860,11 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ // The only caller passes a difference of two pointer Starts, and a Start is
+ // always loop-invariant (getStartAndEndForAccess asserts it). So Expr is
+ // loop-invariant and every term below is loop-invariant too.
+ assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -876,17 +881,14 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return std::nullopt;
D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
- // Canonical 2-operand pattern (constant * loop-invariant).
- if (!SE.isLoopInvariant(Stride, &L))
- return std::nullopt;
+ assert(SE.isLoopInvariant(Stride, &L) && "stride must be loop-invariant");
auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
- } else if (SE.isLoopInvariant(Term, &L)) {
- D.Coefficients[Term] += 1;
} else {
- return std::nullopt;
+ assert(SE.isLoopInvariant(Term, &L) && "term must be loop-invariant");
+ D.Coefficients[Term] += 1;
}
}
return D;
@@ -968,15 +970,26 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
+ // We do not allow predicated accesses. They may result in overestimation
+ // of the boundaries. Imagine a stencil access where we must skip some first
+ // or last iterations because the stencil does not fit the array and has to
+ // go from 1..N-2 although the array is [0..N-1] (for example the dilate
+ // kernel from llvm-test-suite ImageProcessing/Dilate, which reads the
+ // neighbours of every pixel and guards the borders with conditions).
+ // Merging such bounds would widen the already overestimated range further.
+ // This is not necessary in StencilMergePolicy::Auto mode, but skipping it
+ // in StencilMergePolicy::Force mode causes a regression on that benchmark.
+ //
+ // Look at the block of the actual load/store, not of the pointer: a
+ // loop-invariant address is computed in the preheader, outside the loop.
if (any_of(AllMembers, [&](unsigned Idx) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- auto *I = dyn_cast<Instruction>(PtrVal);
- return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT());
+ const PointerInfo &P = Pointers[Idx];
+ return any_of(
+ DC.getInstructionsForAccess(P.PointerValue, P.IsWritePtr),
+ [&](Instruction *I) {
+ return LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ });
})) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with predicated access\n");
@@ -1018,10 +1031,13 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Verify all members have the same recurrence step w.r.t. the analyzed
- // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
- // correct when every member's High-Low range equals BaseHigh - BaseLow.
- // Different steps (e.g., 8 vs 16) produce different ranges.
+ // Require all members to have the same recurrence step. Equal ranges
+ // (checked above) are what the merged bounds actually need, and a different
+ // step usually means a different range. But ranges can be equal by accident
+ // - e.g. an invariant access whose range matches the stride, or a loop with
+ // a single iteration. The base member is picked arbitrarily, so together
+ // with the BaseStep check above this keeps the decision the same no matter
+ // which member comes first: we only merge recurrences with one common step.
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Step = nullptr;
if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
new file mode 100644
index 0000000000000..6855e80d4cffa
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1000 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
+
+define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; AUTOMERGE-LABEL: 'stencil_auto_threshold'
+; AUTOMERGE-NEXT: loop:
+; AUTOMERGE-NEXT: Memory dependences are safe with run-time checks
+; AUTOMERGE-NEXT: Dependences:
+; AUTOMERGE-NEXT: Run-time memory checks:
+; AUTOMERGE-NEXT: Check 0:
+; AUTOMERGE-NEXT: Comparing group GRP0:
+; AUTOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOMERGE-NEXT: Against group GRP1:
+; AUTOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOMERGE-NEXT: Grouped accesses:
+; AUTOMERGE-NEXT: Group GRP0:
+; AUTOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOMERGE-NEXT: Group GRP1:
+; AUTOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOMERGE-NEXT: SCEV assumptions:
+; AUTOMERGE-NEXT: Compare predicate: %cdj sgt) 0
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Expressions re-written:
+;
+; AUTOSKIP-LABEL: 'stencil_auto_threshold'
+; AUTOSKIP-NEXT: loop:
+; AUTOSKIP-NEXT: Memory dependences are safe with run-time checks
+; AUTOSKIP-NEXT: Dependences:
+; AUTOSKIP-NEXT: Run-time memory checks:
+; AUTOSKIP-NEXT: Check 0:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP1:
+; AUTOSKIP-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOSKIP-NEXT: Check 1:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP2:
+; AUTOSKIP-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOSKIP-NEXT: Check 2:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP3:
+; AUTOSKIP-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOSKIP-NEXT: Check 3:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP4:
+; AUTOSKIP-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOSKIP-NEXT: Check 4:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP5:
+; AUTOSKIP-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOSKIP-NEXT: Check 5:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP6:
+; AUTOSKIP-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOSKIP-NEXT: Grouped accesses:
+; AUTOSKIP-NEXT: Group GRP0:
+; AUTOSKIP-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOSKIP-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOSKIP-NEXT: Group GRP1:
+; AUTOSKIP-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP2:
+; AUTOSKIP-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP3:
+; AUTOSKIP-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; AUTOSKIP-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP4:
+; AUTOSKIP-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP5:
+; AUTOSKIP-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP6:
+; AUTOSKIP-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOSKIP-NEXT: SCEV assumptions:
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
index dfb56233e91d5..0799360bf346e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
target datalayout = "e-p:128:128"
@@ -27,16 +27,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -66,16 +66,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 822be3812b3a6..3ef1bd18a3ab2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -1394,4 +1394,314 @@ exit:
ret void
}
+;; Test 12: loop-invariant pointer is computed in the preheader.
+;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
+;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
+;; read, so we reach the predicated-access check. We must look at the load,
+;; which is in the loop, not at the pointer, which is in the preheader -
+;; otherwise we crash.
+define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'invariant_pointer_in_preheader'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (64 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: (%cdj + %a)
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'invariant_pointer_in_preheader'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (64 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: (%cdj + %a)
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+ %sv = load i8, ptr %sp
+ %sv64 = zext i8 %sv to i64
+ %iv8 = load i64, ptr %inv.ptr
+ %sum = add i64 %sv64, %iv8
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+
+;; Test 13: predicated access whose address is computed in the header.
+;; The stride GEPs are in the header, which dominates the latch, but the loads
+;; are in a conditional block guarded by %c. So the accesses are predicated and
+;; must not be merged. We must look at the load block, which is conditional,
+;; not at the GEP block, which is the header. Both modes: groups stay separate.
+define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c) {
+; MERGE-LABEL: 'predicated_access_hoisted_address'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; MERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP4:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_hoisted_address'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+ %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+ %v0 = load i64, ptr %p0
+ %i1 = add i64 %iv, %cdj
+ %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+ %i2 = sub i64 %iv, %cdj
+ %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+ %i3 = add i64 %i1, %cdj
+ %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+ br i1 %c, label %then, label %backedge
+then:
+ %v1 = load i64, ptr %p1
+ %v2 = load i64, ptr %p2
+ %v3 = load i64, ptr %p3
+ %t = add i64 %v1, %v2
+ %u = add i64 %t, %v3
+ br label %backedge
+backedge:
+ %vp = phi i64 [ %u, %then ], [ 0, %loop ]
+ %sum = add i64 %v0, %vp
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+;; Test 14: equal access ranges but different recurrence steps -> no merge.
+;; In a single-iteration loop every access spans just its element size, so the
+;; ranges are equal even though the steps differ (8 vs 16). The access-range
+;; check passes here, so the step check is the guard that stops the merge. Both
+;; modes: groups stay separate.
+define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'equal_range_different_step_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (8 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'equal_range_different_step_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (8 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %s8 = mul i64 %iv, 8
+ %b0 = getelementptr inbounds i8, ptr %a, i64 %s8
+ %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+ %v0 = load i64, ptr %p0
+ %s16 = mul i64 %iv, 16
+ %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+ %v1 = load i64, ptr %p1
+ %sum = add i64 %v0, %v1
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 614c29f83755912b5002707691a4771e7e4066d0 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 10:41:15 +0000
Subject: [PATCH 10/33] Add SCEVCouldNotCompute test
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 26 ++++--
.../runtime-check-group-merging.ll | 85 +++++++++++++++++++
2 files changed, 103 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 98a4713b9d94c..8e837056545a0 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1015,19 +1015,29 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
// Verify all members have the same access range (End - Start).
- // MergedHigh = BaseHigh + max_offsets is only correct when every
- // member's range equals BaseHigh - BaseLow. We check by subtracting
- // ranges and testing for zero, which lets SCEV simplify algebraically
- // even when the individual range SCEVs aren't pointer-identical.
+ // MergedHigh = BaseHigh + max_offsets is only correct when every member's
+ // range equals BaseHigh - BaseLow. Compute the ranges first and compare
+ // them by subtracting, so algebraically equal but non-identical SCEVs still
+ // match. Bail out if any subtraction produces SCEVCouldNotCompute.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ if (isa<SCEVCouldNotCompute>(BaseRange)) {
+ LLVM_DEBUG(dbgs() << "LAA: Base access range not computable, "
+ "skipping DepSet\n");
+ continue;
+ }
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Range =
SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- return Range != BaseRange &&
- !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ if (isa<SCEVCouldNotCompute>(Range))
+ return true;
+ if (Range == BaseRange)
+ return false;
+ const SCEV *RangeDiff = SE->getMinusSCEV(Range, BaseRange);
+ return isa<SCEVCouldNotCompute>(RangeDiff) || !RangeDiff->isZero();
})) {
- LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
- "skipping DepSet\n");
+ LLVM_DEBUG(dbgs()
+ << "LAA: Member with different or not computable access "
+ "range, skipping DepSet\n");
continue;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 3ef1bd18a3ab2..fada2d09d0509 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1704,4 +1704,89 @@ exit:
ret void
}
+;; Test 15: %gep Start/End expressions are min/max expressions.
+;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
+define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
+; MERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %p
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + %q))
+; MERGE-NEXT: Member: %q
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; MERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %p High: (1 + %p))
+; MERGE-NEXT: Member: %p
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %p
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + %q))
+; NOMERGE-NEXT: Member: %q
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; NOMERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %p High: (1 + %p))
+; NOMERGE-NEXT: Member: %p
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %step = add i64 %m, 1
+ br label %loop
+
+loop:
+ %off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
+ %i = phi i1 [ false, %entry ], [ true, %loop ]
+ load i8, ptr %p
+ %gep = getelementptr i8, ptr %p, i64 %off
+ load i8, ptr %gep
+ store i8 0, ptr %q
+ %off.next = add i64 %off, %step
+ br i1 %i, label %done, label %loop
+
+done:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 2c7a639cd640a5afbba6666e39644d0eb678188a Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 11:11:00 +0000
Subject: [PATCH 11/33] Add high level explanation to mergeStencilGroups like
in groupChecks
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 8e837056545a0..fd3692ae64054 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -902,6 +902,28 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
+ // We try to merge groups produced by groupChecks when their pointers follow
+ // a stencil access pattern. groupChecks intentionally only merges pointers
+ // whose min/max bounds differ by constants, because that keeps each runtime
+ // check precise. Stencil kernels often read the same underlying object at
+ // several loop-invariant stride offsets, so those groups remain separate and
+ // can produce too many checks. Here we trade some precision for fewer
+ // checks by replacing a set of same-dependence, same-alias read groups with
+ // one conservative bounding group.
+ //
+ // We use the following algorithm to construct a merged stencil group:
+ // - collect checking groups that share both DependencySetId and AliasSetId;
+ // - reject groups with writes, predicated accesses, different access
+ // ranges, or different recurrence steps;
+ // - use one member as the base and decompose each other member's offset
+ // from that base as C + sum(Coeff[Stride] * Stride), where Stride is
+ // loop-invariant;
+ // - build one bounding range by taking the min/max constant offset and the
+ // min/max coefficient for each stride, adding predicates for strides
+ // that are not already known positive;
+ // - commit the merge only if the local cost model reduces the number of
+ // checks after accounting for any new predicates.
+
// Stencil merging runs when either:
// - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
// - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
>From a61e83f4bc192a39e426d6e28509731b2daba44a Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 12:26:54 +0000
Subject: [PATCH 12/33] Clang format
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index fd3692ae64054..84cb260c56c05 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1057,8 +1057,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
const SCEV *RangeDiff = SE->getMinusSCEV(Range, BaseRange);
return isa<SCEVCouldNotCompute>(RangeDiff) || !RangeDiff->isZero();
})) {
- LLVM_DEBUG(dbgs()
- << "LAA: Member with different or not computable access "
+ LLVM_DEBUG(
+ dbgs() << "LAA: Member with different or not computable access "
"range, skipping DepSet\n");
continue;
}
>From b629d30c9e5b9f02e258cbfd5cbcee78c99ccda5 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 18 Jun 2026 16:29:25 +0100
Subject: [PATCH 13/33] Address comments
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 264 ++++++++++++------
.../runtime-check-group-merging.ll | 107 +++++++
2 files changed, 290 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 84cb260c56c05..79ce2dca7602a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -124,6 +124,13 @@ static cl::opt<unsigned> StencilMergeCheckThreshold(
"-stencil-runtime-check-merge is auto."),
cl::init(128));
+static cl::opt<unsigned> StencilMergeMaxGroups(
+ "stencil-merge-max-groups", cl::Hidden,
+ cl::desc(
+ "Skip stencil group merging when the number of runtime checking groups "
+ "exceeds this limit, to bound compile time (default =4096)."),
+ cl::init(4096));
+
/// Maximum SIMD width.
const unsigned VectorizerParams::MaxVectorWidth = 64;
@@ -850,6 +857,11 @@ struct StencilDecomposition {
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
+/// \p Expr is the difference of two access "Start" SCEVs (Start_member -
+/// Start_base). A "Start" is the low bound of a memory access range as computed
+/// by getStartAndEndForAccess: the address of the first byte the access can
+/// touch. The result describes where one member's range sits relative to the
+/// base member's range.
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
/// Returns std::nullopt if the expression contains non-stencil terms or any
@@ -860,9 +872,9 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
- // The only caller passes a difference of two pointer Starts, and a Start is
- // always loop-invariant (getStartAndEndForAccess asserts it). So Expr is
- // loop-invariant and every term below is loop-invariant too.
+ // A "Start" is always loop-invariant (getStartAndEndForAccess asserts it), so
+ // the difference Expr passed in by the caller is loop-invariant too, and so
+ // is every additive term we pull out of it below.
assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
// Collect top-level additive terms.
@@ -927,15 +939,28 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Stencil merging runs when either:
// - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
// - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
- // current check count exceeds the auto-trigger threshold, where the
- // vectorizer would otherwise reject the loop for having too many runtime
- // checks. In that case the merge can only improve things: at worst we
- // decline to merge and behave as before.
+ // current check count exceeds the auto-trigger threshold, which defaults
+ // to the vectorizer's own runtime-check cutoff
+ // (-vectorize-memory-check-threshold). Above it the vectorizer would
+ // otherwise reject the loop for having too many runtime checks. In that
+ // case the merge can only improve things: at worst we decline to merge
+ // and behave as before.
if (StencilMerge == StencilMergePolicy::Off) {
LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled\n");
return;
}
+ // For each checking group this pass decomposes each member's offset into
+ // stencil form and builds a merged bounding box. That extra SCEV work adds up
+ // on a loop with very many groups, so bail out above a configurable limit as
+ // a safety net against pathological inputs.
+ if (CheckingGroups.size() > StencilMergeMaxGroups) {
+ LLVM_DEBUG(
+ dbgs() << "LAA: " << CheckingGroups.size()
+ << " groups exceeds stencil-merge-max-groups, skipping\n");
+ return;
+ }
+
if (StencilMerge == StencilMergePolicy::Auto) {
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
@@ -976,17 +1001,23 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (GroupIndices.size() < 2)
continue;
- // Collect all member pointers across these groups.
+ // Collect all member pointers across these groups. Only merge read-only
+ // groups: stencil patterns read an array at multiple offsets and write to a
+ // different array (a different DepSet). Mixing reads and writes within a
+ // merged group complicates the cost model and doesn't match known stencil
+ // patterns, so stop and skip the whole DepSet as soon as we see a write.
SmallVector<unsigned, 8> AllMembers;
- for (unsigned GI : GroupIndices)
- append_range(AllMembers, CheckingGroups[GI].Members);
-
- // Only merge read-only groups. Stencil patterns read an array at
- // multiple offsets and write to a different array (different DepSet).
- // Mixing reads and writes within a merged group complicates the cost
- // model and doesn't match known stencil patterns.
- if (any_of(AllMembers,
- [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
+ bool HasWrite = false;
+ for (unsigned GI : GroupIndices) {
+ ArrayRef<unsigned> Members = CheckingGroups[GI].Members;
+ if (any_of(Members,
+ [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
+ HasWrite = true;
+ break;
+ }
+ append_range(AllMembers, Members);
+ }
+ if (HasWrite) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with write access\n");
continue;
@@ -1018,29 +1049,37 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
- << ") with " << AllMembers.size() << " members, base: "
- << *Pointers[AllMembers[0]].Start << "\n");
-
- // Use the first member as the reference for decomposition. All offsets
- // are computed relative to BaseLow, and MergedHigh is built from
+ // Use the first member as the reference for decomposition. All offsets are
+ // computed relative to BaseLow, and the merged upper bound is built from
// BaseHigh (see merged-bounds computation below).
- const SCEV *BaseLow = Pointers[AllMembers[0]].Start;
- const SCEV *BaseHigh = Pointers[AllMembers[0]].End;
+ unsigned Member0 = AllMembers[0];
+ const SCEV *BaseLow = Pointers[Member0].Start;
+ const SCEV *BaseHigh = Pointers[Member0].End;
+
+ LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
+ << ") with " << AllMembers.size()
+ << " members, base: " << *BaseLow << "\n");
- const SCEV *BaseStep = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[AllMembers[0]].Expr))
- if (AR->getLoop() == &L)
- BaseStep = AR->getStepRecurrence(*SE);
+ auto GetStepForPointer = [&](unsigned Idx) -> const SCEV * {
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ return AR->getStepRecurrence(*SE);
+ return nullptr;
+ };
+ const SCEV *BaseStep = GetStepForPointer(Member0);
if (!BaseStep)
continue;
- // Verify all members have the same access range (End - Start).
- // MergedHigh = BaseHigh + max_offsets is only correct when every member's
- // range equals BaseHigh - BaseLow. Compute the ranges first and compare
- // them by subtracting, so algebraically equal but non-identical SCEVs still
- // match. Bail out if any subtraction produces SCEVCouldNotCompute.
+ // Verify all members have the same access range (End - Start). The merged
+ // group gets a single upper bound (MergedHigh, built further below) of the
+ // form BaseHigh + max_offsets. That is only correct when every member's
+ // range equals BaseHigh - BaseLow; otherwise a member with a larger range
+ // could reach past MergedHigh.
+ // Compare each member's range (End - Start) and test Range - BaseRange ==
+ // 0, rather than Range == BaseRange, so algebraically equal but
+ // non-identical SCEVs still match. Bail out if any subtraction produces
+ // SCEVCouldNotCompute.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
if (isa<SCEVCouldNotCompute>(BaseRange)) {
LLVM_DEBUG(dbgs() << "LAA: Base access range not computable, "
@@ -1071,18 +1110,17 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// with the BaseStep check above this keeps the decision the same no matter
// which member comes first: we only merge recurrences with one common step.
if (any_of(AllMembers, [&](unsigned Idx) {
- const SCEV *Step = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
- if (AR->getLoop() == &L)
- Step = AR->getStepRecurrence(*SE);
- return Step != BaseStep;
+ return GetStepForPointer(Idx) != BaseStep;
})) {
LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
"skipping DepSet\n");
continue;
}
+ // Per-stride min/max coefficient seen across all members, and the min/max
+ // of the constant part of the offset. Together they describe the bounding
+ // box.
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
- int64_t CMin = 0, CMax = 0;
+ int64_t MinConstOffset = 0, MaxConstOffset = 0;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
// Decompose one member's offset (relative to BaseLow) and fold its
@@ -1100,9 +1138,12 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return false;
}
- CMin = std::min(CMin, DLow->Constant);
- CMax = std::max(CMax, DLow->Constant);
+ MinConstOffset = std::min(MinConstOffset, DLow->Constant);
+ MaxConstOffset = std::max(MaxConstOffset, DLow->Constant);
+ // Fold this member's coefficient for each stride into the running min and
+ // max for that stride. MinCoeff and MaxCoeff are kept in lock-step: a
+ // stride is always present in both or neither.
for (const auto &[Stride, Coeff] : DLow->Coefficients) {
auto MinIt = MinCoeff.find(Stride);
if (MinIt == MinCoeff.end()) {
@@ -1110,14 +1151,16 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
MaxCoeff[Stride] = Coeff;
} else {
MinIt->second = std::min(MinIt->second, Coeff);
- MaxCoeff[Stride] = std::max(MaxCoeff[Stride], Coeff);
+ auto MaxIt = MaxCoeff.find(Stride);
+ MaxIt->second = std::max(MaxIt->second, Coeff);
}
if (!SE->isKnownPositive(Stride))
LocalStridesNeedingPreds.insert(Stride);
}
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << ": Const=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
return true;
};
@@ -1125,43 +1168,74 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (!all_of(AllMembers, AccumulateOffset))
continue;
- // The base member (offset = 0) implicitly has coefficient 0 for every
- // stride. Members that lack a particular stride term also have an
- // implicit coefficient of 0 for that stride. Clamp so that the
- // bounding-box always includes coefficient 0, which is guaranteed to
- // exist (at least from the base member).
- for (auto &[Stride, MinC] : MinCoeff)
- MinC = std::min(MinC, (int64_t)0);
- for (auto &[Stride, MaxC] : MaxCoeff)
- MaxC = std::max(MaxC, (int64_t)0);
-
- // MergedLow = BaseLow + CMin + sum(MinCoeff[s] * s)
+ // A member implicitly has coefficient 0 for every stride it does not
+ // mention. So the bounding box must always include coefficient 0 per
+ // stride: clamp the running min down to 0 and the running max up to 0.
+ //
+ // For example, three pointers with two strides s1, s2:
+ // base = p
+ // m1 = p - s1 + s2
+ // m2 = p + 2*s2
+ // Collecting only the coefficients that appear gives:
+ // s1: [-1] -> range [-1, -1]
+ // s2: [1, 2] -> range [ 1, 2]
+ // which is wrong, because it forgets the implicit zeros.
+ // Including every stride in every member:
+ // base = p + 0*s1 + 0*s2
+ // m1 = p + -1*s1 + 1*s2
+ // m2 = p + 0*s1 + 2*s2
+ // gives the correct ranges:
+ // s1: [-1, 0] -> range [-1, 0]
+ // s2: [0, 2] -> range [ 0, 2]
+ // The clamps below enforce exactly that by folding 0 into each stride's
+ // minimum and maximum coefficient.
+ for (auto &[_, MinCoeffVal] : MinCoeff)
+ MinCoeffVal = std::min(MinCoeffVal, (int64_t)0);
+ for (auto &[_, MaxCoeffVal] : MaxCoeff)
+ MaxCoeffVal = std::max(MaxCoeffVal, (int64_t)0);
+
+ // OffsetTy is the type of a member's offset from the base,
+ // Start_member - BaseLow, which is an integer. The merged bounds below are
+ // built as BaseLow/BaseHigh (pointers) plus integer terms of this type.
+ Type *OffsetTy = SE->getEffectiveSCEVType(BaseLow->getType());
+
+ // Construct the merged lower bound:
+ // MergedLow = BaseLow + MinConstOffset + sum(MinCoeff[s] * s)
+ // This assumes every stride s > 0, so the smallest coefficient gives the
+ // smallest address. The SCEV predicates added below (before the loop runs)
+ // guarantee that for any stride not already known positive.
const SCEV *MergedLow = BaseLow;
- if (CMin != 0)
+ if (MinConstOffset != 0)
MergedLow =
- SE->getAddExpr(MergedLow, SE->getConstant(BaseLow->getType(), CMin,
+ SE->getAddExpr(MergedLow, SE->getConstant(OffsetTy, MinConstOffset,
/*isSigned=*/true));
- for (const auto &[Stride, MinC] : MinCoeff) {
- if (MinC != 0)
+ for (const auto &[Stride, MinCoeffVal] : MinCoeff) {
+ // MinCoeff and MaxCoeff share their keys, so this also covers MaxCoeff.
+ assert(Stride->getType() == OffsetTy &&
+ "stride type must match the offset type");
+ if (MinCoeffVal != 0)
MergedLow = SE->getAddExpr(
- MergedLow, SE->getMulExpr(SE->getConstant(Stride->getType(), MinC,
+ MergedLow, SE->getMulExpr(SE->getConstant(OffsetTy, MinCoeffVal,
/*isSigned=*/true),
Stride));
}
- // MergedHigh = BaseHigh + CMax + sum(MaxCoeff[s] * s)
- // BaseHigh = BaseLow + Range, so this gives max(Low_j) + Range.
+ // Construct the merged upper bound:
+ // MergedHigh = BaseHigh + MaxConstOffset + sum(MaxCoeff[s] * s)
+ // Since BaseHigh = BaseLow + Range and every member shares Range, this is
+ // max(Start_j) + Range, i.e. the highest address any member can reach. As
+ // with MergedLow this assumes s > 0, guaranteed by the predicates below.
const SCEV *MergedHigh = BaseHigh;
- if (CMax != 0)
+ if (MaxConstOffset != 0)
MergedHigh =
- SE->getAddExpr(MergedHigh, SE->getConstant(BaseHigh->getType(), CMax,
+ SE->getAddExpr(MergedHigh, SE->getConstant(OffsetTy, MaxConstOffset,
/*isSigned=*/true));
- for (const auto &[Stride, MaxC] : MaxCoeff) {
- if (MaxC != 0)
+ for (const auto &[Stride, MaxCoeffVal] : MaxCoeff) {
+ if (MaxCoeffVal != 0)
MergedHigh = SE->getAddExpr(
- MergedHigh, SE->getMulExpr(SE->getConstant(Stride->getType(), MaxC,
+ MergedHigh, SE->getMulExpr(SE->getConstant(OffsetTy, MaxCoeffVal,
/*isSigned=*/true),
Stride));
}
@@ -1169,37 +1243,63 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
<< ", High=" << *MergedHigh << "\n");
- // Local cost model.
- // G = number of groups in this DepSet, C = number of external groups
- // that need checking against any member of this DepSet.
- // Before merge: G * C checks. After merge: C + predicates.
- unsigned G = GroupIndices.size();
+ // Local cost model: decide whether replacing this DepSet's groups with the
+ // single merged group actually reduces the number of runtime checks.
+ //
+ // The unit is one runtime check: a single bounds comparison emitted between
+ // two groups (the low bound of one against the high bound of the other).
+ // Two groups only produce a check when needsChecking() says so. We assume
+ // every such check costs the same before and after merging (each is the
+ // same pair of pointer comparisons in the IR), so we can just count them.
+ //
+ // NumGroups - groups in this DepSet; all of them collapse into
+ // the
+ // one merged group.
+ // NumExternalChecks - groups *outside* this DepSet that need a check
+ // against at least one member of it. The merged group
+ // will still be checked against exactly these.
+ //
+ // Before merging, each of the NumGroups groups is checked against each of
+ // the NumExternalChecks external groups, giving NumGroups *
+ // NumExternalChecks checks. Every group in this DepSet is read-only and
+ // shares the same (DependencySetId, AliasSetId), and needsChecking() looks
+ // only at those IDs and at whether a group writes memory. So toward any
+ // given external group either all of these groups need a check or none do,
+ // making the count exactly the product above. After merging only the single
+ // merged group remains, so just NumExternalChecks checks, plus one extra
+ // check (a SCEV predicate) for each stride we must prove positive and have
+ // not already paid for in an earlier DepSet.
+ //
+ // To find the external groups we walk *all* checking groups and keep the
+ // ones that are neither part of this DepSet (GroupIndexSet) nor already
+ // consumed by a previous merge in this same call (MergedGroupIndices).
+ unsigned NumGroups = GroupIndices.size();
SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
GroupIndices.end());
- unsigned C = 0;
+ unsigned NumExternalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
continue;
- // Check if this external group needs checking against any member
- // of our DepSet.
for (unsigned GI : GroupIndices) {
if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
- ++C;
+ ++NumExternalChecks;
break;
}
}
}
- // Only count predicates we haven't already committed as cost.
+ // Each not-yet-committed positive-stride predicate becomes one extra
+ // runtime check, so it counts against the saving.
unsigned NewPredicates = 0;
for (const SCEV *Stride : LocalStridesNeedingPreds)
if (!CommittedStridePredicates.contains(Stride))
++NewPredicates;
- unsigned ChecksBefore = G * C;
- unsigned ChecksAfter = C + NewPredicates;
+ unsigned ChecksBefore = NumGroups * NumExternalChecks;
+ unsigned ChecksAfter = NumExternalChecks + NewPredicates;
- LLVM_DEBUG(dbgs() << "LAA: Cost model: G=" << G << ", C=" << C
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
+ << ", NumExternalChecks=" << NumExternalChecks
<< ", predicates=" << NewPredicates << ", checks "
<< ChecksBefore << "->" << ChecksAfter);
@@ -1218,7 +1318,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
for (unsigned GI : GroupIndices)
CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
- // COMMIT: add stride predicates (skip already-committed ones).
+ // We have decided to merge, so now actually register the positive-stride
+ // SCEV predicates with PSE (skipping any stride an earlier DepSet already
+ // added a predicate for).
for (const SCEV *Stride : LocalStridesNeedingPreds) {
if (!CommittedStridePredicates.insert(Stride).second)
continue;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index fada2d09d0509..433bd1e83a319 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1790,3 +1790,110 @@ done:
}
declare i64 @llvm.smax.i64(i64, i64)
+
+;; Test 16: A stride that is itself a sum (s1 + s2).
+;; s1 = smax(s1in, 1), s2 = smax(s2in, 1) (known positive); 3 loads from %p at
+;; offsets {0, s1, s1 + s2}, store to %q.
+;; TODO: Distribute s1 + s2 into s1 and s2 in decomposeStencilOffset
+define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_summed_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; MERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: ((1 smax %s1in) + (1 smax %s2in)) sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_summed_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + (1 smax %s2in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s12 = add i64 %s1, %s2
+ %p1 = getelementptr i8, ptr %p, i64 %s1
+ %p2 = getelementptr i8, ptr %p, i64 %s12
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ load i8, ptr %a0
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ load i8, ptr %a1
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ load i8, ptr %a2
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 0, ptr %aq
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
>From 9000701ca89e8a53a6b345ef7c3ce15f7df78755 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 23 Jun 2026 15:54:43 +0100
Subject: [PATCH 14/33] Split mergeStencilGroups loop into helper functions
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 196 +++++++++++++----------
1 file changed, 113 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 79ce2dca7602a..00872f7182afd 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -906,6 +906,109 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return D;
}
+/// Local cost model: count the runtime checks required before and after
+/// replacing one DepSet's groups (\p GroupIndices) with the single merged
+/// group.
+///
+/// The unit is one runtime check: a single bounds comparison emitted between
+/// two groups (the low bound of one against the high bound of the other). Two
+/// groups only produce a check when needsChecking() says so. We assume every
+/// such check costs the same before and after merging (each is the same pair of
+/// pointer comparisons in the IR), so we can just count them.
+///
+/// NumGroups - groups in this DepSet; all of them collapse into the
+/// one merged group.
+/// NumExternalChecks - groups *outside* this DepSet that need a check against
+/// at least one member of it. The merged group will still
+/// be checked against exactly these.
+///
+/// Before merging, each of the NumGroups groups is checked against each of the
+/// NumExternalChecks external groups, giving NumGroups * NumExternalChecks
+/// checks. Every group in this DepSet is read-only and shares the same
+/// (DependencySetId, AliasSetId), and needsChecking() looks only at those IDs
+/// and at whether a group writes memory. So toward any given external group
+/// either all of these groups need a check or none do, making the count exactly
+/// the product above. After merging only the single merged group remains, so
+/// just NumExternalChecks checks, plus one extra check (a SCEV predicate) for
+/// each stride we must prove positive and have not already paid for in an
+/// earlier DepSet (those already in \p CommittedStridePredicates).
+///
+/// To find the external groups we walk *all* checking groups and keep the ones
+/// that are neither part of this DepSet (\p GroupIndices) nor already consumed
+/// by a previous merge in this same call (\p MergedGroupIndices).
+///
+/// Returns {ChecksBefore, ChecksAfter}.
+static std::pair<unsigned, unsigned>
+computeStencilMergeCost(const RuntimePointerChecking &RtCheck,
+ ArrayRef<unsigned> GroupIndices,
+ const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ const SmallDenseSet<const SCEV *, 4>
+ &CommittedStridePredicates) {
+ ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
+ unsigned NumGroups = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned NumExternalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ for (unsigned GI : GroupIndices) {
+ if (RtCheck.needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++NumExternalChecks;
+ break;
+ }
+ }
+ }
+
+ // Each not-yet-committed positive-stride predicate becomes one extra runtime
+ // check, so it counts against the saving.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
+ << ", NumExternalChecks=" << NumExternalChecks
+ << ", predicates=" << NewPredicates << ", checks "
+ << NumGroups * NumExternalChecks << "->"
+ << NumExternalChecks + NewPredicates << "\n");
+
+ return {NumGroups * NumExternalChecks, NumExternalChecks + NewPredicates};
+}
+
+/// Build the merged stencil group for one DepSet, after the cost model has
+/// decided the merge is profitable. Constructs the bounding group over
+/// \p AllMembers with bounds [\p MergedLow, \p MergedHigh], and registers with
+/// \p PSE a positive-stride SCEV predicate for each stride in
+/// \p LocalStridesNeedingPreds that has not already been committed (tracked in
+/// \p CommittedStridePredicates across DepSets). Returns the new group.
+static RuntimeCheckingPtrGroup buildMergedStencilGroup(
+ const RuntimePointerChecking &RtCheck, PredicatedScalarEvolution &PSE,
+ ScalarEvolution &SE, ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
+ const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
+ RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], RtCheck);
+ CandidateGroup.Low = MergedLow;
+ CandidateGroup.High = MergedHigh;
+ append_range(CandidateGroup.Members, drop_begin(AllMembers));
+ for (unsigned GI : GroupIndices)
+ CandidateGroup.NeedsFreeze |= RtCheck.CheckingGroups[GI].NeedsFreeze;
+
+ // Register the positive-stride SCEV predicates with PSE, skipping any stride
+ // an earlier DepSet already added a predicate for.
+ for (const SCEV *Stride : LocalStridesNeedingPreds) {
+ if (!CommittedStridePredicates.insert(Stride).second)
+ continue;
+ const SCEV *Zero = SE.getZero(Stride->getType());
+ PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+ return CandidateGroup;
+}
+
void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
Loop &L) {
LLVM_DEBUG(dbgs() << "LAA: Attempting stencil group merging on "
@@ -1245,94 +1348,21 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Local cost model: decide whether replacing this DepSet's groups with the
// single merged group actually reduces the number of runtime checks.
- //
- // The unit is one runtime check: a single bounds comparison emitted between
- // two groups (the low bound of one against the high bound of the other).
- // Two groups only produce a check when needsChecking() says so. We assume
- // every such check costs the same before and after merging (each is the
- // same pair of pointer comparisons in the IR), so we can just count them.
- //
- // NumGroups - groups in this DepSet; all of them collapse into
- // the
- // one merged group.
- // NumExternalChecks - groups *outside* this DepSet that need a check
- // against at least one member of it. The merged group
- // will still be checked against exactly these.
- //
- // Before merging, each of the NumGroups groups is checked against each of
- // the NumExternalChecks external groups, giving NumGroups *
- // NumExternalChecks checks. Every group in this DepSet is read-only and
- // shares the same (DependencySetId, AliasSetId), and needsChecking() looks
- // only at those IDs and at whether a group writes memory. So toward any
- // given external group either all of these groups need a check or none do,
- // making the count exactly the product above. After merging only the single
- // merged group remains, so just NumExternalChecks checks, plus one extra
- // check (a SCEV predicate) for each stride we must prove positive and have
- // not already paid for in an earlier DepSet.
- //
- // To find the external groups we walk *all* checking groups and keep the
- // ones that are neither part of this DepSet (GroupIndexSet) nor already
- // consumed by a previous merge in this same call (MergedGroupIndices).
- unsigned NumGroups = GroupIndices.size();
- SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
- GroupIndices.end());
- unsigned NumExternalChecks = 0;
- for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
- if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
- continue;
- for (unsigned GI : GroupIndices) {
- if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
- ++NumExternalChecks;
- break;
- }
- }
- }
-
- // Each not-yet-committed positive-stride predicate becomes one extra
- // runtime check, so it counts against the saving.
- unsigned NewPredicates = 0;
- for (const SCEV *Stride : LocalStridesNeedingPreds)
- if (!CommittedStridePredicates.contains(Stride))
- ++NewPredicates;
-
- unsigned ChecksBefore = NumGroups * NumExternalChecks;
- unsigned ChecksAfter = NumExternalChecks + NewPredicates;
-
- LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
- << ", NumExternalChecks=" << NumExternalChecks
- << ", predicates=" << NewPredicates << ", checks "
- << ChecksBefore << "->" << ChecksAfter);
-
+ auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
+ *this, GroupIndices, MergedGroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates);
if (ChecksAfter >= ChecksBefore) {
- LLVM_DEBUG(dbgs() << " (skipping, not beneficial)\n");
+ LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
}
- LLVM_DEBUG(dbgs() << " (merging, net saving " << ChecksBefore - ChecksAfter
- << ")\n");
-
- // Build the merged group (after deciding to merge).
- RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
- CandidateGroup.Low = MergedLow;
- CandidateGroup.High = MergedHigh;
- append_range(CandidateGroup.Members, drop_begin(AllMembers));
- for (unsigned GI : GroupIndices)
- CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
-
- // We have decided to merge, so now actually register the positive-stride
- // SCEV predicates with PSE (skipping any stride an earlier DepSet already
- // added a predicate for).
- for (const SCEV *Stride : LocalStridesNeedingPreds) {
- if (!CommittedStridePredicates.insert(Stride).second)
- continue;
- const SCEV *Zero = SE->getZero(Stride->getType());
- PSE.addPredicate(
- *SE->getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
- LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
- << *Stride << "\n");
- }
+ LLVM_DEBUG(dbgs() << "LAA: Merging, net saving "
+ << ChecksBefore - ChecksAfter << "\n");
+ // Build the merged group and register its positive-stride predicates.
+ NewMergedGroups.push_back(buildMergedStencilGroup(
+ *this, PSE, *SE, AllMembers, MergedLow, MergedHigh, GroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates));
MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
- NewMergedGroups.push_back(std::move(CandidateGroup));
}
// Rebuild CheckingGroups if we merged anything.
>From fa5c37df24b256dac7aaad01594b29d8deb5886f Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 23 Jun 2026 16:13:46 +0100
Subject: [PATCH 15/33] Clang format
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 00872f7182afd..9bebd7dd00f27 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -938,13 +938,11 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
/// by a previous merge in this same call (\p MergedGroupIndices).
///
/// Returns {ChecksBefore, ChecksAfter}.
-static std::pair<unsigned, unsigned>
-computeStencilMergeCost(const RuntimePointerChecking &RtCheck,
- ArrayRef<unsigned> GroupIndices,
- const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
- ArrayRef<const SCEV *> LocalStridesNeedingPreds,
- const SmallDenseSet<const SCEV *, 4>
- &CommittedStridePredicates) {
+static std::pair<unsigned, unsigned> computeStencilMergeCost(
+ const RuntimePointerChecking &RtCheck, ArrayRef<unsigned> GroupIndices,
+ const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
unsigned NumGroups = GroupIndices.size();
SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
>From 30a0728b5741c28459fc9c06fe49954a87e5fa33 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 15:03:31 +0100
Subject: [PATCH 16/33] Build merged stencil bounds as a umin/umax over
candidate members
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 449 +++--
.../runtime-check-group-merging.ll | 1494 +++++++++++++----
2 files changed, 1458 insertions(+), 485 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9bebd7dd00f27..75556599c8db9 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -855,6 +855,57 @@ struct StencilDecomposition {
SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
};
+/// Recursion cap for addScaledStencilTerm. Depth 3 covers the deepest chain
+/// we distribute: a top-level add (depth 0), a constant-times-X term inside
+/// it (depth 1), a factored add inside that (depth 2), and the leaves
+/// (depth 3). SCEV can nest deeper; a deeper term stays one opaque key with
+/// its multiplier, which is safe but less precise. Constants and leaf
+/// strides are handled at any depth; only the add and constant-times-X
+/// cases recurse.
+constexpr unsigned MaxStencilDecomposeDepth = 3;
+
+/// Fold one term of a stencil offset into \p D, scaled by \p Mult:
+/// constant K -> D.Constant += Mult * K
+/// (K * X) -> recurse into X with multiplier Mult * K
+/// (a + b + ...) -> recurse into each operand with the same Mult
+/// anything else -> D.Coefficients[Term] += Mult
+/// Example: 8 + (-64 * (s1 + s2)) + (-32 * s1) gives Constant = 8 and
+/// coefficients {s1: -96, s2: -64}.
+/// The two recursive cases only fire while Depth is below
+/// MaxStencilDecomposeDepth. At the cap a term becomes one opaque stride key
+/// with coefficient Mult; that is not a bailout.
+/// Returns false when a constant does not fit in int64_t or an update
+/// overflows. The caller then drops the whole decomposition.
+static bool addScaledStencilTerm(const SCEV *Term, int64_t Mult, unsigned Depth,
+ StencilDecomposition &D) {
+ const SCEVConstant *C;
+ // A constant folds into the running constant at any depth.
+ if (match(Term, m_SCEVConstant(C))) {
+ std::optional<int64_t> V = C->getAPInt().trySExtValue();
+ int64_t Scaled;
+ return V && !MulOverflow(Mult, *V, Scaled) &&
+ !AddOverflow(D.Constant, Scaled, D.Constant);
+ }
+
+ if (Depth < MaxStencilDecomposeDepth) {
+ const SCEV *Inner;
+ if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Inner)))) {
+ std::optional<int64_t> V = C->getAPInt().trySExtValue();
+ int64_t NewMult;
+ return V && !MulOverflow(Mult, *V, NewMult) &&
+ addScaledStencilTerm(Inner, NewMult, Depth + 1, D);
+ }
+ if (auto *Add = dyn_cast<SCEVAddExpr>(Term))
+ return all_of(Add->operands(), [&](const SCEV *Op) {
+ return addScaledStencilTerm(Op, Mult, Depth + 1, D);
+ });
+ }
+
+ // Anything else is one stride key.
+ int64_t &Coeff = D.Coefficients[Term];
+ return !AddOverflow(Coeff, Mult, Coeff);
+}
+
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
/// \p Expr is the difference of two access "Start" SCEVs (Start_member -
@@ -862,87 +913,143 @@ struct StencilDecomposition {
/// by getStartAndEndForAccess: the address of the first byte the access can
/// touch. The result describes where one member's range sits relative to the
/// base member's range.
+/// Constant factors are distributed over sums. SCEV can keep a factored form:
+/// -64*s1 + -64*s2 is stored as (-64 * (s1 + s2)). Distributing the -64 gives
+/// the coefficients {s1: -64, s2: -64}, so every member of a group is keyed
+/// on the same base strides.
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
-/// Returns std::nullopt if the expression contains non-stencil terms or any
-/// SCEV constant doesn't fit in int64_t (we commit to the signed
-/// interpretation; values that need more than 64 significant bits are
-/// out of scope).
+/// Returns std::nullopt if a constant does not fit in int64_t or a multiplier
+/// or coefficient update overflows (we commit to the signed interpretation;
+/// values that need more than 64 significant bits are out of scope).
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
- StencilDecomposition D;
-
// A "Start" is always loop-invariant (getStartAndEndForAccess asserts it), so
// the difference Expr passed in by the caller is loop-invariant too, and so
- // is every additive term we pull out of it below.
+ // is every term addScaledStencilTerm visits.
assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
- // Collect top-level additive terms.
- SmallVector<const SCEV *, 4> Terms;
- if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
- append_range(Terms, Add->operands());
- else
- Terms.push_back(Expr);
+ StencilDecomposition D;
+ if (!addScaledStencilTerm(Expr, /*Mult=*/1, /*Depth=*/0, D))
+ return std::nullopt;
+ return D;
+}
- const SCEVConstant *C;
- const SCEV *Stride;
- for (const SCEV *Term : Terms) {
- if (match(Term, m_SCEVConstant(C))) {
- auto V = C->getAPInt().trySExtValue();
- if (!V)
- return std::nullopt;
- D.Constant += *V;
- } else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
- assert(SE.isLoopInvariant(Stride, &L) && "stride must be loop-invariant");
- auto V = C->getAPInt().trySExtValue();
- if (!V)
- return std::nullopt;
- D.Coefficients[Stride] += *V;
- } else {
- assert(SE.isLoopInvariant(Term, &L) && "term must be loop-invariant");
- D.Coefficients[Term] += 1;
+/// Return true if member \p A can never sit at a higher address than member
+/// \p B, for any whole-number stride values of 1 or more.
+/// Example:
+/// A: 0 - 80*s1
+/// B: -40 - 40*s1
+/// At s1 = 1 both sit at -80. For bigger s1, A's steeper slope puts it
+/// lower. So A is never above B.
+/// The general rule needs two facts:
+/// - every coefficient of A is <= the matching coefficient of B, and
+/// - A's address at all-strides-1 (Constant plus the sum of all
+/// coefficients) is <= B's address at that same point.
+/// Strides of 1 or more is a safe assumption: every stride is an integer,
+/// and a merged check only runs with an "s > 0" predicate on each stride
+/// (or the stride is already known positive), so every stride is at least 1.
+/// The decompositions hold signed offsets, but the merged bounds compare
+/// unsigned addresses at runtime. The order carries over: both members
+/// read the same underlying object, and an object never wraps around the
+/// address space, so the member with the smaller offset sits at the
+/// smaller address.
+/// A stride missing from a member's map counts as coefficient 0.
+/// Returns false when a sum overflows. The caller then keeps the member,
+/// which is the safe direction.
+static bool isNeverAbove(const StencilDecomposition &A,
+ const StencilDecomposition &B) {
+ int64_t ACorner = A.Constant, BCorner = B.Constant;
+ for (const auto &[Stride, ACoeff] : A.Coefficients) {
+ if (ACoeff > B.Coefficients.lookup(Stride))
+ return false;
+ if (AddOverflow(ACorner, ACoeff, ACorner))
+ return false;
+ }
+ for (const auto &[Stride, BCoeff] : B.Coefficients) {
+ if (A.Coefficients.lookup(Stride) > BCoeff)
+ return false;
+ if (AddOverflow(BCorner, BCoeff, BCorner))
+ return false;
+ }
+ return ACorner <= BCorner;
+}
+
+/// Find the members that can define the merged bound on one side.
+/// Example for the minimum side (\p ForMin == true), two members:
+/// A: 0 - 80*s1
+/// B: -40 - 40*s1
+/// For every s1 >= 1, A sits at or below B, so B can never be the lowest
+/// member: A beats B. The members nobody beats are the candidates.
+/// The maximum side works the same way with the comparison flipped.
+/// When two members have equal offsets, only the first one is kept.
+/// Returns indices into \p Offsets.
+/// TODO: Worst case compares every pair of members: O(N^2). Fine for real
+/// stencils.
+static SmallVector<unsigned, 4>
+collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
+ // A beats B when A always bounds at least as well as B: for the minimum
+ // side A is never above B, for the maximum side A is never below B.
+ auto Beats = [&](unsigned A, unsigned B) {
+ return ForMin ? isNeverAbove(Offsets[A], Offsets[B])
+ : isNeverAbove(Offsets[B], Offsets[A]);
+ };
+ // Skipping a beaten member loses nothing: Beats is transitive, so
+ // whoever beat it also beats anyone it would have beaten.
+ SmallVector<bool, 8> Beaten(Offsets.size(), false);
+ for (unsigned K = 0; K < Offsets.size(); ++K) {
+ if (Beaten[K])
+ continue;
+ for (unsigned J = K + 1; J < Offsets.size(); ++J) {
+ if (Beaten[J])
+ continue;
+ // Checking K first settles ties: on equal offsets K survives.
+ if (Beats(K, J)) {
+ Beaten[J] = true;
+ } else if (Beats(J, K)) {
+ Beaten[K] = true;
+ break;
+ }
}
}
- return D;
+ SmallVector<unsigned, 4> Candidates;
+ for (unsigned K = 0; K < Offsets.size(); ++K)
+ if (!Beaten[K])
+ Candidates.push_back(K);
+ return Candidates;
}
/// Local cost model: count the runtime checks required before and after
/// replacing one DepSet's groups (\p GroupIndices) with the single merged
-/// group.
+/// group. Everything is counted in the same unit, one check, even though a
+/// stride predicate or an extra umin/umax operand is cheaper at runtime
+/// than a full group-pair check. The cheaper items only appear on the
+/// After side, and we merge only when After < Before, so the rounding
+/// always errs toward not merging.
///
-/// The unit is one runtime check: a single bounds comparison emitted between
-/// two groups (the low bound of one against the high bound of the other). Two
-/// groups only produce a check when needsChecking() says so. We assume every
-/// such check costs the same before and after merging (each is the same pair of
-/// pointer comparisons in the IR), so we can just count them.
+/// Before = NumGroups * NumExternalChecks, where NumExternalChecks is the
+/// number of groups outside this DepSet that need a check against it. The
+/// product is exact: needsChecking() looks only at (DependencySetId,
+/// AliasSetId) and at whether a group writes, and all groups in this
+/// DepSet agree on those, so an external group is checked against all of
+/// them or against none.
///
-/// NumGroups - groups in this DepSet; all of them collapse into the
-/// one merged group.
-/// NumExternalChecks - groups *outside* this DepSet that need a check against
-/// at least one member of it. The merged group will still
-/// be checked against exactly these.
-///
-/// Before merging, each of the NumGroups groups is checked against each of the
-/// NumExternalChecks external groups, giving NumGroups * NumExternalChecks
-/// checks. Every group in this DepSet is read-only and shares the same
-/// (DependencySetId, AliasSetId), and needsChecking() looks only at those IDs
-/// and at whether a group writes memory. So toward any given external group
-/// either all of these groups need a check or none do, making the count exactly
-/// the product above. After merging only the single merged group remains, so
-/// just NumExternalChecks checks, plus one extra check (a SCEV predicate) for
-/// each stride we must prove positive and have not already paid for in an
-/// earlier DepSet (those already in \p CommittedStridePredicates).
-///
-/// To find the external groups we walk *all* checking groups and keep the ones
-/// that are neither part of this DepSet (\p GroupIndices) nor already consumed
-/// by a previous merge in this same call (\p MergedGroupIndices).
+/// After = NumExternalChecks + NewPredicates + NumBoundOperands:
+/// - the merged group keeps the same IDs, so it is checked against exactly
+/// the same external groups;
+/// - one predicate per stride we must prove positive, unless an earlier
+/// DepSet already paid for it (\p CommittedStridePredicates);
+/// - a umin over k members costs k-1 compare+selects, same for the umax
+/// (\p NumMinCandidates - 1 plus \p NumMaxCandidates - 1). A single
+/// candidate costs nothing: the bound is that member's own address.
///
/// Returns {ChecksBefore, ChecksAfter}.
static std::pair<unsigned, unsigned> computeStencilMergeCost(
const RuntimePointerChecking &RtCheck, ArrayRef<unsigned> GroupIndices,
const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
ArrayRef<const SCEV *> LocalStridesNeedingPreds,
- const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
+ const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates,
+ unsigned NumMinCandidates, unsigned NumMaxCandidates) {
ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
unsigned NumGroups = GroupIndices.size();
SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
@@ -966,13 +1073,20 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
if (!CommittedStridePredicates.contains(Stride))
++NewPredicates;
+ // Extra bound operands: one compare-and-select per operand past the first
+ // in the merged umin, and the same for the umax.
+ unsigned NumBoundOperands = (NumMinCandidates - 1) + (NumMaxCandidates - 1);
+
LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
<< ", NumExternalChecks=" << NumExternalChecks
- << ", predicates=" << NewPredicates << ", checks "
+ << ", predicates=" << NewPredicates
+ << ", bound operands=" << NumBoundOperands << ", checks "
<< NumGroups * NumExternalChecks << "->"
- << NumExternalChecks + NewPredicates << "\n");
+ << NumExternalChecks + NewPredicates + NumBoundOperands
+ << "\n");
- return {NumGroups * NumExternalChecks, NumExternalChecks + NewPredicates};
+ return {NumGroups * NumExternalChecks,
+ NumExternalChecks + NewPredicates + NumBoundOperands};
}
/// Build the merged stencil group for one DepSet, after the cost model has
@@ -983,7 +1097,7 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
/// \p CommittedStridePredicates across DepSets). Returns the new group.
static RuntimeCheckingPtrGroup buildMergedStencilGroup(
const RuntimePointerChecking &RtCheck, PredicatedScalarEvolution &PSE,
- ScalarEvolution &SE, ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
+ ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
ArrayRef<const SCEV *> LocalStridesNeedingPreds,
SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
@@ -996,6 +1110,7 @@ static RuntimeCheckingPtrGroup buildMergedStencilGroup(
// Register the positive-stride SCEV predicates with PSE, skipping any stride
// an earlier DepSet already added a predicate for.
+ ScalarEvolution &SE = *PSE.getSE();
for (const SCEV *Stride : LocalStridesNeedingPreds) {
if (!CommittedStridePredicates.insert(Stride).second)
continue;
@@ -1015,14 +1130,16 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
- // We try to merge groups produced by groupChecks when their pointers follow
- // a stencil access pattern. groupChecks intentionally only merges pointers
- // whose min/max bounds differ by constants, because that keeps each runtime
- // check precise. Stencil kernels often read the same underlying object at
- // several loop-invariant stride offsets, so those groups remain separate and
- // can produce too many checks. Here we trade some precision for fewer
- // checks by replacing a set of same-dependence, same-alias read groups with
- // one conservative bounding group.
+ // groupChecks merges two pointers only when their bounds differ by a
+ // compile-time constant, because only then it can tell which bound is
+ // lower or higher. A stencil kernel reads one object at several
+ // loop-invariant offsets, so its bounds differ by expressions like
+ // -40 - 40*s1, and every such pointer stays in its own group - often
+ // too many checks. Here we merge those groups anyway: what we cannot
+ // compare at compile time we compare at runtime, with a umin/umax over
+ // the few members that can be lowest or highest. The cost: the merged
+ // range also covers the gaps between the members, so the merged check
+ // can report a conflict where the per-group checks would not.
//
// We use the following algorithm to construct a merged stencil group:
// - collect checking groups that share both DependencySetId and AliasSetId;
@@ -1031,9 +1148,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// - use one member as the base and decompose each other member's offset
// from that base as C + sum(Coeff[Stride] * Stride), where Stride is
// loop-invariant;
- // - build one bounding range by taking the min/max constant offset and the
- // min/max coefficient for each stride, adding predicates for strides
- // that are not already known positive;
+ // - keep the members that can hold the lowest or the highest address at
+ // runtime (the candidate members), and build the merged bounds as a
+ // umin over their Start values and a umax over their End values,
+ // adding predicates for strides that are not already known positive;
// - commit the merge only if the local cost model reduces the number of
// checks after accounting for any new predicates.
@@ -1052,9 +1170,11 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
}
// For each checking group this pass decomposes each member's offset into
- // stencil form and builds a merged bounding box. That extra SCEV work adds up
- // on a loop with very many groups, so bail out above a configurable limit as
- // a safety net against pathological inputs.
+ // stencil form, keeps the candidate members (the ones that can hold the
+ // lowest or highest address at runtime), and builds the merged bounds from
+ // their own Start and End values. That extra SCEV work adds up on a loop
+ // with very many groups, so bail out above a configurable limit as a
+ // safety net against pathological inputs.
if (CheckingGroups.size() > StencilMergeMaxGroups) {
LLVM_DEBUG(
dbgs() << "LAA: " << CheckingGroups.size()
@@ -1150,9 +1270,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Use the first member as the reference for decomposition. All offsets are
- // computed relative to BaseLow, and the merged upper bound is built from
- // BaseHigh (see merged-bounds computation below).
+ // Use the first member as the reference for decomposition. All offsets
+ // are computed relative to BaseLow. BaseHigh is only used to check that
+ // every member covers the same range. The merged bounds are built later
+ // from the members' own Start and End values.
unsigned Member0 = AllMembers[0];
const SCEV *BaseLow = Pointers[Member0].Start;
const SCEV *BaseHigh = Pointers[Member0].End;
@@ -1172,11 +1293,11 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (!BaseStep)
continue;
- // Verify all members have the same access range (End - Start). The merged
- // group gets a single upper bound (MergedHigh, built further below) of the
- // form BaseHigh + max_offsets. That is only correct when every member's
- // range equals BaseHigh - BaseLow; otherwise a member with a larger range
- // could reach past MergedHigh.
+ // Verify all members have the same access range (End - Start). The
+ // merged upper bound is a umax over the members' own End values. The
+ // same decompositions order both the Start values and the End values
+ // only when End = Start + Range with one shared Range for every member.
+ // That is what this check enforces.
// Compare each member's range (End - Start) and test Range - BaseRange ==
// 0, rather than Range == BaseRange, so algebraically equal but
// non-identical SCEVs still match. Bail out if any subtraction produces
@@ -1217,18 +1338,16 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
"skipping DepSet\n");
continue;
}
- // Per-stride min/max coefficient seen across all members, and the min/max
- // of the constant part of the offset. Together they describe the bounding
- // box.
- SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
- int64_t MinConstOffset = 0, MaxConstOffset = 0;
+ // One decomposition per member, in AllMembers order. Each entry holds the
+ // member's constant offset and its coefficient for each stride, all
+ // relative to BaseLow.
+ SmallVector<StencilDecomposition, 8> MemberOffsets;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
- // Decompose one member's offset (relative to BaseLow) and fold its
- // constant and per-stride coefficients into the running bounding box.
- // Returns false if the offset is not in stencil form (so the whole
- // DepSet is skipped).
- const auto AccumulateOffset = [&](unsigned Idx) -> bool {
+ // Decompose one member's offset (relative to BaseLow) and append it to
+ // MemberOffsets. Returns false if the offset is not in stencil form (so
+ // the whole DepSet is skipped).
+ const auto CollectOffset = [&](unsigned Idx) -> bool {
const SCEV *LowOffset = SE->getMinusSCEV(Pointers[Idx].Start, BaseLow);
if (isa<SCEVCouldNotCompute>(LowOffset))
return false;
@@ -1239,106 +1358,71 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return false;
}
- MinConstOffset = std::min(MinConstOffset, DLow->Constant);
- MaxConstOffset = std::max(MaxConstOffset, DLow->Constant);
-
- // Fold this member's coefficient for each stride into the running min and
- // max for that stride. MinCoeff and MaxCoeff are kept in lock-step: a
- // stride is always present in both or neither.
- for (const auto &[Stride, Coeff] : DLow->Coefficients) {
- auto MinIt = MinCoeff.find(Stride);
- if (MinIt == MinCoeff.end()) {
- MinCoeff[Stride] = Coeff;
- MaxCoeff[Stride] = Coeff;
- } else {
- MinIt->second = std::min(MinIt->second, Coeff);
- auto MaxIt = MaxCoeff.find(Stride);
- MaxIt->second = std::max(MaxIt->second, Coeff);
- }
-
+ for (const auto &[Stride, Coeff] : DLow->Coefficients)
if (!SE->isKnownPositive(Stride))
LocalStridesNeedingPreds.insert(Stride);
- }
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< ": Const=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
+ MemberOffsets.push_back(std::move(*DLow));
return true;
};
- if (!all_of(AllMembers, AccumulateOffset))
+ if (!all_of(AllMembers, CollectOffset))
continue;
- // A member implicitly has coefficient 0 for every stride it does not
- // mention. So the bounding box must always include coefficient 0 per
- // stride: clamp the running min down to 0 and the running max up to 0.
- //
- // For example, three pointers with two strides s1, s2:
- // base = p
- // m1 = p - s1 + s2
- // m2 = p + 2*s2
- // Collecting only the coefficients that appear gives:
- // s1: [-1] -> range [-1, -1]
- // s2: [1, 2] -> range [ 1, 2]
- // which is wrong, because it forgets the implicit zeros.
- // Including every stride in every member:
- // base = p + 0*s1 + 0*s2
- // m1 = p + -1*s1 + 1*s2
- // m2 = p + 0*s1 + 2*s2
- // gives the correct ranges:
- // s1: [-1, 0] -> range [-1, 0]
- // s2: [0, 2] -> range [ 0, 2]
- // The clamps below enforce exactly that by folding 0 into each stride's
- // minimum and maximum coefficient.
- for (auto &[_, MinCoeffVal] : MinCoeff)
- MinCoeffVal = std::min(MinCoeffVal, (int64_t)0);
- for (auto &[_, MaxCoeffVal] : MaxCoeff)
- MaxCoeffVal = std::max(MaxCoeffVal, (int64_t)0);
-
- // OffsetTy is the type of a member's offset from the base,
- // Start_member - BaseLow, which is an integer. The merged bounds below are
- // built as BaseLow/BaseHigh (pointers) plus integer terms of this type.
+ SmallVector<unsigned, 4> MinCandidates =
+ collectCandidateMembers(MemberOffsets, /*ForMin=*/true);
+ SmallVector<unsigned, 4> MaxCandidates =
+ collectCandidateMembers(MemberOffsets, /*ForMin=*/false);
+ assert(!MinCandidates.empty() && !MaxCandidates.empty() &&
+ "a non-empty member list always has a candidate");
+ LLVM_DEBUG(dbgs() << "LAA: Candidate members: min="
+ << MinCandidates.size()
+ << ", max=" << MaxCandidates.size() << " of "
+ << MemberOffsets.size() << "\n");
+
+ // OffsetTy is an integer type as wide as an address. The umin/umax
+ // operands below are member addresses converted to this type.
Type *OffsetTy = SE->getEffectiveSCEVType(BaseLow->getType());
- // Construct the merged lower bound:
- // MergedLow = BaseLow + MinConstOffset + sum(MinCoeff[s] * s)
- // This assumes every stride s > 0, so the smallest coefficient gives the
- // smallest address. The SCEV predicates added below (before the loop runs)
- // guarantee that for any stride not already known positive.
- const SCEV *MergedLow = BaseLow;
- if (MinConstOffset != 0)
- MergedLow =
- SE->getAddExpr(MergedLow, SE->getConstant(OffsetTy, MinConstOffset,
- /*isSigned=*/true));
-
- for (const auto &[Stride, MinCoeffVal] : MinCoeff) {
- // MinCoeff and MaxCoeff share their keys, so this also covers MaxCoeff.
- assert(Stride->getType() == OffsetTy &&
- "stride type must match the offset type");
- if (MinCoeffVal != 0)
- MergedLow = SE->getAddExpr(
- MergedLow, SE->getMulExpr(SE->getConstant(OffsetTy, MinCoeffVal,
- /*isSigned=*/true),
- Stride));
- }
+ // Build one side of the merged bounds from its candidate members.
+ // With one candidate the bound is that member's own Start (or End): the
+ // exact value the member's own check used before the merge. With several
+ // candidates the bound is a umin (umax) over their Starts (Ends). Either
+ // way every value is a real member address, so the merge computes no new
+ // address and no new overflow is possible.
+ // SCEV umin/umax needs integer operands, so convert each pointer with
+ // ptrtoint. That fails for non-integral pointers; return null then, and
+ // the DepSet is skipped.
+ const auto BuildBound = [&](ArrayRef<unsigned> Candidates,
+ bool IsLow) -> const SCEV * {
+ auto GetBound = [&](unsigned K) {
+ const PointerInfo &P = Pointers[AllMembers[K]];
+ return IsLow ? P.Start : P.End;
+ };
+ if (Candidates.size() == 1)
+ return GetBound(Candidates.front());
+ SmallVector<SCEVUse, 4> Ops;
+ for (unsigned K : Candidates) {
+ const SCEV *Bound = GetBound(K);
+ if (!Bound->getType()->isPointerTy())
+ return nullptr;
+ const SCEV *IntBound = SE->getPtrToIntExpr(Bound, OffsetTy);
+ if (isa<SCEVCouldNotCompute>(IntBound))
+ return nullptr;
+ Ops.push_back(IntBound);
+ }
+ return IsLow ? SE->getUMinExpr(Ops) : SE->getUMaxExpr(Ops);
+ };
- // Construct the merged upper bound:
- // MergedHigh = BaseHigh + MaxConstOffset + sum(MaxCoeff[s] * s)
- // Since BaseHigh = BaseLow + Range and every member shares Range, this is
- // max(Start_j) + Range, i.e. the highest address any member can reach. As
- // with MergedLow this assumes s > 0, guaranteed by the predicates below.
- const SCEV *MergedHigh = BaseHigh;
- if (MaxConstOffset != 0)
- MergedHigh =
- SE->getAddExpr(MergedHigh, SE->getConstant(OffsetTy, MaxConstOffset,
- /*isSigned=*/true));
-
- for (const auto &[Stride, MaxCoeffVal] : MaxCoeff) {
- if (MaxCoeffVal != 0)
- MergedHigh = SE->getAddExpr(
- MergedHigh, SE->getMulExpr(SE->getConstant(OffsetTy, MaxCoeffVal,
- /*isSigned=*/true),
- Stride));
+ const SCEV *MergedLow = BuildBound(MinCandidates, /*IsLow=*/true);
+ const SCEV *MergedHigh = BuildBound(MaxCandidates, /*IsLow=*/false);
+ if (!MergedLow || !MergedHigh) {
+ LLVM_DEBUG(dbgs() << "LAA: Cannot build umin/umax bounds, "
+ "skipping DepSet\n");
+ continue;
}
LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
@@ -1348,7 +1432,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// single merged group actually reduces the number of runtime checks.
auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
*this, GroupIndices, MergedGroupIndices,
- LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates);
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates,
+ MinCandidates.size(), MaxCandidates.size());
if (ChecksAfter >= ChecksBefore) {
LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
@@ -1358,7 +1443,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Build the merged group and register its positive-stride predicates.
NewMergedGroups.push_back(buildMergedStencilGroup(
- *this, PSE, *SE, AllMembers, MergedLow, MergedHigh, GroupIndices,
+ *this, PSE, AllMembers, MergedLow, MergedHigh, GroupIndices,
LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates));
MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 433bd1e83a319..06e4a40b0158b 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -156,163 +156,6 @@ exit:
}
-;; Test 2: Two runtime strides with constant offsets.
-;; 5 loads from %vol at offsets: +8+cdj-cdk, -16-2*cdj, +24+3*cdj+2*cdk,
-;; -8+2*cdj-3*cdk, +32-4*cdj+cdk. Store to %out.
-;; With merge: 1 merged group with bounds spanning the bounding box over
-;; both cdj and cdk coefficients, 1 check, 2 stride predicates.
-;; Without merge: 5 separate groups, 5 checks, no predicates.
-define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64 %cdk) {
-; MERGE-LABEL: 'stencil_merge_two_strides'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (16 + (-4 * %cdj) + (-3 * %cdk) + %vol) High: ((2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
-; MERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-NEXT: Compare predicate: %cdj sgt) 0
-; MERGE-NEXT: Compare predicate: %cdk sgt) 0
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'stencil_merge_two_strides'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
-; NOMERGE-NEXT: Check 3:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
-; NOMERGE-NEXT: Check 4:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (64 + (-4 * %cdj) + %cdk + %vol) High: ((8 * %n) + (-4 * %cdj) + %cdk + %vol))
-; NOMERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + (-3 * %cdk) + %vol) High: (-40 + (2 * %cdj) + (8 * %n) + (-3 * %cdk) + %vol))
-; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (56 + (2 * %cdk) + (3 * %cdj) + %vol) High: (-8 + (2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
-; NOMERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %vol) High: (-48 + (8 * %n) + (-2 * %cdj) + %vol))
-; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP5:
-; NOMERGE-NEXT: (Low: (40 + (-1 * %cdk) + %cdj + %vol) High: (-24 + (8 * %n) + (-1 * %cdk) + %cdj + %vol))
-; NOMERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
-;
-entry:
- %cmp = icmp sgt i64 %n, 8
- br i1 %cmp, label %loop, label %exit
-
-loop:
- %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
- %base = getelementptr inbounds double, ptr %vol, i64 %iv
-
- ; +8 + cdj - cdk
- %off0a = add nsw i64 8, %cdj
- %negcdk = sub nsw i64 0, %cdk
- %off0 = add nsw i64 %off0a, %negcdk
- %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
- %v0 = load double, ptr %p0, align 8
-
- ; -16 - 2*cdj
- %neg2cdj = mul nsw i64 %cdj, -2
- %off1 = add nsw i64 -16, %neg2cdj
- %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
- %v1 = load double, ptr %p1, align 8
-
- ; +24 + 3*cdj + 2*cdk
- %pos3cdj = mul nsw i64 %cdj, 3
- %pos2cdk = mul nsw i64 %cdk, 2
- %off2a = add nsw i64 24, %pos3cdj
- %off2 = add nsw i64 %off2a, %pos2cdk
- %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
- %v2 = load double, ptr %p2, align 8
-
- ; -8 + 2*cdj - 3*cdk
- %pos2cdj = mul nsw i64 %cdj, 2
- %neg3cdk = mul nsw i64 %cdk, -3
- %off3a = add nsw i64 -8, %pos2cdj
- %off3 = add nsw i64 %off3a, %neg3cdk
- %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
- %v3 = load double, ptr %p3, align 8
-
- ; +32 - 4*cdj + cdk
- %neg4cdj = mul nsw i64 %cdj, -4
- %off4a = add nsw i64 32, %neg4cdj
- %off4 = add nsw i64 %off4a, %cdk
- %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
- %v4 = load double, ptr %p4, align 8
-
- %s0 = fadd double %v0, %v1
- %s1 = fadd double %s0, %v2
- %s2 = fadd double %s1, %v3
- %s3 = fadd double %s2, %v4
-
- %outp = getelementptr inbounds double, ptr %out, i64 %iv
- store double %s3, ptr %outp, align 8
-
- %iv.next = add nuw nsw i64 %iv, 1
- %sub = sub nsw i64 %n, 4
- %cond = icmp slt i64 %iv.next, %sub
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-
;; Test 3: Constant offsets only — existing grouping handles this.
;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
@@ -607,10 +450,11 @@ exit:
}
-;; Test 6: Coefficient clamping in merged bounds.
-;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. Members that lack a stride
-;; term have an implicit coefficient of 0, which must be included in the
-;; min/max computation via clamping.
+;; Test 6: A member with no stride term.
+;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. The base load has no cdj
+;; term at all, so its cdj coefficient counts as 0. The candidate comparison
+;; must see that 0: the base member (+0) is the high bound, above -cdj and
+;; -2*cdj, even though it never mentions cdj.
define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-LABEL: 'coefficient_clamping_regression'
; MERGE-NEXT: loop:
@@ -994,145 +838,6 @@ exit:
}
-;; Test 9: Mixed constant offsets + runtime stride.
-;; 4 constant-offset loads at {-16, -8, +8, +16} + 2 runtime-stride loads
-;; at {-cdj, +cdj}. Store to %out.
-;; Without merge: standard grouping merges the 4 constant-offset loads into
-;; 1 group but the 2 runtime-stride loads remain separate -> 3 checks.
-;; With merge: all 6 loads form 1 merged group -> 1 check + 1 predicate.
-;; Merged Low includes -cdj, Merged High includes +cdj:
-define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
-; MERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + %cdj + %a))
-; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {%a,+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-NEXT: Compare predicate: %cdj sgt) 0
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Member: {%a,+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
-;
-entry:
- %cmp = icmp sgt i64 %n, 4
- br i1 %cmp, label %loop, label %exit
-
-loop:
- %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
- %base = getelementptr inbounds double, ptr %a, i64 %iv
-
- ; -16 (constant)
- %p0 = getelementptr inbounds i8, ptr %base, i64 -16
- %v0 = load double, ptr %p0, align 8
-
- ; -8 (constant)
- %p1 = getelementptr inbounds i8, ptr %base, i64 -8
- %v1 = load double, ptr %p1, align 8
-
- ; +8 (constant)
- %p2 = getelementptr inbounds i8, ptr %base, i64 8
- %v2 = load double, ptr %p2, align 8
-
- ; +16 (constant)
- %p3 = getelementptr inbounds i8, ptr %base, i64 16
- %v3 = load double, ptr %p3, align 8
-
- ; -cdj (runtime)
- %negcdj = sub nsw i64 0, %cdj
- %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
- %v4 = load double, ptr %p4, align 8
-
- ; +cdj (runtime)
- %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
- %v5 = load double, ptr %p5, align 8
-
- %s0 = fadd double %v0, %v1
- %s1 = fadd double %s0, %v2
- %s2 = fadd double %s1, %v3
- %s3 = fadd double %s2, %v4
- %s4 = fadd double %s3, %v5
-
- %outp = getelementptr inbounds double, ptr %out, i64 %iv
- store double %s4, ptr %outp, align 8
-
- %iv.next = add nuw nsw i64 %iv, 1
- %sub = sub nsw i64 %n, 2
- %cond = icmp slt i64 %iv.next, %sub
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-
;; Test 10: Shared stride predicate deduplication across two DepSets.
;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
@@ -1794,7 +1499,11 @@ declare i64 @llvm.smax.i64(i64, i64)
;; Test 16: A stride that is itself a sum (s1 + s2).
;; s1 = smax(s1in, 1), s2 = smax(s2in, 1) (known positive); 3 loads from %p at
;; offsets {0, s1, s1 + s2}, store to %q.
-;; TODO: Distribute s1 + s2 into s1 and s2 in decomposeStencilOffset
+;; One member offset is (-1 * (s1 + s2)). decomposeStencilOffset distributes
+;; the -1, so that member is keyed on s1 and s2 like the others. It then
+;; defines the low bound alone: Low is plain %p with no umin. Both strides
+;; are known positive, so no predicate is emitted.
+;; The merge saves 3 loads x 1 external check = 3 and costs 1 check.
define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
; MERGE-LABEL: 'stencil_merge_summed_stride'
; MERGE-NEXT: loop:
@@ -1813,7 +1522,7 @@ define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i
; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
; MERGE-NEXT: Member: {%q,+,8}<%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; MERGE-NEXT: (Low: %p High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
; MERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
; MERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
; MERGE-NEXT: Member: {%p,+,8}<%loop>
@@ -1824,7 +1533,6 @@ define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i
; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
-; MERGE-NEXT: Compare predicate: ((1 smax %s1in) + (1 smax %s2in)) sgt) 0
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -1897,3 +1605,1183 @@ loop:
done:
ret void
}
+
+
+;; Test 17: A member that uses two strides at once.
+;; 6 loads from %a: +0, -5*s1, +5*s1, -5*s2, +5*s2, and the mixed
+;; +5*s1+5*s2. Store to %out.
+;; The members at -5*s1 and -5*s2 can each be the lowest address: which one
+;; is lower depends on whether s1 or s2 is bigger. So Low is a umin over
+;; those two members. The mixed member is at or above every other member in
+;; both strides at once, so High is just that member's own End - no umax.
+;; No bound is an invented corner like base - 5*s1 - 5*s2.
+define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2) {
+; MERGE-LABEL: 'stencil_merge_mixed_member'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + (ptrtoint ptr %a to i64)) umin (32 + (-5 * %s1) + (ptrtoint ptr %a to i64))) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (5 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-5 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_mixed_member'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s1) + (5 * %s2) + %a) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s2) + %a) High: (-32 + (5 * %s2) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (32 + (-5 * %s2) + %a) High: (-32 + (8 * %n) + (-5 * %s2) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s1) + %a) High: (-32 + (5 * %s1) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (32 + (-5 * %s1) + %a) High: (-32 + (8 * %n) + (-5 * %s1) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-5 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (32 + %a) High: (-32 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; +0 (the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; -5*s1
+ %neg5s1 = mul nsw i64 %s1, -5
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+ %v1 = load double, ptr %p1, align 8
+
+ ; +5*s1
+ %pos5s1 = mul nsw i64 %s1, 5
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+ %v2 = load double, ptr %p2, align 8
+
+ ; -5*s2
+ %neg5s2 = mul nsw i64 %s2, -5
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+ %v3 = load double, ptr %p3, align 8
+
+ ; +5*s2
+ %pos5s2 = mul nsw i64 %s2, 5
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+ %v4 = load double, ptr %p4, align 8
+
+ ; +5*s1 + 5*s2 (the mixed member)
+ %mixed = add nsw i64 %pos5s1, %pos5s2
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+ %v5 = load double, ptr %p5, align 8
+
+ %t0 = fadd double %v0, %v1
+ %t1 = fadd double %t0, %v2
+ %t2 = fadd double %t1, %v3
+ %t3 = fadd double %t2, %v4
+ %t4 = fadd double %t3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 18: Too many candidate members - the cost model rejects the merge.
+;; 8 loads from %a at offsets {-s1, +s1, -s2, +s2, -s3, +s3, -s4, +s4}.
+;; The strides are unrelated, so all four -s_i members can be the lowest
+;; address and all four +s_i members can be the highest. The merged bounds
+;; would need a 4-way umin and a 4-way umax.
+;; Cost: 8 checks before. After: 1 check + 4 stride predicates + 3 umin
+;; operands + 3 umax operands = 11. 11 >= 8, so no merge, even with
+;; -stencil-runtime-check-merge=force.
+define void @too_many_candidates_no_merge(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2, i64 %s3, i64 %s4) {
+; MERGE-LABEL: 'too_many_candidates_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP5:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+; MERGE-NEXT: Check 5:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP6:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; MERGE-NEXT: Check 6:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP7:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+; MERGE-NEXT: Check 7:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP8:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
+; MERGE-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
+; MERGE-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
+; MERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP4:
+; MERGE-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
+; MERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP5:
+; MERGE-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
+; MERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP6:
+; MERGE-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
+; MERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP7:
+; MERGE-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
+; MERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP8:
+; MERGE-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
+; MERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'too_many_candidates_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP7:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP8:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP7:
+; NOMERGE-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP8:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %negs1 = sub nsw i64 0, %s1
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+ %v0 = load double, ptr %p0, align 8
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+
+ %negs2 = sub nsw i64 0, %s2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+
+ %negs3 = sub nsw i64 0, %s3
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+
+ %negs4 = sub nsw i64 0, %s4
+ %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+ %v6 = load double, ptr %p6, align 8
+ %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+ %v7 = load double, ptr %p7, align 8
+
+ %t0 = fadd double %v0, %v1
+ %t1 = fadd double %t0, %v2
+ %t2 = fadd double %t1, %v3
+ %t3 = fadd double %t2, %v4
+ %t4 = fadd double %t3, %v5
+ %t5 = fadd double %t4, %v6
+ %t6 = fadd double %t5, %v7
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 19: One member's offset stays factored as (64 * (s1 + s2)).
+;; 4 loads from %p at byte offsets {0, 64*s1, 64*s2, 64*(s1+s2)}; the last
+;; offset is computed as (s1 + s2) * 64, and SCEV keeps the factored form.
+;; Stores to %q and %q2.
+;; decomposeStencilOffset distributes the 64, so the factored member is keyed
+;; on s1 and s2 like the others. With strides of at least 1 that member sits
+;; at or above every other member, and the member at offset 0 sits at or
+;; below every other member. So High is the factored member's own End, Low is
+;; plain %p with no umin, and only two predicates (s1 > 0, s2 > 0) are
+;; emitted - no (s1 + s2) > 0.
+;; The two stores keep the merge profitable: the merge saves 4 loads x 2
+;; external checks = 8 and costs 2 checks + 2 predicates = 4.
+define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i64 %s2, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_factored_diagonal'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; MERGE-NEXT: Member: {%q2,+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %p High: (1 + (8 * %n) + (64 * (%s1 + %s2)) + %p))
+; MERGE-NEXT: Member: {((64 * (%s1 + %s2)) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s2) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s2) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_factored_diagonal'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 8:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; NOMERGE-NEXT: Member: {%q2,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((64 * (%s1 + %s2)) + %p) High: (1 + (8 * %n) + (64 * (%s1 + %s2)) + %p))
+; NOMERGE-NEXT: Member: {((64 * (%s1 + %s2)) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((64 * %s2) + %p) High: (1 + (8 * %n) + (64 * %s2) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s2) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((64 * %s1) + %p) High: (1 + (8 * %n) + (64 * %s1) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s2) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %pos64s1 = mul i64 %s1, 64
+ %pos64s2 = mul i64 %s2, 64
+ %s12 = add i64 %s1, %s2
+ %diag = mul i64 %s12, 64
+ %p1 = getelementptr i8, ptr %p, i64 %pos64s1
+ %p2 = getelementptr i8, ptr %p, i64 %pos64s2
+ %p3 = getelementptr i8, ptr %p, i64 %diag
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ load i8, ptr %a0
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ load i8, ptr %a1
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ load i8, ptr %a2
+ %a3 = getelementptr i8, ptr %p3, i64 %idx
+ load i8, ptr %a3
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 0, ptr %aq
+ %aq2 = getelementptr i8, ptr %q2, i64 %idx
+ store i8 0, ptr %aq2
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
+
+;; Test 20: The depth cap keeps a deep term as one opaque key.
+;; 3 loads from %p at byte offsets {0, 64*s1, 8 + 64*(s1 + s2 + 4*s3)}.
+;; The deep offset is an add (depth 0) holding 64*(...) (depth 1) holding a
+;; 3-operand add (depth 2) whose term 4*s3 sits at depth 3.
+;; SCEV keeps 64*(s1 + s2 + 4*s3) factored: it only distributes a constant
+;; over a 2-operand add.
+;; decomposeStencilOffset distributes down to depth 3 and then stops: s1 and
+;; s2 get their own keys, and (4 * s3) stays one opaque key. The
+;; decomposition still succeeds - the cap is not a bailout - so the group
+;; still merges.
+;; Predicates: s1 > 0, s2 > 0, and (4 * s3) > 0 for the opaque key.
+;; The deep load comes first in the loop on purpose. That makes %p the
+;; base of the relative decomposition. With the deep member as base, SCEV
+;; cancellation flattens 4*s3 into a plain s3 coefficient and the cap is
+;; never tested. If the (4 * %s3) predicate ever disappears from the CHECK
+;; lines, the test has stopped covering the cap.
+;; The three stores keep the merge profitable: the merge saves 3 loads x 3
+;; external checks = 9 and costs 3 checks + 3 predicates = 6.
+define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1, i64 %s2, i64 %s3, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_depth_cap'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 5:
+; MERGE-NEXT: Comparing group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; MERGE-NEXT: Member: {%q2,+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %q3 High: (1 + (8 * %n) + %q3))
+; MERGE-NEXT: Member: {%q3,+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %p High: (9 + (8 * %n) + (64 * ((4 * %s3) + %s1 + %s2)) + %p))
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q3,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: (4 * %s3) sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_depth_cap'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 8:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 9:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 10:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 11:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; NOMERGE-NEXT: Member: {%q2,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %q3 High: (1 + (8 * %n) + %q3))
+; NOMERGE-NEXT: Member: {%q3,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((64 * %s1) + %p) High: (1 + (8 * %n) + (64 * %s1) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p) High: (9 + (8 * %n) + (64 * ((4 * %s3) + %s1 + %s2)) + %p))
+; NOMERGE-NEXT: Member: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q3,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %pos64s1 = mul i64 %s1, 64
+ %four.s3 = mul i64 %s3, 4
+ %sum12 = add i64 %s1, %s2
+ %sum = add i64 %sum12, %four.s3
+ %deep = mul i64 %sum, 64
+ %deep8 = add i64 %deep, 8
+ %p1 = getelementptr i8, ptr %p, i64 %pos64s1
+ %p2 = getelementptr i8, ptr %p, i64 %deep8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ load i8, ptr %a2
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ load i8, ptr %a1
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ load i8, ptr %a0
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 0, ptr %aq
+ %aq2 = getelementptr i8, ptr %q2, i64 %idx
+ store i8 0, ptr %aq2
+ %aq3 = getelementptr i8, ptr %q3, i64 %idx
+ store i8 0, ptr %aq3
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
+
+
+;; Test 21: A constant-offset member and a stride member share the bounds.
+;; 4 loads from %a at byte offsets {-16, +16, -cdj, +cdj}. Stores to %out
+;; and %out2.
+;; The member at -16 has the smaller constant; the member at -cdj has the
+;; smaller cdj coefficient. Neither sits at or below the other for every
+;; positive cdj: at cdj = 1 the -16 member is lower, at cdj = 100 the -cdj
+;; member is lower. So Low is a umin over those two members. The +16 and
+;; +cdj members mirror this on the high side, so High is a umax over them.
+;; The two constant-offset loads share one group before the merge, so the
+;; merge saves 3 groups x 2 external checks = 6 and costs 2 checks + 1
+;; predicate + 1 umin operand + 1 umax operand = 5.
+define void @stencil_merge_constant_and_stride_candidates(ptr %a, ptr %out, ptr %out2, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_constant_and_stride_candidates'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %out2) High: (-16 + (8 * %n) + %out2))
+; MERGE-NEXT: Member: {(16 + %out2),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((16 + (-1 * %cdj) + (ptrtoint ptr %a to i64)) umin (ptrtoint ptr %a to i64)) High: (((8 * %n) + (ptrtoint ptr %a to i64)) umax (-16 + (8 * %n) + (ptrtoint ptr %a to i64) + %cdj)))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_constant_and_stride_candidates'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %out2) High: (-16 + (8 * %n) + %out2))
+; NOMERGE-NEXT: Member: {(16 + %out2),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; -16 (constant)
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ ; +16 (constant)
+ %p1 = getelementptr inbounds i8, ptr %base, i64 16
+ %v1 = load double, ptr %p1, align 8
+
+ ; -cdj (runtime)
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ ; +cdj (runtime)
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s2, ptr %outp, align 8
+ %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+ store double %s2, ptr %outp2, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 22: Three strides in a star shape - multi-operand umin and umax.
+;; 7 loads from %a at byte offsets {0, -s1, +s1, -s2, +s2, -s3, +s3}.
+;; Stores to %out and %out2.
+;; Any of -s1, -s2, -s3 can be the lowest address: which one is lowest
+;; depends on which stride is biggest. So Low is a umin over those three
+;; members, and High is a umax over +s1, +s2, +s3. The center member at +0
+;; sits between -s1 and +s1, so it is never a bound.
+;; The merge saves 7 groups x 2 external checks = 14 and costs 2 checks +
+;; 3 predicates + 2 umin operands + 2 umax operands = 9.
+define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n, i64 %s1, i64 %s2, i64 %s3) {
+; MERGE-LABEL: 'stencil_merge_three_stride_star'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %s3
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %s2
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %s1
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %s3
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %s2
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %s1
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (32 + %out2) High: (-32 + (8 * %n) + %out2))
+; MERGE-NEXT: Member: {(32 + %out2),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((32 + (-1 * %s3) + (ptrtoint ptr %a to i64)) umin (32 + (-1 * %s2) + (ptrtoint ptr %a to i64)) umin (32 + (-1 * %s1) + (ptrtoint ptr %a to i64))) High: ((-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s1) umax (-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s2) umax (-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s3)))
+; MERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %s3 sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_three_stride_star'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %s3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %s2
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP7:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP8:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 8:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %s3
+; NOMERGE-NEXT: Check 9:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; NOMERGE-NEXT: Check 10:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %s2
+; NOMERGE-NEXT: Check 11:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; NOMERGE-NEXT: Check 12:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 13:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP7:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; NOMERGE-NEXT: Check 14:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+; NOMERGE-NEXT: Against group GRP8:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (32 + %out2) High: (-32 + (8 * %n) + %out2))
+; NOMERGE-NEXT: Member: {(32 + %out2),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
+; NOMERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP7:
+; NOMERGE-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP8:
+; NOMERGE-NEXT: (Low: (32 + %a) High: (-32 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; +0 (the center member; never a bound)
+ %v0 = load double, ptr %base, align 8
+
+ %negs1 = sub nsw i64 0, %s1
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %negs1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %s1
+ %v2 = load double, ptr %p2, align 8
+
+ %negs2 = sub nsw i64 0, %s2
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %negs2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %s2
+ %v4 = load double, ptr %p4, align 8
+
+ %negs3 = sub nsw i64 0, %s3
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %negs3
+ %v5 = load double, ptr %p5, align 8
+ %p6 = getelementptr inbounds i8, ptr %base, i64 %s3
+ %v6 = load double, ptr %p6, align 8
+
+ %t0 = fadd double %v0, %v1
+ %t1 = fadd double %t0, %v2
+ %t2 = fadd double %t1, %v3
+ %t3 = fadd double %t2, %v4
+ %t4 = fadd double %t3, %v5
+ %t5 = fadd double %t4, %v6
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t5, ptr %outp, align 8
+ %outp2 = getelementptr inbounds double, ptr %out2, i64 %iv
+ store double %t5, ptr %outp2, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
>From aa92771bc2730440c8b1a02ca4638d46e980a96c Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 15:24:55 +0100
Subject: [PATCH 17/33] Renumber test headers after removing two tests
---
.../runtime-check-group-merging.ll | 38 +++++++++----------
1 file changed, 19 insertions(+), 19 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 06e4a40b0158b..d3477575fb0e5 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -156,7 +156,7 @@ exit:
}
-;; Test 3: Constant offsets only — existing grouping handles this.
+;; Test 2: Constant offsets only — existing grouping handles this.
;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
;; Both with and without flag: 1 check, 2 groups, no predicates.
@@ -256,7 +256,7 @@ exit:
}
-;; Test 4: Cost model rejection — only 2 loads with runtime stride.
+;; Test 3: Cost model rejection — only 2 loads with runtime stride.
;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
@@ -350,7 +350,7 @@ exit:
}
-;; Test 5: Invariant + strided reads from same base -> different access ranges.
+;; Test 4: Invariant + strided reads from same base -> different access ranges.
;; A strided read {%a,+,8} and an invariant read from %a have different
;; access ranges (8*n vs 8), so merging must NOT combine them.
;; Both reads are in the same DepSet (same underlying object, both reads),
@@ -450,7 +450,7 @@ exit:
}
-;; Test 6: A member with no stride term.
+;; Test 5: A member with no stride term.
;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. The base load has no cdj
;; term at all, so its cdj coefficient counts as 0. The candidate comparison
;; must see that 0: the base member (+0) is the high bound, above -cdj and
@@ -560,7 +560,7 @@ exit:
}
-;; Test 7: Predicated accesses are rejected from stencil merging.
+;; Test 6: Predicated accesses are rejected from stencil merging.
;; Models the dilateKernel pattern (llvm-test-suite MicroBenchmarks/ImageProcessing/Dilate):
;; 3 loads at {-cdj, 0, +cdj} where the -cdj and +cdj loads are conditional.
;; Predicated loads have overapproximated SCEV bounds; merging would widen
@@ -697,7 +697,7 @@ exit:
}
-;; Test 8: Write pointer in a group prevents stencil merging.
+;; Test 7: Write pointer in a group prevents stencil merging.
;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
;; All three share the same base %a (same DepSet). The write prevents merging.
;; Both modes: groups stay separate.
@@ -838,7 +838,7 @@ exit:
}
-;; Test 10: Shared stride predicate deduplication across two DepSets.
+;; Test 8: Shared stride predicate deduplication across two DepSets.
;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
;; should be emitted, not 2.
@@ -991,7 +991,7 @@ exit:
}
-;; Test 11: Known-positive stride skips predicate emission.
+;; Test 9: Known-positive stride skips predicate emission.
;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
@@ -1099,7 +1099,7 @@ exit:
ret void
}
-;; Test 12: loop-invariant pointer is computed in the preheader.
+;; Test 10: loop-invariant pointer is computed in the preheader.
;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
;; read, so we reach the predicated-access check. We must look at the load,
@@ -1188,7 +1188,7 @@ exit:
}
-;; Test 13: predicated access whose address is computed in the header.
+;; Test 11: predicated access whose address is computed in the header.
;; The stride GEPs are in the header, which dominates the latch, but the loads
;; are in a conditional block guarded by %c. So the accesses are predicated and
;; must not be merged. We must look at the load block, which is conditional,
@@ -1320,7 +1320,7 @@ exit:
ret void
}
-;; Test 14: equal access ranges but different recurrence steps -> no merge.
+;; Test 12: equal access ranges but different recurrence steps -> no merge.
;; In a single-iteration loop every access spans just its element size, so the
;; ranges are equal even though the steps differ (8 vs 16). The access-range
;; check passes here, so the step check is the guard that stops the merge. Both
@@ -1409,7 +1409,7 @@ exit:
ret void
}
-;; Test 15: %gep Start/End expressions are min/max expressions.
+;; Test 13: %gep Start/End expressions are min/max expressions.
;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
; MERGE-LABEL: 'stencil_merge_range_could_not_compute'
@@ -1496,7 +1496,7 @@ done:
declare i64 @llvm.smax.i64(i64, i64)
-;; Test 16: A stride that is itself a sum (s1 + s2).
+;; Test 14: A stride that is itself a sum (s1 + s2).
;; s1 = smax(s1in, 1), s2 = smax(s2in, 1) (known positive); 3 loads from %p at
;; offsets {0, s1, s1 + s2}, store to %q.
;; One member offset is (-1 * (s1 + s2)). decomposeStencilOffset distributes
@@ -1607,7 +1607,7 @@ done:
}
-;; Test 17: A member that uses two strides at once.
+;; Test 15: A member that uses two strides at once.
;; 6 loads from %a: +0, -5*s1, +5*s1, -5*s2, +5*s2, and the mixed
;; +5*s1+5*s2. Store to %out.
;; The members at -5*s1 and -5*s2 can each be the lowest address: which one
@@ -1768,7 +1768,7 @@ exit:
ret void
}
-;; Test 18: Too many candidate members - the cost model rejects the merge.
+;; Test 16: Too many candidate members - the cost model rejects the merge.
;; 8 loads from %a at offsets {-s1, +s1, -s2, +s2, -s3, +s3, -s4, +s4}.
;; The strides are unrelated, so all four -s_i members can be the lowest
;; address and all four +s_i members can be the highest. The merged bounds
@@ -1987,7 +1987,7 @@ exit:
ret void
}
-;; Test 19: One member's offset stays factored as (64 * (s1 + s2)).
+;; Test 17: One member's offset stays factored as (64 * (s1 + s2)).
;; 4 loads from %p at byte offsets {0, 64*s1, 64*s2, 64*(s1+s2)}; the last
;; offset is computed as (s1 + s2) * 64, and SCEV keeps the factored form.
;; Stores to %q and %q2.
@@ -2167,7 +2167,7 @@ done:
ret void
}
-;; Test 20: The depth cap keeps a deep term as one opaque key.
+;; Test 18: The depth cap keeps a deep term as one opaque key.
;; 3 loads from %p at byte offsets {0, 64*s1, 8 + 64*(s1 + s2 + 4*s3)}.
;; The deep offset is an add (depth 0) holding 64*(...) (depth 1) holding a
;; 3-operand add (depth 2) whose term 4*s3 sits at depth 3.
@@ -2388,7 +2388,7 @@ done:
}
-;; Test 21: A constant-offset member and a stride member share the bounds.
+;; Test 19: A constant-offset member and a stride member share the bounds.
;; 4 loads from %a at byte offsets {-16, +16, -cdj, +cdj}. Stores to %out
;; and %out2.
;; The member at -16 has the smaller constant; the member at -cdj has the
@@ -2555,7 +2555,7 @@ exit:
}
-;; Test 22: Three strides in a star shape - multi-operand umin and umax.
+;; Test 20: Three strides in a star shape - multi-operand umin and umax.
;; 7 loads from %a at byte offsets {0, -s1, +s1, -s2, +s2, -s3, +s3}.
;; Stores to %out and %out2.
;; Any of -s1, -s2, -s3 can be the lowest address: which one is lowest
>From 00c1184202623e7439cae78b37d2c01a505ae145 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 15:57:48 +0100
Subject: [PATCH 18/33] Share CHECK lines between the merge and no-merge RUN
lines
---
.../runtime-check-group-merging-i128.ll | 119 +-
.../runtime-check-group-merging.ll | 1203 ++++++-----------
2 files changed, 442 insertions(+), 880 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
index 0799360bf346e..4a5906e5da552 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -1,87 +1,48 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck %s
target datalayout = "e-p:128:128"
define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj) {
-; MERGE-LABEL: 'stencil_wide_index_huge_coeff'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
-; MERGE-NEXT: Check 2:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
-; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
-; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
-; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
-; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
-; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'stencil_wide_index_huge_coeff'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
-; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'stencil_wide_index_huge_coeff'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
+; CHECK-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
+; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
br label %loop
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index d3477575fb0e5..b86f6ed5d0a05 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -161,63 +161,34 @@ exit:
;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
;; Both with and without flag: 1 check, 2 groups, no predicates.
define void @constant_offsets_only(ptr %a, ptr %out, i64 %n) {
-; MERGE-LABEL: 'constant_offsets_only'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; MERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'constant_offsets_only'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; NOMERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'constant_offsets_only'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
%cmp = icmp sgt i64 %n, 8
@@ -260,67 +231,36 @@ exit:
;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
-; MERGE-LABEL: 'cost_model_rejection'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
-; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'cost_model_rejection'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'cost_model_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
%cmp = icmp sgt i64 %n, 4
@@ -357,67 +297,36 @@ exit:
;; so they pass the write-access guard and reach the access-range check.
;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
-; MERGE-LABEL: 'different_steps_no_merge'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: ptr %a
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: %a
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'different_steps_no_merge'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: ptr %a
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: %a
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'different_steps_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %a
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: %a
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
; GRP0: store to %out (write, different DepSet):
; GRP1: strided read from %a (step=8):
@@ -567,83 +476,44 @@ exit:
;; them further, causing false runtime overlap detection.
;; Both modes: 3 checks (groups stay separate), no predicates.
define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c1, i1 %c2) {
-; MERGE-LABEL: 'predicated_access_rejection'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: Check 2:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
-; MERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
-; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'predicated_access_rejection'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'predicated_access_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; CHECK-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
; 3 checks: each temp group separately vs %out (not merged due to predication):
; 4 groups: 1 for %out, 3 separate temp groups:
@@ -702,103 +572,54 @@ exit:
;; All three share the same base %a (same DepSet). The write prevents merging.
;; Both modes: groups stay separate.
define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
-; MERGE-LABEL: 'write_in_group_rejection'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Check 2:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: Check 3:
-; MERGE-NEXT: Comparing group GRP1:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Check 4:
-; MERGE-NEXT: Comparing group GRP1:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
-; MERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
-; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'write_in_group_rejection'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Check 3:
-; NOMERGE-NEXT: Comparing group GRP1:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Check 4:
-; NOMERGE-NEXT: Comparing group GRP1:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
-; NOMERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
-; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'write_in_group_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
; 5 checks: write group vs all others, %out vs the two read groups:
; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
@@ -1095,78 +916,47 @@ loop:
%cond = icmp slt i64 %iv.next, %sub
br i1 %cond, label %loop, label %exit
-exit:
- ret void
-}
-
-;; Test 10: loop-invariant pointer is computed in the preheader.
-;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
-;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
-;; read, so we reach the predicated-access check. We must look at the load,
-;; which is in the loop, not at the pointer, which is in the preheader -
-;; otherwise we crash.
-define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
-; MERGE-LABEL: 'invariant_pointer_in_preheader'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: (64 + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
-; MERGE-NEXT: Member: (%cdj + %a)
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'invariant_pointer_in_preheader'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: (64 + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
-; NOMERGE-NEXT: Member: (%cdj + %a)
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+exit:
+ ret void
+}
+
+;; Test 10: loop-invariant pointer is computed in the preheader.
+;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
+;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
+;; read, so we reach the predicated-access check. We must look at the load,
+;; which is in the loop, not at the pointer, which is in the preheader -
+;; otherwise we crash.
+define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
+; CHECK-LABEL: 'invariant_pointer_in_preheader'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (64 + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; CHECK-NEXT: Member: (%cdj + %a)
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: {%a,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
%inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
@@ -1194,99 +984,52 @@ exit:
;; must not be merged. We must look at the load block, which is conditional,
;; not at the GEP block, which is the header. Both modes: groups stay separate.
define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c) {
-; MERGE-LABEL: 'predicated_access_hoisted_address'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
-; MERGE-NEXT: Check 2:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
-; MERGE-NEXT: Check 3:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP4:
-; MERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
-; MERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
-; MERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
-; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
-; MERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
-; MERGE-NEXT: Group GRP4:
-; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; MERGE-NEXT: Member: {%a,+,8}<%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'predicated_access_hoisted_address'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
-; NOMERGE-NEXT: Check 3:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
-; NOMERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
-; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
-; NOMERGE-NEXT: Member: {%a,+,8}<%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'predicated_access_hoisted_address'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; CHECK-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; CHECK-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; CHECK-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
br label %loop
@@ -1326,67 +1069,36 @@ exit:
;; check passes here, so the step check is the guard that stops the merge. Both
;; modes: groups stay separate.
define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
-; MERGE-LABEL: 'equal_range_different_step_no_merge'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: (8 + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
-; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'equal_range_different_step_no_merge'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: (8 + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
-; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'equal_range_different_step_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (8 + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: {%a,+,16}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
br label %loop
@@ -1412,69 +1124,37 @@ exit:
;; Test 13: %gep Start/End expressions are min/max expressions.
;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
-; MERGE-LABEL: 'stencil_merge_range_could_not_compute'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: ptr %q
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: ptr %q
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: ptr %p
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %q High: (1 + %q))
-; MERGE-NEXT: Member: %q
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
-; MERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %p High: (1 + %p))
-; MERGE-NEXT: Member: %p
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'stencil_merge_range_could_not_compute'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: ptr %q
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: ptr %q
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: ptr %p
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %q High: (1 + %q))
-; NOMERGE-NEXT: Member: %q
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
-; NOMERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %p High: (1 + %p))
-; NOMERGE-NEXT: Member: %p
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'stencil_merge_range_could_not_compute'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: ptr %q
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: ptr %q
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %p
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %q High: (1 + %q))
+; CHECK-NEXT: Member: %q
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; CHECK-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %p High: (1 + %p))
+; CHECK-NEXT: Member: %p
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
%step = add i64 %m, 1
@@ -1777,163 +1457,84 @@ exit:
;; operands + 3 umax operands = 11. 11 >= 8, so no merge, even with
;; -stencil-runtime-check-merge=force.
define void @too_many_candidates_no_merge(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2, i64 %s3, i64 %s4) {
-; MERGE-LABEL: 'too_many_candidates_no_merge'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
-; MERGE-NEXT: Check 2:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP3:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
-; MERGE-NEXT: Check 3:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP4:
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
-; MERGE-NEXT: Check 4:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP5:
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
-; MERGE-NEXT: Check 5:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP6:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
-; MERGE-NEXT: Check 6:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP7:
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
-; MERGE-NEXT: Check 7:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP8:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
-; MERGE-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
-; MERGE-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
-; MERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP4:
-; MERGE-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
-; MERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP5:
-; MERGE-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
-; MERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP6:
-; MERGE-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
-; MERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP7:
-; MERGE-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
-; MERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Group GRP8:
-; MERGE-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
-; MERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'too_many_candidates_no_merge'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
-; NOMERGE-NEXT: Check 3:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
-; NOMERGE-NEXT: Check 4:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
-; NOMERGE-NEXT: Check 5:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
-; NOMERGE-NEXT: Check 6:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP7:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
-; NOMERGE-NEXT: Check 7:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP8:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
-; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
-; NOMERGE-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
-; NOMERGE-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
-; NOMERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
-; NOMERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP5:
-; NOMERGE-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
-; NOMERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP6:
-; NOMERGE-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
-; NOMERGE-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP7:
-; NOMERGE-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
-; NOMERGE-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP8:
-; NOMERGE-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
-; NOMERGE-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+; CHECK-LABEL: 'too_many_candidates_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP5:
+; CHECK-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+; CHECK-NEXT: Check 5:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP6:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; CHECK-NEXT: Check 6:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP7:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+; CHECK-NEXT: Check 7:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP8:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
+; CHECK-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
+; CHECK-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP5:
+; CHECK-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
+; CHECK-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP6:
+; CHECK-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP7:
+; CHECK-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
+; CHECK-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP8:
+; CHECK-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
%cmp = icmp sgt i64 %n, 8
>From e60e3eddf97fba2b205b84f61c614923f7cb4500 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 16:22:35 +0100
Subject: [PATCH 19/33] Give every load in the tests a value that feeds the
store
---
.../runtime-check-group-merging.ll | 46 +++++++++++--------
1 file changed, 27 insertions(+), 19 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index b86f6ed5d0a05..213acf9018e5f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1163,10 +1163,11 @@ entry:
loop:
%off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
%i = phi i1 [ false, %entry ], [ true, %loop ]
- load i8, ptr %p
+ %v0 = load i8, ptr %p
%gep = getelementptr i8, ptr %p, i64 %off
- load i8, ptr %gep
- store i8 0, ptr %q
+ %v1 = load i8, ptr %gep
+ %s = add i8 %v0, %v1
+ store i8 %s, ptr %q
%off.next = add i64 %off, %step
br i1 %i, label %done, label %loop
@@ -1271,13 +1272,15 @@ loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%idx = mul i64 %iv, 8
%a0 = getelementptr i8, ptr %p, i64 %idx
- load i8, ptr %a0
+ %v0 = load i8, ptr %a0
%a1 = getelementptr i8, ptr %p1, i64 %idx
- load i8, ptr %a1
+ %v1 = load i8, ptr %a1
%a2 = getelementptr i8, ptr %p2, i64 %idx
- load i8, ptr %a2
+ %v2 = load i8, ptr %a2
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
%aq = getelementptr i8, ptr %q, i64 %idx
- store i8 0, ptr %aq
+ store i8 %s, ptr %aq
%iv.next = add i64 %iv, 1
%c = icmp eq i64 %iv, %n
br i1 %c, label %done, label %loop
@@ -1749,17 +1752,20 @@ loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%idx = mul i64 %iv, 8
%a0 = getelementptr i8, ptr %p, i64 %idx
- load i8, ptr %a0
+ %v0 = load i8, ptr %a0
%a1 = getelementptr i8, ptr %p1, i64 %idx
- load i8, ptr %a1
+ %v1 = load i8, ptr %a1
%a2 = getelementptr i8, ptr %p2, i64 %idx
- load i8, ptr %a2
+ %v2 = load i8, ptr %a2
%a3 = getelementptr i8, ptr %p3, i64 %idx
- load i8, ptr %a3
+ %v3 = load i8, ptr %a3
+ %s01 = add i8 %v0, %v1
+ %s23 = add i8 %v2, %v3
+ %s = add i8 %s01, %s23
%aq = getelementptr i8, ptr %q, i64 %idx
- store i8 0, ptr %aq
+ store i8 %s, ptr %aq
%aq2 = getelementptr i8, ptr %q2, i64 %idx
- store i8 0, ptr %aq2
+ store i8 %s, ptr %aq2
%iv.next = add i64 %iv, 1
%c = icmp eq i64 %iv, %n
br i1 %c, label %done, label %loop
@@ -1969,17 +1975,19 @@ loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%idx = mul i64 %iv, 8
%a2 = getelementptr i8, ptr %p2, i64 %idx
- load i8, ptr %a2
+ %v2 = load i8, ptr %a2
%a1 = getelementptr i8, ptr %p1, i64 %idx
- load i8, ptr %a1
+ %v1 = load i8, ptr %a1
%a0 = getelementptr i8, ptr %p, i64 %idx
- load i8, ptr %a0
+ %v0 = load i8, ptr %a0
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
%aq = getelementptr i8, ptr %q, i64 %idx
- store i8 0, ptr %aq
+ store i8 %s, ptr %aq
%aq2 = getelementptr i8, ptr %q2, i64 %idx
- store i8 0, ptr %aq2
+ store i8 %s, ptr %aq2
%aq3 = getelementptr i8, ptr %q3, i64 %idx
- store i8 0, ptr %aq3
+ store i8 %s, ptr %aq3
%iv.next = add i64 %iv, 1
%c = icmp eq i64 %iv, %n
br i1 %c, label %done, label %loop
>From dd8360664f94f50b62b6cde2d2900d4068a5c993 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 16:24:44 +0100
Subject: [PATCH 20/33] Add a test with a member GEP that is not inbounds
---
.../runtime-check-group-merging.ll | 102 ++++++++++++++++++
1 file changed, 102 insertions(+)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 213acf9018e5f..63dfd9b2c63ce 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -2394,3 +2394,105 @@ loop:
exit:
ret void
}
+
+;; Test 21: One member's GEP is not inbounds.
+;; 3 loads from %a at byte offsets {0, cdj, 2*cdj}; the cdj member's GEP has
+;; no inbounds. The merge does not need inbounds facts of its own: the merged
+;; bounds reuse the members' Start/End values, the same values the unmerged
+;; checks would compare, so no new address is computed. The group merges as
+;; usual with the cdj > 0 predicate.
+define void @stencil_merge_no_inbounds_member(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_no_inbounds_member'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; MERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: ((2 * %cdj)<nsw> + (8 * (1 smax %n)) + %a))
+; MERGE-NEXT: Member: {((2 * %cdj)<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_no_inbounds_member'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((2 * %cdj)<nsw> + %a) High: ((2 * %cdj)<nsw> + (8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {((2 * %cdj)<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %v0 = load double, ptr %base, align 8
+
+ %p1 = getelementptr i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
>From c8ba2ba36c8984078691c66f7bdca27967d62a05 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 16:25:02 +0100
Subject: [PATCH 21/33] Add an early-exit loop test
---
.../runtime-check-group-merging.ll | 76 +++++++++++++++++++
1 file changed, 76 insertions(+)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 63dfd9b2c63ce..f232be73f707d 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -2496,3 +2496,79 @@ loop:
exit:
ret void
}
+
+;; Test 22: An early-exit loop; the loads at {cdj, 2*cdj} sit after the
+;; early exit. LAA still emits runtime checks for this loop, but it cannot
+;; compute where the accesses end (every High bound is the unknown-end
+;; marker), so Range = End - Start is not computable and the merge skips
+;; the DepSet. The checks stay exactly as without merging.
+define void @early_exit_no_merge(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; CHECK-LABEL: 'early_exit_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT: Member: {%out,+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (%cdj + %a) High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: %a High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %v0 = load double, ptr %base, align 8
+ %ee = fcmp oeq double %v0, 0.0
+ br i1 %ee, label %exit, label %latch
+
+latch:
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
>From edf3c6ba236f649f61905f57b01f50b992a7a1c3 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 12 Aug 2026 16:25:29 +0100
Subject: [PATCH 22/33] Add a forked-pointer test
---
.../runtime-check-group-merging.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index f232be73f707d..2ea9512102518 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -2572,3 +2572,82 @@ latch:
exit:
ret void
}
+
+;; Test 23: A forked pointer - the last member's offset is a select between
+;; 2*cdj and 8. LAA makes two runtime-check entries for that load, one per
+;; select arm. Each entry is an ordinary member with its own Start and End,
+;; so a merge would cover both arms and stay sound. Here it is simply not
+;; profitable: 3 groups x 1 external check before; 1 check + 1 predicate
+;; + 1 umax operand after (the 8 arm and the 2*cdj arm are incomparable,
+;; so both stay max candidates). 3 -> 3 is no win, the merge is rejected.
+define void @forked_pointer_no_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
+; CHECK-LABEL: 'forked_pointer_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: ((2 * %cdj) + (8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %a High: (8 + (8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %v0 = load double, ptr %base, align 8
+
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %cmp = icmp eq i32 %c, 0
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %off = select i1 %cmp, i64 %pos2cdj, i64 8
+ %p2 = getelementptr i8, ptr %base, i64 %off
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
>From ad543aee47b6c6f6d35833af5e3964326ea08749 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 13 Aug 2026 11:09:01 +0100
Subject: [PATCH 23/33] Rebase, run cost model before building
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 39 +++++++++++--------
.../runtime-check-group-merging.ll | 9 ++---
2 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 75556599c8db9..37ed312335a41 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1383,8 +1383,23 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< ", max=" << MaxCandidates.size() << " of "
<< MemberOffsets.size() << "\n");
- // OffsetTy is an integer type as wide as an address. The umin/umax
- // operands below are member addresses converted to this type.
+ // Local cost model: decide whether replacing this DepSet's groups with the
+ // single merged group actually reduces the number of runtime checks. Run
+ // it before building the merged bounds: a rejected DepSet then creates no
+ // umin/umax expressions that would only be thrown away.
+ auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
+ *this, GroupIndices, MergedGroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates,
+ MinCandidates.size(), MaxCandidates.size());
+ if (ChecksAfter >= ChecksBefore) {
+ LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
+ continue;
+ }
+
+ // OffsetTy is the integer type ptrtoaddr produces for the base pointer.
+ // Every umin/umax operand below must have this one type. A member in an
+ // address space with a different address width would produce a different
+ // type; BuildBound returns null then, and the DepSet is skipped.
Type *OffsetTy = SE->getEffectiveSCEVType(BaseLow->getType());
// Build one side of the merged bounds from its candidate members.
@@ -1394,8 +1409,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// way every value is a real member address, so the merge computes no new
// address and no new overflow is possible.
// SCEV umin/umax needs integer operands, so convert each pointer with
- // ptrtoint. That fails for non-integral pointers; return null then, and
- // the DepSet is skipped.
+ // ptrtoaddr. That fails for pointers whose address bits are not stable;
+ // return null then, and the DepSet is skipped.
const auto BuildBound = [&](ArrayRef<unsigned> Candidates,
bool IsLow) -> const SCEV * {
auto GetBound = [&](unsigned K) {
@@ -1409,8 +1424,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
const SCEV *Bound = GetBound(K);
if (!Bound->getType()->isPointerTy())
return nullptr;
- const SCEV *IntBound = SE->getPtrToIntExpr(Bound, OffsetTy);
- if (isa<SCEVCouldNotCompute>(IntBound))
+ const SCEV *IntBound = SE->getPtrToAddrExpr(Bound);
+ if (isa<SCEVCouldNotCompute>(IntBound) ||
+ IntBound->getType() != OffsetTy)
return nullptr;
Ops.push_back(IntBound);
}
@@ -1427,17 +1443,6 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
<< ", High=" << *MergedHigh << "\n");
-
- // Local cost model: decide whether replacing this DepSet's groups with the
- // single merged group actually reduces the number of runtime checks.
- auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
- *this, GroupIndices, MergedGroupIndices,
- LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates,
- MinCandidates.size(), MaxCandidates.size());
- if (ChecksAfter >= ChecksBefore) {
- LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
- continue;
- }
LLVM_DEBUG(dbgs() << "LAA: Merging, net saving "
<< ChecksBefore - ChecksAfter << "\n");
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 2ea9512102518..de2c3d2c871f2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1145,14 +1145,13 @@ define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
; CHECK-NEXT: Member: %q
; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
-; CHECK-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; CHECK-NEXT: Member: {%p,+,(1 + %m)}<nw><%loop>
; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: %p High: (1 + %p))
; CHECK-NEXT: Member: %p
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
@@ -1319,7 +1318,7 @@ define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %
; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + (ptrtoint ptr %a to i64)) umin (32 + (-5 * %s1) + (ptrtoint ptr %a to i64))) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + (ptrtoaddr ptr %a to i64)) umin (32 + (-5 * %s1) + (ptrtoaddr ptr %a to i64))) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
; MERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
@@ -2043,7 +2042,7 @@ define void @stencil_merge_constant_and_stride_candidates(ptr %a, ptr %out, ptr
; MERGE-NEXT: (Low: (16 + %out2) High: (-16 + (8 * %n) + %out2))
; MERGE-NEXT: Member: {(16 + %out2),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((16 + (-1 * %cdj) + (ptrtoint ptr %a to i64)) umin (ptrtoint ptr %a to i64)) High: (((8 * %n) + (ptrtoint ptr %a to i64)) umax (-16 + (8 * %n) + (ptrtoint ptr %a to i64) + %cdj)))
+; MERGE-NEXT: (Low: ((16 + (-1 * %cdj) + (ptrtoaddr ptr %a to i64)) umin (ptrtoaddr ptr %a to i64)) High: (((8 * %n) + (ptrtoaddr ptr %a to i64)) umax (-16 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %cdj)))
; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
@@ -2214,7 +2213,7 @@ define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n
; MERGE-NEXT: (Low: (32 + %out2) High: (-32 + (8 * %n) + %out2))
; MERGE-NEXT: Member: {(32 + %out2),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((32 + (-1 * %s3) + (ptrtoint ptr %a to i64)) umin (32 + (-1 * %s2) + (ptrtoint ptr %a to i64)) umin (32 + (-1 * %s1) + (ptrtoint ptr %a to i64))) High: ((-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s1) umax (-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s2) umax (-32 + (8 * %n) + (ptrtoint ptr %a to i64) + %s3)))
+; MERGE-NEXT: (Low: ((32 + (-1 * %s3) + (ptrtoaddr ptr %a to i64)) umin (32 + (-1 * %s2) + (ptrtoaddr ptr %a to i64)) umin (32 + (-1 * %s1) + (ptrtoaddr ptr %a to i64))) High: ((-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s1) umax (-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s2) umax (-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s3)))
; MERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
>From a9431bd7709b495f9ce64c1f3dbaf180213877af Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 13 Aug 2026 16:17:42 +0100
Subject: [PATCH 24/33] Make the forked-pointer test merge instead of losing on
cost
---
.../runtime-check-group-merging.ll | 139 +++++++++++-------
1 file changed, 89 insertions(+), 50 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index de2c3d2c871f2..83b6ed48f6d66 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -2572,54 +2572,92 @@ exit:
ret void
}
-;; Test 23: A forked pointer - the last member's offset is a select between
-;; 2*cdj and 8. LAA makes two runtime-check entries for that load, one per
-;; select arm. Each entry is an ordinary member with its own Start and End,
-;; so a merge would cover both arms and stay sound. Here it is simply not
-;; profitable: 3 groups x 1 external check before; 1 check + 1 predicate
-;; + 1 umax operand after (the 8 arm and the 2*cdj arm are incomparable,
-;; so both stay max candidates). 3 -> 3 is no win, the merge is rejected.
-define void @forked_pointer_no_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
-; CHECK-LABEL: 'forked_pointer_no_merge'
-; CHECK-NEXT: loop:
-; CHECK-NEXT: Memory dependences are safe with run-time checks
-; CHECK-NEXT: Dependences:
-; CHECK-NEXT: Run-time memory checks:
-; CHECK-NEXT: Check 0:
-; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; CHECK-NEXT: Check 1:
-; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; CHECK-NEXT: Check 2:
-; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; CHECK-NEXT: Against group GRP3:
-; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; CHECK-NEXT: Grouped accesses:
-; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
-; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
-; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: ((2 * %cdj) + (8 * (1 smax %n)) + %a))
-; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
-; CHECK-NEXT: Group GRP2:
-; CHECK-NEXT: (Low: %a High: (8 + (8 * (1 smax %n)) + %a))
-; CHECK-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
-; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
-; CHECK-NEXT: Group GRP3:
-; CHECK-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
-; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT: SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT: Expressions re-written:
+;; Test 23: A forked pointer. The last load's address is a select between
+;; base + 2*cdj and base + 3*cdj. LAA splits that one load into two
+;; runtime-check entries, one per select arm, each with its own Start and
+;; End. The merge sees them as two ordinary members and needs no special
+;; case for the fork. The merged High comes from the 3*cdj arm, so both
+;; arms stay inside the merged bounds.
+;; With merge: 4 groups become 1 group, 4 checks become 1 check plus the
+;; cdj > 0 predicate.
+;; Without merge: 4 groups, 4 checks, no predicate.
+define void @forked_pointer_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
+; MERGE-LABEL: 'forked_pointer_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: ((3 * %cdj) + (8 * (1 smax %n)) + %a))
+; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'forked_pointer_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((2 * %cdj) + (8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((3 * %cdj) + %a) High: ((3 * %cdj) + (8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {((3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
;
entry:
br label %loop
@@ -2633,8 +2671,9 @@ loop:
%v1 = load double, ptr %p1, align 8
%cmp = icmp eq i32 %c, 0
- %pos2cdj = mul nsw i64 %cdj, 2
- %off = select i1 %cmp, i64 %pos2cdj, i64 8
+ %two.cdj = mul nsw i64 %cdj, 2
+ %three.cdj = mul nsw i64 %cdj, 3
+ %off = select i1 %cmp, i64 %two.cdj, i64 %three.cdj
%p2 = getelementptr i8, ptr %base, i64 %off
%v2 = load double, ptr %p2, align 8
>From 4bd184074c97b782d7877070cff92331ceeeb640 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 21 Aug 2026 18:43:01 +0100
Subject: [PATCH 25/33] Address comments
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 110 ++++--
.../runtime-check-group-merging.ll | 360 +++++++++++++++---
2 files changed, 379 insertions(+), 91 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 37ed312335a41..77ece3a445c90 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -13,6 +13,7 @@
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EquivalenceClasses.h"
#include "llvm/ADT/MapVector.h"
@@ -855,25 +856,50 @@ struct StencilDecomposition {
SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
};
-/// Recursion cap for addScaledStencilTerm. Depth 3 covers the deepest chain
-/// we distribute: a top-level add (depth 0), a constant-times-X term inside
-/// it (depth 1), a factored add inside that (depth 2), and the leaves
-/// (depth 3). SCEV can nest deeper; a deeper term stays one opaque key with
-/// its multiplier, which is safe but less precise. Constants and leaf
-/// strides are handled at any depth; only the add and constant-times-X
-/// cases recurse.
+/// Recursion cap for addScaledStencilTerm. Depth counts how deep a term
+/// sits inside the offset expression. Example, the offset
+/// 8 + (64 * (s1 + s2 + (4 * s3)))
+/// is visited like this:
+/// depth 0: the whole add
+/// depth 1: its operands 8 and (64 * (s1 + s2 + (4 * s3)))
+/// depth 2: (s1 + s2 + (4 * s3)), the operand of the multiply
+/// depth 3: s1, s2 and (4 * s3), the operands of that add
+/// At depth 3 addScaledStencilTerm stops going deeper. s1 and s2 are plain
+/// strides anyway. (4 * s3) is not split into 4 times s3: it becomes one
+/// stride key as it is, with coefficient 64. The result is Constant = 8
+/// and coefficients {s1: 64, s2: 64, (4 * s3): 64}.
+/// Three levels cover the stencil offsets we care about: a top-level add,
+/// a constant times a sum inside it, and the strides in that sum. A deeper
+/// term is kept whole as one stride key. The merge does not care what is
+/// inside a key. It only needs a loop-invariant value with a
+/// positive-stride predicate, and a whole term has both. The only cost is
+/// precision, when another member uses a part of that term, here s3 alone,
+/// as a key of its own. isNeverAbove sees two unrelated keys, so a member
+/// that is in fact always lower or higher may stay a candidate.
constexpr unsigned MaxStencilDecomposeDepth = 3;
-/// Fold one term of a stencil offset into \p D, scaled by \p Mult:
-/// constant K -> D.Constant += Mult * K
-/// (K * X) -> recurse into X with multiplier Mult * K
-/// (a + b + ...) -> recurse into each operand with the same Mult
-/// anything else -> D.Coefficients[Term] += Mult
-/// Example: 8 + (-64 * (s1 + s2)) + (-32 * s1) gives Constant = 8 and
-/// coefficients {s1: -96, s2: -64}.
+/// Add one term of a stencil offset to \p D. \p Mult is the factor in
+/// front of the term; the top-level call passes 1.
+/// Example: the offset 8 + (-64 * (s1 + s2)) + (-32 * s1), Mult = 1. It is
+/// an add, so each operand is visited in turn with the same Mult = 1:
+/// 8 a constant: D.Constant += 1 * 8
+/// (-64 * (s1 + s2)) a constant times X: visit X = (s1 + s2) with
+/// Mult = 1 * -64. X is an add, so each operand is
+/// visited with Mult = -64:
+/// s1 a stride: D.Coefficients[s1] += -64
+/// s2 a stride: D.Coefficients[s2] += -64
+/// (-32 * s1) a constant times X: visit X = s1 with Mult = -32:
+/// s1 a stride: D.Coefficients[s1] += -32
+/// Result: Constant = 8, Coefficients {s1: -96, s2: -64}. The -64 and the
+/// -32 for s1 come from two different terms and add up in the map.
+/// So, by the kind of term:
+/// constant K D.Constant += Mult * K
+/// (K * X) visit X with Mult * K
+/// (a + b + ...) visit a, b, ... each with this same Mult
+/// anything else a stride key: D.Coefficients[Term] += Mult
/// The two recursive cases only fire while Depth is below
-/// MaxStencilDecomposeDepth. At the cap a term becomes one opaque stride key
-/// with coefficient Mult; that is not a bailout.
+/// MaxStencilDecomposeDepth. At the cap, (K * X) and (a + b + ...) are
+/// stride keys like anything else; that is not a bailout.
/// Returns false when a constant does not fit in int64_t or an update
/// overflows. The caller then drops the whole decomposition.
static bool addScaledStencilTerm(const SCEV *Term, int64_t Mult, unsigned Depth,
@@ -935,28 +961,30 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return D;
}
-/// Return true if member \p A can never sit at a higher address than member
-/// \p B, for any whole-number stride values of 1 or more.
+/// Return true if A's address is never higher than B's address.
+/// A and B are these sums:
+/// A = A.Constant + CoefA_1 * stride_1 + CoefA_2 * stride_2 + ...
+/// B = B.Constant + CoefB_1 * stride_1 + CoefB_2 * stride_2 + ...
+/// A stride missing from a member's map has coefficient 0. Every stride
+/// is 1 or more: strides are integers, and the merge adds an "s > 0"
+/// predicate for each stride that is not already known positive.
/// Example:
/// A: 0 - 80*s1
/// B: -40 - 40*s1
-/// At s1 = 1 both sit at -80. For bigger s1, A's steeper slope puts it
-/// lower. So A is never above B.
-/// The general rule needs two facts:
-/// - every coefficient of A is <= the matching coefficient of B, and
-/// - A's address at all-strides-1 (Constant plus the sum of all
-/// coefficients) is <= B's address at that same point.
-/// Strides of 1 or more is a safe assumption: every stride is an integer,
-/// and a merged check only runs with an "s > 0" predicate on each stride
-/// (or the stride is already known positive), so every stride is at least 1.
-/// The decompositions hold signed offsets, but the merged bounds compare
-/// unsigned addresses at runtime. The order carries over: both members
-/// read the same underlying object, and an object never wraps around the
-/// address space, so the member with the smaller offset sits at the
-/// smaller address.
-/// A stride missing from a member's map counts as coefficient 0.
-/// Returns false when a sum overflows. The caller then keeps the member,
-/// which is the safe direction.
+/// At s1 = 1 both are -80. For bigger s1, A goes down faster. So A is
+/// never above B.
+/// The rule checks two things:
+/// 1. CoefA_i <= CoefB_i for every stride. So when a stride grows, B - A
+/// grows too, or stays the same.
+/// 2. B - A >= 0 when every stride is 1. That is ACorner <= BCorner, with
+/// ACorner = A.Constant + the sum of all CoefA_i, same for BCorner.
+/// B - A starts at or above zero and never goes down, so B - A >= 0 for
+/// all stride values.
+/// Offsets are signed and addresses are unsigned, but both members read
+/// one object, and an object does not wrap around the address space, so
+/// the smaller offset is the smaller address.
+/// Returns false when ACorner or BCorner overflows int64_t. The caller
+/// then keeps the member, which is the safe side.
static bool isNeverAbove(const StencilDecomposition &A,
const StencilDecomposition &B) {
int64_t ACorner = A.Constant, BCorner = B.Constant;
@@ -996,25 +1024,25 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
};
// Skipping a beaten member loses nothing: Beats is transitive, so
// whoever beat it also beats anyone it would have beaten.
- SmallVector<bool, 8> Beaten(Offsets.size(), false);
+ BitVector Beaten(Offsets.size());
for (unsigned K = 0; K < Offsets.size(); ++K) {
- if (Beaten[K])
+ if (Beaten.test(K))
continue;
for (unsigned J = K + 1; J < Offsets.size(); ++J) {
- if (Beaten[J])
+ if (Beaten.test(J))
continue;
// Checking K first settles ties: on equal offsets K survives.
if (Beats(K, J)) {
- Beaten[J] = true;
+ Beaten.set(J);
} else if (Beats(J, K)) {
- Beaten[K] = true;
+ Beaten.set(K);
break;
}
}
}
SmallVector<unsigned, 4> Candidates;
for (unsigned K = 0; K < Offsets.size(); ++K)
- if (!Beaten[K])
+ if (!Beaten.test(K))
Candidates.push_back(K);
return Candidates;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 83b6ed48f6d66..36fa91678b5b8 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -293,8 +293,6 @@ exit:
;; Test 4: Invariant + strided reads from same base -> different access ranges.
;; A strided read {%a,+,8} and an invariant read from %a have different
;; access ranges (8*n vs 8), so merging must NOT combine them.
-;; Both reads are in the same DepSet (same underlying object, both reads),
-;; so they pass the write-access guard and reach the access-range check.
;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; CHECK-LABEL: 'different_steps_no_merge'
@@ -1065,9 +1063,7 @@ exit:
;; Test 12: equal access ranges but different recurrence steps -> no merge.
;; In a single-iteration loop every access spans just its element size, so the
-;; ranges are equal even though the steps differ (8 vs 16). The access-range
-;; check passes here, so the step check is the guard that stops the merge. Both
-;; modes: groups stay separate.
+;; ranges are equal even though the steps differ (8 vs 16).
define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
; CHECK-LABEL: 'equal_range_different_step_no_merge'
; CHECK-NEXT: loop:
@@ -1183,7 +1179,7 @@ declare i64 @llvm.smax.i64(i64, i64)
;; the -1, so that member is keyed on s1 and s2 like the others. It then
;; defines the low bound alone: Low is plain %p with no umin. Both strides
;; are known positive, so no predicate is emitted.
-;; The merge saves 3 loads x 1 external check = 3 and costs 1 check.
+;; Checks: 3 before the merge, 1 after.
define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
; MERGE-LABEL: 'stencil_merge_summed_stride'
; MERGE-NEXT: loop:
@@ -1292,11 +1288,9 @@ done:
;; Test 15: A member that uses two strides at once.
;; 6 loads from %a: +0, -5*s1, +5*s1, -5*s2, +5*s2, and the mixed
;; +5*s1+5*s2. Store to %out.
-;; The members at -5*s1 and -5*s2 can each be the lowest address: which one
-;; is lower depends on whether s1 or s2 is bigger. So Low is a umin over
-;; those two members. The mixed member is at or above every other member in
-;; both strides at once, so High is just that member's own End - no umax.
-;; No bound is an invented corner like base - 5*s1 - 5*s2.
+;; Low is a umin over the members at -5*s1 and -5*s2: which one is lower
+;; depends on s1 and s2. High is the End of the +5*s1+5*s2 member, because it
+;; is always above every other member. No umax.
define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2) {
; MERGE-LABEL: 'stencil_merge_mixed_member'
; MERGE-NEXT: loop:
@@ -1594,14 +1588,12 @@ exit:
;; 4 loads from %p at byte offsets {0, 64*s1, 64*s2, 64*(s1+s2)}; the last
;; offset is computed as (s1 + s2) * 64, and SCEV keeps the factored form.
;; Stores to %q and %q2.
-;; decomposeStencilOffset distributes the 64, so the factored member is keyed
-;; on s1 and s2 like the others. With strides of at least 1 that member sits
-;; at or above every other member, and the member at offset 0 sits at or
-;; below every other member. So High is the factored member's own End, Low is
-;; plain %p with no umin, and only two predicates (s1 > 0, s2 > 0) are
-;; emitted - no (s1 + s2) > 0.
-;; The two stores keep the merge profitable: the merge saves 4 loads x 2
-;; external checks = 8 and costs 2 checks + 2 predicates = 4.
+;; The depth cap is not reached here, so the 64 is multiplied into the sum
+;; and the member is keyed on s1 and s2 like the others. That is enough to
+;; prove it is the highest member and %p is the lowest. So the merged bounds
+;; are simple: Low is %p, High is the End of the 64*(s1+s2) member, no umin
+;; or umax, and only two predicates, s1 > 0 and s2 > 0.
+;; Two stores, so the cost model accepts the merge: 8 checks before, 4 after.
define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i64 %s2, i64 %n) {
; MERGE-LABEL: 'stencil_merge_factored_diagonal'
; MERGE-NEXT: loop:
@@ -1773,24 +1765,13 @@ done:
ret void
}
-;; Test 18: The depth cap keeps a deep term as one opaque key.
+;; Test 18: A term at the depth cap stays one stride key.
;; 3 loads from %p at byte offsets {0, 64*s1, 8 + 64*(s1 + s2 + 4*s3)}.
-;; The deep offset is an add (depth 0) holding 64*(...) (depth 1) holding a
-;; 3-operand add (depth 2) whose term 4*s3 sits at depth 3.
-;; SCEV keeps 64*(s1 + s2 + 4*s3) factored: it only distributes a constant
-;; over a 2-operand add.
-;; decomposeStencilOffset distributes down to depth 3 and then stops: s1 and
-;; s2 get their own keys, and (4 * s3) stays one opaque key. The
-;; decomposition still succeeds - the cap is not a bailout - so the group
-;; still merges.
-;; Predicates: s1 > 0, s2 > 0, and (4 * s3) > 0 for the opaque key.
-;; The deep load comes first in the loop on purpose. That makes %p the
-;; base of the relative decomposition. With the deep member as base, SCEV
-;; cancellation flattens 4*s3 into a plain s3 coefficient and the cap is
-;; never tested. If the (4 * %s3) predicate ever disappears from the CHECK
-;; lines, the test has stopped covering the cap.
-;; The three stores keep the merge profitable: the merge saves 3 loads x 3
-;; external checks = 9 and costs 3 checks + 3 predicates = 6.
+;; Stores to %q, %q2 and %q3.
+;; In the last offset, 4*s3 sits at depth 3, the cap. It is not split into
+;; 4 times s3. The keys are s1, s2 and (4 * s3), each with a > 0 predicate,
+;; and the group still merges.
+;; Three stores, so the cost model accepts the merge: 9 checks before, 6 after.
define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1, i64 %s2, i64 %s3, i64 %n) {
; MERGE-LABEL: 'stencil_merge_depth_cap'
; MERGE-NEXT: loop:
@@ -2004,9 +1985,7 @@ done:
;; positive cdj: at cdj = 1 the -16 member is lower, at cdj = 100 the -cdj
;; member is lower. So Low is a umin over those two members. The +16 and
;; +cdj members mirror this on the high side, so High is a umax over them.
-;; The two constant-offset loads share one group before the merge, so the
-;; merge saves 3 groups x 2 external checks = 6 and costs 2 checks + 1
-;; predicate + 1 umin operand + 1 umax operand = 5.
+;; Checks: 6 before the merge, 5 after.
define void @stencil_merge_constant_and_stride_candidates(ptr %a, ptr %out, ptr %out2, i64 %n, i64 %cdj) {
; MERGE-LABEL: 'stencil_merge_constant_and_stride_candidates'
; MERGE-NEXT: loop:
@@ -2170,8 +2149,7 @@ exit:
;; depends on which stride is biggest. So Low is a umin over those three
;; members, and High is a umax over +s1, +s2, +s3. The center member at +0
;; sits between -s1 and +s1, so it is never a bound.
-;; The merge saves 7 groups x 2 external checks = 14 and costs 2 checks +
-;; 3 predicates + 2 umin operands + 2 umax operands = 9.
+;; Checks: 14 before the merge, 9 after.
define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n, i64 %s1, i64 %s2, i64 %s3) {
; MERGE-LABEL: 'stencil_merge_three_stride_star'
; MERGE-NEXT: loop:
@@ -2572,15 +2550,14 @@ exit:
ret void
}
-;; Test 23: A forked pointer. The last load's address is a select between
-;; base + 2*cdj and base + 3*cdj. LAA splits that one load into two
-;; runtime-check entries, one per select arm, each with its own Start and
-;; End. The merge sees them as two ordinary members and needs no special
-;; case for the fork. The merged High comes from the 3*cdj arm, so both
-;; arms stay inside the merged bounds.
-;; With merge: 4 groups become 1 group, 4 checks become 1 check plus the
-;; cdj > 0 predicate.
-;; Without merge: 4 groups, 4 checks, no predicate.
+;; Test 23: A forked pointer: one load's address is a select.
+;; 3 loads from %a at byte offsets {0, cdj, select(2*cdj, 3*cdj)}. Store to
+;; %out.
+;; LAA turns the select load into two members, one per arm, each with its
+;; own Start and End. The merge treats them like any other member. High is
+;; the End of the 3*cdj arm, so both arms are inside the merged bounds.
+;; With merge: 1 group, 1 check, 1 predicate (cdj > 0).
+;; Without merge: 4 groups, 4 checks, no predicates.
define void @forked_pointer_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
; MERGE-LABEL: 'forked_pointer_merge'
; MERGE-NEXT: loop:
@@ -2689,3 +2666,286 @@ loop:
exit:
ret void
}
+
+;; Test 24: A sum at the depth cap becomes one stride key, next to a plain
+;; key it overlaps with.
+;; 3 loads from %p at byte offsets {0, 64*s1, 64*(s3 + s4 + 65*(s1 + s2))}.
+;; Stores to %q, %q2, %q3 and %q4.
+;; In the last offset, (s1 + s2) sits at depth 3, the cap. It is not split
+;; into s1 and s2: it becomes one stride key, like a new stride. So the keys
+;; are s1 (from the second load), s3, s4 and (s1 + s2), each with a > 0
+;; predicate. The merge cannot tell that 64*s1 is below the deep member,
+;; because s1 and (s1 + s2) are different keys to it. So High is a umax over
+;; those two members, and Low is %p. Still correct, just less precise.
+;; The inner sum (s1 + s2) must not share a stride with the outer sum:
+;; SCEV would combine them and nothing would be left at depth 3. The deep
+;; load comes first in the loop on purpose, so that %p is the base for the
+;; offsets. If the (%s1 + %s2) predicate disappears from the CHECK lines,
+;; the test no longer covers the cap.
+;; Four stores, so the cost model accepts the merge: 12 checks before, 9 after.
+define void @stencil_merge_depth_cap_sum(ptr %p, ptr %q, ptr %q2, ptr %q3, ptr %q4, i64 %s1, i64 %s2, i64 %s3, i64 %s4, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_depth_cap_sum'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 5:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; MERGE-NEXT: Check 6:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 7:
+; MERGE-NEXT: Comparing group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; MERGE-NEXT: Check 8:
+; MERGE-NEXT: Comparing group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 9:
+; MERGE-NEXT: Comparing group GRP3:
+; MERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; MERGE-NEXT: Member: {%q2,+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %q3 High: (1 + (8 * %n) + %q3))
+; MERGE-NEXT: Member: {%q3,+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %q4 High: (1 + (8 * %n) + %q4))
+; MERGE-NEXT: Member: {%q4,+,8}<%loop>
+; MERGE-NEXT: Group GRP4:
+; MERGE-NEXT: (Low: %p High: ((1 + (8 * %n) + (64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + (ptrtoaddr ptr %p to i64)) umax (1 + (8 * %n) + (64 * %s1) + (ptrtoaddr ptr %p to i64))))
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p),+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q3,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q4,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: (%s1 + %s2) sgt) 0
+; MERGE-NEXT: Compare predicate: %s3 sgt) 0
+; MERGE-NEXT: Compare predicate: %s4 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_depth_cap_sum'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Check 8:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 9:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 10:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 11:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Check 12:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 13:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 14:
+; NOMERGE-NEXT: Comparing group GRP2:
+; NOMERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 15:
+; NOMERGE-NEXT: Comparing group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 16:
+; NOMERGE-NEXT: Comparing group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 17:
+; NOMERGE-NEXT: Comparing group GRP3:
+; NOMERGE-NEXT: %aq4 = getelementptr i8, ptr %q4, i64 %idx
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; NOMERGE-NEXT: Member: {%q2,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %q3 High: (1 + (8 * %n) + %q3))
+; NOMERGE-NEXT: Member: {%q3,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %q4 High: (1 + (8 * %n) + %q4))
+; NOMERGE-NEXT: Member: {%q4,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: ((64 * %s1) + %p) High: (1 + (8 * %n) + (64 * %s1) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: ((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p) High: (1 + (8 * %n) + (64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p))
+; NOMERGE-NEXT: Member: {((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p),+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q3,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q4,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %pos64s1 = mul i64 %s1, 64
+ %sum12 = add i64 %s1, %s2
+ %m65 = mul i64 %sum12, 65
+ %sum34 = add i64 %s3, %s4
+ %sum = add i64 %sum34, %m65
+ %deep = mul i64 %sum, 64
+ %p1 = getelementptr i8, ptr %p, i64 %pos64s1
+ %p2 = getelementptr i8, ptr %p, i64 %deep
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ %v2 = load i8, ptr %a2
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ %v1 = load i8, ptr %a1
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ %v0 = load i8, ptr %a0
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 %s, ptr %aq
+ %aq2 = getelementptr i8, ptr %q2, i64 %idx
+ store i8 %s, ptr %aq2
+ %aq3 = getelementptr i8, ptr %q3, i64 %idx
+ store i8 %s, ptr %aq3
+ %aq4 = getelementptr i8, ptr %q4, i64 %idx
+ store i8 %s, ptr %aq4
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
>From 7020def0934ee451387e751cb9f37a141d32ee6d Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 1 Sep 2026 14:16:47 +0100
Subject: [PATCH 26/33] Address first wave of new comments
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 89 ++++++++-----------
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +
.../runtime-check-group-merging.ll | 8 +-
3 files changed, 46 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 77ece3a445c90..c57584800cc59 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -118,6 +118,9 @@ static cl::opt<StencilMergePolicy> StencilMerge(
"Always attempt stencil merge regardless of check "
"count")));
+// Keep the default in sync with -vectorize-memory-check-threshold in
+// LoopVectorize.cpp. Above that many checks the vectorizer gives up on the
+// loop, so that is where merging starts to matter.
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
@@ -857,7 +860,7 @@ struct StencilDecomposition {
};
/// Recursion cap for addScaledStencilTerm. Depth counts how deep a term
-/// sits inside the offset expression. Example, the offset
+/// sits inside the offset expression. For example, the offset
/// 8 + (64 * (s1 + s2 + (4 * s3)))
/// is visited like this:
/// depth 0: the whole add
@@ -961,7 +964,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return D;
}
-/// Return true if A's address is never higher than B's address.
+/// Return true if offset A is never higher than offset B.
/// A and B are these sums:
/// A = A.Constant + CoefA_1 * stride_1 + CoefA_2 * stride_2 + ...
/// B = B.Constant + CoefB_1 * stride_1 + CoefB_2 * stride_2 + ...
@@ -1011,6 +1014,8 @@ static bool isNeverAbove(const StencilDecomposition &A,
/// member: A beats B. The members nobody beats are the candidates.
/// The maximum side works the same way with the comparison flipped.
/// When two members have equal offsets, only the first one is kept.
+/// In other words: "beats" is a partial order on the offsets, and the
+/// candidates are its minimal elements.
/// Returns indices into \p Offsets.
/// TODO: Worst case compares every pair of members: O(N^2). Fine for real
/// stencils.
@@ -1028,6 +1033,8 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
for (unsigned K = 0; K < Offsets.size(); ++K) {
if (Beaten.test(K))
continue;
+ // Walk J = K + 1 .. N to avoid checking the same pair twice, as
+ // (K, J) and again as (J, K). The order in a pair does not matter.
for (unsigned J = K + 1; J < Offsets.size(); ++J) {
if (Beaten.test(J))
continue;
@@ -1067,9 +1074,9 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
/// the same external groups;
/// - one predicate per stride we must prove positive, unless an earlier
/// DepSet already paid for it (\p CommittedStridePredicates);
-/// - a umin over k members costs k-1 compare+selects, same for the umax
-/// (\p NumMinCandidates - 1 plus \p NumMaxCandidates - 1). A single
-/// candidate costs nothing: the bound is that member's own address.
+/// - a umin over k members costs k-1 compare+selects, same for the umax.
+/// \p NumBoundOperands is the sum of the two. A single candidate costs
+/// nothing: the bound is that member's own address.
///
/// Returns {ChecksBefore, ChecksAfter}.
static std::pair<unsigned, unsigned> computeStencilMergeCost(
@@ -1077,7 +1084,7 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
ArrayRef<const SCEV *> LocalStridesNeedingPreds,
const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates,
- unsigned NumMinCandidates, unsigned NumMaxCandidates) {
+ unsigned NumBoundOperands) {
ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
unsigned NumGroups = GroupIndices.size();
SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
@@ -1101,10 +1108,6 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
if (!CommittedStridePredicates.contains(Stride))
++NewPredicates;
- // Extra bound operands: one compare-and-select per operand past the first
- // in the merged umin, and the same for the umax.
- unsigned NumBoundOperands = (NumMinCandidates - 1) + (NumMaxCandidates - 1);
-
LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
<< ", NumExternalChecks=" << NumExternalChecks
<< ", predicates=" << NewPredicates
@@ -1133,8 +1136,9 @@ static RuntimeCheckingPtrGroup buildMergedStencilGroup(
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
append_range(CandidateGroup.Members, drop_begin(AllMembers));
- for (unsigned GI : GroupIndices)
- CandidateGroup.NeedsFreeze |= RtCheck.CheckingGroups[GI].NeedsFreeze;
+ CandidateGroup.NeedsFreeze = any_of(GroupIndices, [&](unsigned GI) {
+ return RtCheck.CheckingGroups[GI].NeedsFreeze;
+ });
// Register the positive-stride SCEV predicates with PSE, skipping any stride
// an earlier DepSet already added a predicate for.
@@ -1286,8 +1290,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// loop-invariant address is computed in the preheader, outside the loop.
if (any_of(AllMembers, [&](unsigned Idx) {
const PointerInfo &P = Pointers[Idx];
+ assert(!P.IsWritePtr && "only read members reach this point");
return any_of(
- DC.getInstructionsForAccess(P.PointerValue, P.IsWritePtr),
+ DC.getInstructionsForAccess(P.PointerValue, /*isWrite=*/false),
[&](Instruction *I) {
return LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
DC.getDT());
@@ -1336,7 +1341,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
"skipping DepSet\n");
continue;
}
- if (any_of(AllMembers, [&](unsigned Idx) {
+ if (any_of(drop_begin(AllMembers), [&](unsigned Idx) {
const SCEV *Range =
SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
if (isa<SCEVCouldNotCompute>(Range))
@@ -1359,7 +1364,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// a single iteration. The base member is picked arbitrarily, so together
// with the BaseStep check above this keeps the decision the same no matter
// which member comes first: we only merge recurrences with one common step.
- if (any_of(AllMembers, [&](unsigned Idx) {
+ if (any_of(drop_begin(AllMembers), [&](unsigned Idx) {
return GetStepForPointer(Idx) != BaseStep;
})) {
LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
@@ -1370,6 +1375,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// member's constant offset and its coefficient for each stride, all
// relative to BaseLow.
SmallVector<StencilDecomposition, 8> MemberOffsets;
+ MemberOffsets.reserve(AllMembers.size());
+ // The base member's offset from itself is zero: Constant 0, no strides.
+ MemberOffsets.emplace_back();
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
// Decompose one member's offset (relative to BaseLow) and append it to
@@ -1397,7 +1405,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return true;
};
- if (!all_of(AllMembers, CollectOffset))
+ if (!all_of(drop_begin(AllMembers), CollectOffset))
continue;
SmallVector<unsigned, 4> MinCandidates =
@@ -1411,63 +1419,44 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< ", max=" << MaxCandidates.size() << " of "
<< MemberOffsets.size() << "\n");
+ // Extra bound operands: one compare-and-select per operand past the first
+ // in the merged umin, and the same for the umax.
+ unsigned NumBoundOperands =
+ (MinCandidates.size() - 1) + (MaxCandidates.size() - 1);
+
// Local cost model: decide whether replacing this DepSet's groups with the
// single merged group actually reduces the number of runtime checks. Run
// it before building the merged bounds: a rejected DepSet then creates no
// umin/umax expressions that would only be thrown away.
- auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
- *this, GroupIndices, MergedGroupIndices,
- LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates,
- MinCandidates.size(), MaxCandidates.size());
+ auto [ChecksBefore, ChecksAfter] =
+ computeStencilMergeCost(*this, GroupIndices, MergedGroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(),
+ CommittedStridePredicates, NumBoundOperands);
if (ChecksAfter >= ChecksBefore) {
LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
}
- // OffsetTy is the integer type ptrtoaddr produces for the base pointer.
- // Every umin/umax operand below must have this one type. A member in an
- // address space with a different address width would produce a different
- // type; BuildBound returns null then, and the DepSet is skipped.
- Type *OffsetTy = SE->getEffectiveSCEVType(BaseLow->getType());
-
// Build one side of the merged bounds from its candidate members.
// With one candidate the bound is that member's own Start (or End): the
// exact value the member's own check used before the merge. With several
// candidates the bound is a umin (umax) over their Starts (Ends). Either
// way every value is a real member address, so the merge computes no new
// address and no new overflow is possible.
- // SCEV umin/umax needs integer operands, so convert each pointer with
- // ptrtoaddr. That fails for pointers whose address bits are not stable;
- // return null then, and the DepSet is skipped.
- const auto BuildBound = [&](ArrayRef<unsigned> Candidates,
- bool IsLow) -> const SCEV * {
- auto GetBound = [&](unsigned K) {
- const PointerInfo &P = Pointers[AllMembers[K]];
- return IsLow ? P.Start : P.End;
- };
- if (Candidates.size() == 1)
- return GetBound(Candidates.front());
+ // The umin/umax are on pointers. The expander turns them into the same
+ // icmp and select that a plain check uses, so no address conversion is
+ // needed.
+ const auto BuildBound = [&](ArrayRef<unsigned> Candidates, bool IsLow) {
SmallVector<SCEVUse, 4> Ops;
for (unsigned K : Candidates) {
- const SCEV *Bound = GetBound(K);
- if (!Bound->getType()->isPointerTy())
- return nullptr;
- const SCEV *IntBound = SE->getPtrToAddrExpr(Bound);
- if (isa<SCEVCouldNotCompute>(IntBound) ||
- IntBound->getType() != OffsetTy)
- return nullptr;
- Ops.push_back(IntBound);
+ const PointerInfo &P = Pointers[AllMembers[K]];
+ Ops.push_back(IsLow ? P.Start : P.End);
}
return IsLow ? SE->getUMinExpr(Ops) : SE->getUMaxExpr(Ops);
};
const SCEV *MergedLow = BuildBound(MinCandidates, /*IsLow=*/true);
const SCEV *MergedHigh = BuildBound(MaxCandidates, /*IsLow=*/false);
- if (!MergedLow || !MergedHigh) {
- LLVM_DEBUG(dbgs() << "LAA: Cannot build umin/umax bounds, "
- "skipping DepSet\n");
- continue;
- }
LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
<< ", High=" << *MergedHigh << "\n");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5c4e20ee66b65..e2d36e20b413c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,6 +198,9 @@ static cl::opt<unsigned> TinyTripCountVectorThreshold(
"value are vectorized only if no scalar iteration overheads "
"are incurred."));
+// Keep the default in sync with -stencil-merge-check-threshold in
+// LoopAccessAnalysis.cpp, which starts merging stencil runtime checks once
+// the count goes above this limit.
static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 36fa91678b5b8..ab4c83451b48f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1312,7 +1312,7 @@ define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %
; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + (ptrtoaddr ptr %a to i64)) umin (32 + (-5 * %s1) + (ptrtoaddr ptr %a to i64))) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + %a) umin (32 + (-5 * %s1) + %a)) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
; MERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
@@ -2021,7 +2021,7 @@ define void @stencil_merge_constant_and_stride_candidates(ptr %a, ptr %out, ptr
; MERGE-NEXT: (Low: (16 + %out2) High: (-16 + (8 * %n) + %out2))
; MERGE-NEXT: Member: {(16 + %out2),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((16 + (-1 * %cdj) + (ptrtoaddr ptr %a to i64)) umin (ptrtoaddr ptr %a to i64)) High: (((8 * %n) + (ptrtoaddr ptr %a to i64)) umax (-16 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %cdj)))
+; MERGE-NEXT: (Low: ((16 + (-1 * %cdj) + %a) umin %a) High: (((8 * %n) + %a) umax (-16 + (8 * %n) + %cdj + %a)))
; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
@@ -2191,7 +2191,7 @@ define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n
; MERGE-NEXT: (Low: (32 + %out2) High: (-32 + (8 * %n) + %out2))
; MERGE-NEXT: Member: {(32 + %out2),+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((32 + (-1 * %s3) + (ptrtoaddr ptr %a to i64)) umin (32 + (-1 * %s2) + (ptrtoaddr ptr %a to i64)) umin (32 + (-1 * %s1) + (ptrtoaddr ptr %a to i64))) High: ((-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s1) umax (-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s2) umax (-32 + (8 * %n) + (ptrtoaddr ptr %a to i64) + %s3)))
+; MERGE-NEXT: (Low: ((32 + (-1 * %s3) + %a) umin (32 + (-1 * %s2) + %a) umin (32 + (-1 * %s1) + %a)) High: ((-32 + (8 * %n) + %s1 + %a) umax (-32 + (8 * %n) + %s2 + %a) umax (-32 + (8 * %n) + %s3 + %a)))
; MERGE-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
@@ -2761,7 +2761,7 @@ define void @stencil_merge_depth_cap_sum(ptr %p, ptr %q, ptr %q2, ptr %q3, ptr %
; MERGE-NEXT: (Low: %q4 High: (1 + (8 * %n) + %q4))
; MERGE-NEXT: Member: {%q4,+,8}<%loop>
; MERGE-NEXT: Group GRP4:
-; MERGE-NEXT: (Low: %p High: ((1 + (8 * %n) + (64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + (ptrtoaddr ptr %p to i64)) umax (1 + (8 * %n) + (64 * %s1) + (ptrtoaddr ptr %p to i64))))
+; MERGE-NEXT: (Low: %p High: ((1 + (8 * %n) + (64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p) umax (1 + (8 * %n) + (64 * %s1) + %p)))
; MERGE-NEXT: Member: {%p,+,8}<%loop>
; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
; MERGE-NEXT: Member: {((64 * ((65 * (%s1 + %s2)) + %s3 + %s4)) + %p),+,8}<%loop>
>From f01997cee4f61fee97ea13ad4a5b65f22558cc36 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 2 Sep 2026 15:42:14 +0100
Subject: [PATCH 27/33] Address test-related comments
---
...-auto.ll => stencil-group-merging-auto.ll} | 4 +-
...-i128.ll => stencil-group-merging-i128.ll} | 2 +
...up-merging.ll => stencil-group-merging.ll} | 649 +++++++++++++++++-
3 files changed, 634 insertions(+), 21 deletions(-)
rename llvm/test/Analysis/LoopAccessAnalysis/{runtime-check-group-merging-auto.ll => stencil-group-merging-auto.ll} (97%)
rename llvm/test/Analysis/LoopAccessAnalysis/{runtime-check-group-merging-i128.ll => stencil-group-merging-i128.ll} (99%)
rename llvm/test/Analysis/LoopAccessAnalysis/{runtime-check-group-merging.ll => stencil-group-merging.ll} (81%)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
similarity index 97%
rename from llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
index 6855e80d4cffa..c9fd8b8966200 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1000 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=5 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=6 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; AUTOMERGE-LABEL: 'stencil_auto_threshold'
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
similarity index 99%
rename from llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
index 4a5906e5da552..8dae803c5b6ba 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
@@ -46,6 +46,7 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
;
entry:
br label %loop
+
loop:
%iv = phi i64 [0, %entry], [%iv.next, %loop]
%ivx = zext i64 %iv to i128
@@ -73,6 +74,7 @@ loop:
%iv.next = add i64 %iv, 1
%c = icmp slt i64 %iv.next, %n
br i1 %c, label %loop, label %exit
+
exit:
ret void
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
similarity index 81%
rename from llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index ab4c83451b48f..17b2326f200d7 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -229,7 +229,6 @@ exit:
;; Test 3: Cost model rejection — only 2 loads with runtime stride.
;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
-;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; CHECK-LABEL: 'cost_model_rejection'
; CHECK-NEXT: loop:
@@ -293,7 +292,6 @@ exit:
;; Test 4: Invariant + strided reads from same base -> different access ranges.
;; A strided read {%a,+,8} and an invariant read from %a have different
;; access ranges (8*n vs 8), so merging must NOT combine them.
-;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; CHECK-LABEL: 'different_steps_no_merge'
; CHECK-NEXT: loop:
@@ -326,9 +324,6 @@ define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
-; GRP0: store to %out (write, different DepSet):
-; GRP1: strided read from %a (step=8):
-; GRP2: invariant read from %a (step=0, different from GRP1):
entry:
%cmp = icmp sgt i64 %n, 2
br i1 %cmp, label %loop, label %exit
@@ -513,14 +508,12 @@ define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
-; 3 checks: each temp group separately vs %out (not merged due to predication):
-; 4 groups: 1 for %out, 3 separate temp groups:
entry:
%cmp = icmp sgt i64 %n, 0
br i1 %cmp, label %loop, label %exit
loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end2 ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
; Unconditional load at base + 0
@@ -553,9 +546,6 @@ if.end2:
%outp = getelementptr inbounds double, ptr %out, i64 %iv
store double %s, ptr %outp, align 8
- br label %latch
-
-latch:
%iv.next = add nuw nsw i64 %iv, 1
%cond = icmp slt i64 %iv.next, %n
br i1 %cond, label %loop, label %exit
@@ -568,7 +558,6 @@ exit:
;; Test 7: Write pointer in a group prevents stencil merging.
;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
;; All three share the same base %a (same DepSet). The write prevents merging.
-;; Both modes: groups stay separate.
define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; CHECK-LABEL: 'write_in_group_rejection'
; CHECK-NEXT: loop:
@@ -619,9 +608,6 @@ define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
-; 5 checks: write group vs all others, %out vs the two read groups:
-; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
-; Both modes produce identical output (write prevents merging).
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -959,6 +945,7 @@ define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
entry:
%inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%sp = getelementptr inbounds i8, ptr %a, i64 %iv
@@ -971,6 +958,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 8
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -1031,6 +1019,7 @@ define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cd
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
%p0 = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -1042,6 +1031,7 @@ loop:
%i3 = add i64 %i1, %cdj
%p3 = getelementptr inbounds i64, ptr %a, i64 %i3
br i1 %c, label %then, label %backedge
+
then:
%v1 = load i64, ptr %p1
%v2 = load i64, ptr %p2
@@ -1049,6 +1039,7 @@ then:
%t = add i64 %v1, %v2
%u = add i64 %t, %v3
br label %backedge
+
backedge:
%vp = phi i64 [ %u, %then ], [ 0, %loop ]
%sum = add i64 %v0, %vp
@@ -1057,6 +1048,7 @@ backedge:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -1098,6 +1090,7 @@ define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%s8 = mul i64 %iv, 8
@@ -1113,12 +1106,14 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-;; Test 13: %gep Start/End expressions are min/max expressions.
-;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
+;; Test 13: Start/End of %gep are umin/umax expressions, because the step
+;; (%m + 1) has an unknown sign. Their difference is SCEVCouldNotCompute, so
+;; the range check rejects the merge.
define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
; CHECK-LABEL: 'stencil_merge_range_could_not_compute'
; CHECK-NEXT: loop:
@@ -1156,15 +1151,17 @@ entry:
br label %loop
loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
- %i = phi i1 [ false, %entry ], [ true, %loop ]
%v0 = load i8, ptr %p
%gep = getelementptr i8, ptr %p, i64 %off
%v1 = load i8, ptr %gep
%s = add i8 %v0, %v1
store i8 %s, ptr %q
%off.next = add i64 %off, %step
- br i1 %i, label %done, label %loop
+ %iv.next = add i64 %iv, 1
+ %cmp = icmp eq i64 %iv.next, 2
+ br i1 %cmp, label %done, label %loop
done:
ret void
@@ -2949,3 +2946,617 @@ loop:
done:
ret void
}
+
+;; Test 25: A nested multiplier product that overflows int64.
+;; 6 loads from %a at offsets {0, +s1, -s1, +2*s1, -2*s1, C * (s2 + s4 + C*s3)}.
+;; The last offset stays factored, so decomposing it multiplies C * C.
+;; @nested_multiplier_ok: C = 1024. The product fits, the merge fires.
+;; Checks: 6 before the merge, 3 after.
+;; @nested_multiplier_overflow: C = 2^32. The product is 2^64, it does not
+;; fit int64_t, the offset is not decomposable and the DepSet is skipped.
+define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in, i64 %s4in) {
+; MERGE-LABEL: 'nested_multiplier_ok'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %off
+; MERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns1x2
+; MERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s1x2
+; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; MERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (((-2 * (1 smax %s1in)) + %a) umin %a) High: (((2 * (1 smax %s1in))<nuw> + (8 * (1 smax %n)) + %a) umax ((8 * (1 smax %n)) + (1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a)))
+; MERGE-NEXT: Member: {((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-2 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((2 * (1 smax %s1in))<nuw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: (1024 * (1 smax %s3in)) sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'nested_multiplier_ok'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %off
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns1x2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s1x2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a) High: ((8 * (1 smax %n)) + (1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a))
+; NOMERGE-NEXT: Member: {((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((-2 * (1 smax %s1in)) + %a) High: ((8 * (1 smax %n)) + (-2 * (1 smax %s1in)) + %a))
+; NOMERGE-NEXT: Member: {((-2 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((2 * (1 smax %s1in))<nuw> + %a) High: ((2 * (1 smax %s1in))<nuw> + (8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {((2 * (1 smax %s1in))<nuw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %s4 = call i64 @llvm.smax.i64(i64 %s4in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %s1x2 = shl i64 %s1, 1
+ %ns1x2 = sub i64 0, %s1x2
+ %m = mul i64 %s3, 1024
+ %sum24 = add i64 %s2, %s4
+ %sum234 = add i64 %sum24, %m
+ %off = mul i64 %sum234, 1024
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %v0 = load double, ptr %base, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s1x2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns1x2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %off
+ %v5 = load double, ptr %p5, align 8
+ %t1 = fadd double %v0, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t5, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Same loop with C = 2^32. The multiplier product overflows int64, no merge.
+define void @nested_multiplier_overflow(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in, i64 %s4in) {
+; CHECK-LABEL: 'nested_multiplier_overflow'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p5 = getelementptr i8, ptr %base, i64 %off
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns1x2
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s1x2
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP5:
+; CHECK-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; CHECK-NEXT: Check 5:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP6:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((4294967296 * ((4294967296 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a) High: ((8 * (1 smax %n)) + (4294967296 * ((4294967296 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a))
+; CHECK-NEXT: Member: {((4294967296 * ((4294967296 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((-2 * (1 smax %s1in)) + %a) High: ((8 * (1 smax %n)) + (-2 * (1 smax %s1in)) + %a))
+; CHECK-NEXT: Member: {((-2 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: ((2 * (1 smax %s1in))<nuw> + %a) High: ((2 * (1 smax %s1in))<nuw> + (8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {((2 * (1 smax %s1in))<nuw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP5:
+; CHECK-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP6:
+; CHECK-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %s4 = call i64 @llvm.smax.i64(i64 %s4in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %s1x2 = shl i64 %s1, 1
+ %ns1x2 = sub i64 0, %s1x2
+ %m = mul i64 %s3, 4294967296 ; 2^32
+ %sum24 = add i64 %s2, %s4
+ %sum234 = add i64 %sum24, %m
+ %off = mul i64 %sum234, 4294967296 ; 2^32
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %v0 = load double, ptr %base, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s1x2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns1x2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %off
+ %v5 = load double, ptr %p5, align 8
+ %t1 = fadd double %v0, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t5, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 26: A member whose corner sum overflows int64 in isNeverAbove.
+;; 7 loads from %a at offsets {-C*(s1+s2+s3), +-s1, +-s2, +-s3}.
+;; @corner_sum_overflow: C = 2^62. The far member's corner sum (its offset
+;; at s1 = s2 = s3 = 1) is -3 * 2^62, below int64 min, so isNeverAbove gives
+;; up and keeps the member as a candidate on both sides: 4 candidates for
+;; Low, 4 for High, 6 bound operands, and the cost model rejects the merge.
+;; @corner_sum_no_overflow: C = 2^60. The corner sums fit, the far member
+;; alone is Low and High is a 3-way umax. Checks: 7 before the merge, 3 after.
+define void @corner_sum_overflow(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
+; CHECK-LABEL: 'corner_sum_overflow'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP5:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; CHECK-NEXT: Check 5:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP6:
+; CHECK-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; CHECK-NEXT: Check 6:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP7:
+; CHECK-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s3in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s3in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((1 smax %s3in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s3in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s2in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: ((1 smax %s2in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s2in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP5:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP6:
+; CHECK-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP7:
+; CHECK-NEXT: (Low: ((-4611686018427387904 * (1 smax %s3in)) + (-4611686018427387904 * (1 smax %s2in)) + (-4611686018427387904 * (1 smax %s1in)) + %a) High: ((8 * (1 smax %n)) + (-4611686018427387904 * (1 smax %s3in)) + (-4611686018427387904 * (1 smax %s2in)) + (-4611686018427387904 * (1 smax %s1in)) + %a))
+; CHECK-NEXT: Member: {((-4611686018427387904 * (1 smax %s3in)) + (-4611686018427387904 * (1 smax %s2in)) + (-4611686018427387904 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %ns2 = sub i64 0, %s2
+ %ns3 = sub i64 0, %s3
+ %b1 = mul i64 %s1, -4611686018427387904 ; -2^62
+ %b2 = mul i64 %s2, -4611686018427387904 ; -2^62
+ %b3 = mul i64 %s3, -4611686018427387904 ; -2^62
+ %b12 = add i64 %b1, %b2
+ %boff = add i64 %b12, %b3
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %pb = getelementptr i8, ptr %base, i64 %boff
+ %vb = load double, ptr %pb, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+ %p6 = getelementptr i8, ptr %base, i64 %ns3
+ %v6 = load double, ptr %p6, align 8
+ %t1 = fadd double %vb, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %t6 = fadd double %t5, %v6
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Same loop with C = 2^60.
+define void @corner_sum_no_overflow(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
+; MERGE-LABEL: 'corner_sum_no_overflow'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; MERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; MERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; MERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; MERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; MERGE-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1152921504606846976 * (1 smax %s3in)) + (-1152921504606846976 * (1 smax %s2in)) + (-1152921504606846976 * (1 smax %s1in)) + %a) High: (((8 * (1 smax %n)) + (1 smax %s3in) + %a) umax ((8 * (1 smax %n)) + (1 smax %s2in) + %a) umax ((8 * (1 smax %n)) + (1 smax %s1in) + %a)))
+; MERGE-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1152921504606846976 * (1 smax %s3in)) + (-1152921504606846976 * (1 smax %s2in)) + (-1152921504606846976 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'corner_sum_no_overflow'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP7:
+; NOMERGE-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s3in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s3in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((1 smax %s3in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s3in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s2in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((1 smax %s2in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s2in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP7:
+; NOMERGE-NEXT: (Low: ((-1152921504606846976 * (1 smax %s3in)) + (-1152921504606846976 * (1 smax %s2in)) + (-1152921504606846976 * (1 smax %s1in)) + %a) High: ((8 * (1 smax %n)) + (-1152921504606846976 * (1 smax %s3in)) + (-1152921504606846976 * (1 smax %s2in)) + (-1152921504606846976 * (1 smax %s1in)) + %a))
+; NOMERGE-NEXT: Member: {((-1152921504606846976 * (1 smax %s3in)) + (-1152921504606846976 * (1 smax %s2in)) + (-1152921504606846976 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %ns2 = sub i64 0, %s2
+ %ns3 = sub i64 0, %s3
+ %b1 = mul i64 %s1, -1152921504606846976 ; -2^60
+ %b2 = mul i64 %s2, -1152921504606846976 ; -2^60
+ %b3 = mul i64 %s3, -1152921504606846976 ; -2^60
+ %b12 = add i64 %b1, %b2
+ %boff = add i64 %b12, %b3
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %pb = getelementptr i8, ptr %base, i64 %boff
+ %vb = load double, ptr %pb, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+ %p6 = getelementptr i8, ptr %base, i64 %ns3
+ %v6 = load double, ptr %p6, align 8
+ %t1 = fadd double %vb, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %t6 = fadd double %t5, %v6
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 27: Members whose pointer steps with the outer loop.
+;; The inner loop reads %a at {8*i, 8*i + cdj}, where %i is the outer
+;; induction variable. Both pointers are invariant in the inner loop and
+;; their SCEVs are addrecs of the outer loop, so there is no step in the
+;; analyzed loop and the merge skips the DepSet.
+define void @outer_loop_addrec_no_merge(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; CHECK-LABEL: 'outer_loop_addrec_no_merge'
+; CHECK-NEXT: inner:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds double, ptr %out, i64 %j
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %pc = getelementptr i8, ptr %po, i64 %cdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds double, ptr %out, i64 %j
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %po = getelementptr inbounds double, ptr %a, i64 %i
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%inner>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: {(%cdj + %a),+,8}<nw><%outer.hdr> High: {(8 + %cdj + %a),+,8}<nw><%outer.hdr>)
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%outer.hdr>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: {%a,+,8}<nuw><%outer.hdr> High: {(8 + %a),+,8}<nw><%outer.hdr>)
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%outer.hdr>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+; CHECK-NEXT: outer.hdr:
+; CHECK-NEXT: Report: loop is not the innermost loop
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %outer.hdr
+
+outer.hdr:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+ %po = getelementptr inbounds double, ptr %a, i64 %i
+ %pc = getelementptr i8, ptr %po, i64 %cdj
+ br label %inner
+
+inner:
+ %j = phi i64 [ 0, %outer.hdr ], [ %j.next, %inner ]
+ %v0 = load double, ptr %po, align 8
+ %v1 = load double, ptr %pc, align 8
+ %sum = fadd double %v0, %v1
+ %op = getelementptr inbounds double, ptr %out, i64 %j
+ store double %sum, ptr %op, align 8
+ %j.next = add nuw nsw i64 %j, 1
+ %ic = icmp slt i64 %j.next, %n
+ br i1 %ic, label %inner, label %outer.latch
+
+outer.latch:
+ %i.next = add nuw nsw i64 %i, 1
+ %oc = icmp slt i64 %i.next, %n
+ br i1 %oc, label %outer.hdr, label %exit
+
+exit:
+ ret void
+}
>From e817e3061d1134b9a0baefc9d95942dabef0b824 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 4 Sep 2026 12:27:09 +0100
Subject: [PATCH 28/33] Move VectorizeMemoryCheckThreshold into struct
VectorizerParams
---
.../llvm/Analysis/LoopAccessAnalysis.h | 4 ++++
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 23 +++++++++----------
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +++--------
.../stencil-group-merging-auto.ll | 4 ++--
4 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 4c76fa2e98ebe..8cd98c03675b4 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -47,6 +47,10 @@ struct VectorizerParams {
/// make more than this number of comparisons.
LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;
+ /// The maximum allowed number of runtime memory checks. Above this many
+ /// checks the vectorizer gives up on the loop.
+ LLVM_ABI static unsigned VectorizeMemoryCheckThreshold;
+
// When creating runtime checks for nested loops, where possible try to
// write the checks in a form that allows them to be easily hoisted out of
// the outermost loop. For example, we can do this by expanding the range of
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index c57584800cc59..145ea64843d4d 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -95,6 +95,13 @@ static cl::opt<unsigned, true> RuntimeMemoryCheckThreshold(
cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8));
unsigned VectorizerParams::RuntimeMemoryCheckThreshold;
+static cl::opt<unsigned, true> VectorizeMemoryCheckThreshold(
+ "vectorize-memory-check-threshold", cl::Hidden,
+ cl::desc("The maximum allowed number of runtime memory checks"),
+ cl::location(VectorizerParams::VectorizeMemoryCheckThreshold),
+ cl::init(128));
+unsigned VectorizerParams::VectorizeMemoryCheckThreshold;
+
/// The maximum iterations used to merge memory checks
static cl::opt<unsigned> MemoryCheckMergeThreshold(
"memory-check-merge-threshold", cl::Hidden,
@@ -113,21 +120,11 @@ static cl::opt<StencilMergePolicy> StencilMerge(
"Disable stencil merge (default)"),
clEnumValN(StencilMergePolicy::Auto, "auto",
"Enable stencil merge when runtime check count exceeds "
- "the threshold"),
+ "-vectorize-memory-check-threshold"),
clEnumValN(StencilMergePolicy::Force, "force",
"Always attempt stencil merge regardless of check "
"count")));
-// Keep the default in sync with -vectorize-memory-check-threshold in
-// LoopVectorize.cpp. Above that many checks the vectorizer gives up on the
-// loop, so that is where merging starts to matter.
-static cl::opt<unsigned> StencilMergeCheckThreshold(
- "stencil-merge-check-threshold", cl::Hidden,
- cl::desc("Auto-trigger stencil group merging when runtime check count "
- "exceeds this threshold (default = 128). Only used when "
- "-stencil-runtime-check-merge is auto."),
- cl::init(128));
-
static cl::opt<unsigned> StencilMergeMaxGroups(
"stencil-merge-max-groups", cl::Hidden,
cl::desc(
@@ -1221,7 +1218,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
++TotalChecks;
- if (TotalChecks <= StencilMergeCheckThreshold) {
+ // Above this many checks the vectorizer gives up on the loop, so that is
+ // where merging starts to matter.
+ if (TotalChecks <= VectorizerParams::VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
<< " checks <= threshold, skipping stencil merge\n");
return;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e2d36e20b413c..c44ccc718589d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,13 +198,6 @@ static cl::opt<unsigned> TinyTripCountVectorThreshold(
"value are vectorized only if no scalar iteration overheads "
"are incurred."));
-// Keep the default in sync with -stencil-merge-check-threshold in
-// LoopAccessAnalysis.cpp, which starts merging stencil runtime checks once
-// the count goes above this limit.
-static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
- "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
- cl::desc("The maximum allowed number of runtime memory checks"));
-
static cl::opt<bool> ForcePartialAliasingVectorization(
"force-partial-aliasing-vectorization", cl::init(false), cl::Hidden,
cl::desc("Replace pointer diff checks with alias masks."));
@@ -1587,8 +1580,8 @@ class GeneratedRTChecks {
// runtime checks needs to be generated.
// TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
// profile info.
- CostTooHigh =
- LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold;
+ CostTooHigh = LAI.getNumRuntimePointerChecks() >
+ VectorizerParams::VectorizeMemoryCheckThreshold;
if (CostTooHigh) {
// Mark runtime checks as never succeeding when they exceed the threshold.
MemRuntimeCheckCond = ConstantInt::getTrue(L->getHeader()->getContext());
@@ -7138,7 +7131,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// would lead to a divide by 0. Fall back to hard threshold.
if (VF.Width.isScalar()) {
// TODO: Should we rename VectorizeMemoryCheckThreshold?
- if (RtC > VectorizeMemoryCheckThreshold) {
+ if (RtC > VectorizerParams::VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(
dbgs()
<< "LV: Interleaving only is not profitable due to runtime checks\n");
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
index c9fd8b8966200..bc0b7d2023210 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=5 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
-; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=6 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -vectorize-memory-check-threshold=5 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -vectorize-memory-check-threshold=6 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; AUTOMERGE-LABEL: 'stencil_auto_threshold'
>From 5f54ee131713af7748a7f51108c3dbfe37a76c7c Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 10 Sep 2026 10:01:04 +0000
Subject: [PATCH 29/33] Add upper limits for strides
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 142 +++++++++++++-----
.../stencil-group-merging-auto.ll | 1 +
.../stencil-group-merging.ll | 108 +++++++------
3 files changed, 173 insertions(+), 78 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 145ea64843d4d..b8dec07e29a14 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -945,9 +945,8 @@ static bool addScaledStencilTerm(const SCEV *Term, int64_t Mult, unsigned Depth,
/// on the same base strides.
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
-/// Returns std::nullopt if a constant does not fit in int64_t or a multiplier
-/// or coefficient update overflows (we commit to the signed interpretation;
-/// values that need more than 64 significant bits are out of scope).
+/// Returns std::nullopt if a constant, multiplier, or coefficient update does
+/// not fit in int64_t.
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// A "Start" is always loop-invariant (getStartAndEndForAccess asserts it), so
@@ -961,13 +960,42 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return D;
}
+/// Find a common upper limit M for the positive strides in D. If every stride
+/// is between 1 and M, the decomposed offset fits in the signed index type.
+/// This lets isNeverAbove compare offsets as ordinary signed integers.
+///
+/// Subtract abs(Constant) from SignedMax, then divide the remaining budget by
+/// the sum of absolute coefficients:
+/// M = (SignedMax - abs(Constant)) / sum(abs(Coefficient)).
+/// For example, both 8 + 4*s and 8 - 4*s get M = (SignedMax - 8) / 4.
+///
+/// Return nullopt if abs(Constant) exceeds SignedMax or no positive stride
+/// fits. Otherwise, if all coefficients are zero, no stride limit is needed;
+/// return SignedMax.
+static std::optional<APInt>
+getStencilStrideUpperLimit(const StencilDecomposition &D, unsigned BitWidth) {
+ uint64_t SignedMax = maxIntN(BitWidth);
+ uint64_t AbsConstant = AbsoluteValue(D.Constant);
+ if (AbsConstant > SignedMax)
+ return std::nullopt;
+ uint64_t Budget = SignedMax - AbsConstant;
+ uint64_t CoeffSum = 0;
+ for (const auto &[Stride, Coeff] : D.Coefficients) {
+ uint64_t AbsCoeff = AbsoluteValue(Coeff);
+ if (AbsCoeff > Budget - CoeffSum)
+ return std::nullopt;
+ CoeffSum += AbsCoeff;
+ }
+ return APInt(BitWidth, CoeffSum ? Budget / CoeffSum : SignedMax);
+}
+
/// Return true if offset A is never higher than offset B.
/// A and B are these sums:
/// A = A.Constant + CoefA_1 * stride_1 + CoefA_2 * stride_2 + ...
/// B = B.Constant + CoefB_1 * stride_1 + CoefB_2 * stride_2 + ...
/// A stride missing from a member's map has coefficient 0. Every stride
-/// is 1 or more: strides are integers, and the merge adds an "s > 0"
-/// predicate for each stride that is not already known positive.
+/// is 1 or more: the caller proves or predicates each stride to be positive
+/// and that the whole expression does not overflow.
/// Example:
/// A: 0 - 80*s1
/// B: -40 - 40*s1
@@ -1069,8 +1097,8 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
/// After = NumExternalChecks + NewPredicates + NumBoundOperands:
/// - the merged group keeps the same IDs, so it is checked against exactly
/// the same external groups;
-/// - one predicate per stride we must prove positive, unless an earlier
-/// DepSet already paid for it (\p CommittedStridePredicates);
+/// - one check per stride needing a lower or upper limit, unless an earlier
+/// DepSet already paid for either limit;
/// - a umin over k members costs k-1 compare+selects, same for the umax.
/// \p NumBoundOperands is the sum of the two. A single candidate costs
/// nothing: the bound is that member's own address.
@@ -1079,8 +1107,10 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
static std::pair<unsigned, unsigned> computeStencilMergeCost(
const RuntimePointerChecking &RtCheck, ArrayRef<unsigned> GroupIndices,
const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
- ArrayRef<const SCEV *> LocalStridesNeedingPreds,
- const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates,
+ ArrayRef<const SCEV *> LocalStrideLowerLimits,
+ const SmallMapVector<const SCEV *, APInt, 4> &LocalStrideUpperLimits,
+ const SmallDenseSet<const SCEV *, 4> &StrideLowerLimits,
+ const SmallMapVector<const SCEV *, APInt, 4> &StrideUpperLimits,
unsigned NumBoundOperands) {
ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
unsigned NumGroups = GroupIndices.size();
@@ -1098,11 +1128,18 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
}
}
- // Each not-yet-committed positive-stride predicate becomes one extra runtime
- // check, so it counts against the saving.
+ // Charge once per stride, even when both limits need runtime predicates.
+ // Later DepSets can narrow an upper limit without adding another check.
+ const auto NeedsNewCheck = [&](const SCEV *Stride) {
+ return !StrideLowerLimits.contains(Stride) &&
+ !StrideUpperLimits.contains(Stride);
+ };
unsigned NewPredicates = 0;
- for (const SCEV *Stride : LocalStridesNeedingPreds)
- if (!CommittedStridePredicates.contains(Stride))
+ for (const SCEV *Stride : LocalStrideLowerLimits)
+ if (NeedsNewCheck(Stride))
+ ++NewPredicates;
+ for (const auto &[Stride, UpperLimit] : LocalStrideUpperLimits)
+ if (!is_contained(LocalStrideLowerLimits, Stride) && NeedsNewCheck(Stride))
++NewPredicates;
LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
@@ -1121,14 +1158,15 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
/// decided the merge is profitable. Constructs the bounding group over
/// \p AllMembers with bounds [\p MergedLow, \p MergedHigh], and registers with
/// \p PSE a positive-stride SCEV predicate for each stride in
-/// \p LocalStridesNeedingPreds that has not already been committed (tracked in
-/// \p CommittedStridePredicates across DepSets). Returns the new group.
-static RuntimeCheckingPtrGroup buildMergedStencilGroup(
- const RuntimePointerChecking &RtCheck, PredicatedScalarEvolution &PSE,
- ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
- const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
- ArrayRef<const SCEV *> LocalStridesNeedingPreds,
- SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
+/// \p LocalStrideLowerLimits that has not already been committed (tracked in
+/// \p StrideLowerLimits across DepSets). Returns the new group.
+static RuntimeCheckingPtrGroup
+buildMergedStencilGroup(const RuntimePointerChecking &RtCheck,
+ PredicatedScalarEvolution &PSE,
+ ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
+ const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
+ ArrayRef<const SCEV *> LocalStrideLowerLimits,
+ SmallDenseSet<const SCEV *, 4> &StrideLowerLimits) {
RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], RtCheck);
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
@@ -1140,8 +1178,8 @@ static RuntimeCheckingPtrGroup buildMergedStencilGroup(
// Register the positive-stride SCEV predicates with PSE, skipping any stride
// an earlier DepSet already added a predicate for.
ScalarEvolution &SE = *PSE.getSE();
- for (const SCEV *Stride : LocalStridesNeedingPreds) {
- if (!CommittedStridePredicates.insert(Stride).second)
+ for (const SCEV *Stride : LocalStrideLowerLimits) {
+ if (!StrideLowerLimits.insert(Stride).second)
continue;
const SCEV *Zero = SE.getZero(Stride->getType());
PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
@@ -1180,7 +1218,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// - keep the members that can hold the lowest or the highest address at
// runtime (the candidate members), and build the merged bounds as a
// umin over their Start values and a umax over their End values,
- // adding predicates for strides that are not already known positive;
+ // adding predicates for strides not already known positive and within
+ // their limits;
// - commit the merge only if the local cost model reduces the number of
// checks after accounting for any new predicates.
@@ -1245,8 +1284,11 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
SmallDenseSet<unsigned, 4> MergedGroupIndices;
SmallVector<RuntimeCheckingPtrGroup, 2> NewMergedGroups;
- // Track strides that already have committed predicates (across all DepSets).
- SmallDenseSet<const SCEV *, 4> CommittedStridePredicates;
+ // Track lower limits from accepted DepSets. Each is 1, so store only strides.
+ SmallDenseSet<const SCEV *, 4> StrideLowerLimits;
+ // Accepted DepSets may narrow these limits. Emit their predicates only
+ // after all DepSets have been analyzed.
+ SmallMapVector<const SCEV *, APInt, 4> StrideUpperLimits;
for (auto &[DepAliasKey, GroupIndices] : DepSetToGroups) {
[[maybe_unused]] auto [DepId, ASId] = DepAliasKey;
@@ -1310,6 +1352,11 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
const SCEV *BaseLow = Pointers[Member0].Start;
const SCEV *BaseHigh = Pointers[Member0].End;
+ // Keep stencil decomposition and stride-limit arithmetic within 64 bits.
+ // All offsets relative to BaseLow have the same index width.
+ if (SE->getTypeSizeInBits(BaseLow->getType()) > 64)
+ continue;
+
LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
<< ") with " << AllMembers.size()
<< " members, base: " << *BaseLow << "\n");
@@ -1377,7 +1424,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
MemberOffsets.reserve(AllMembers.size());
// The base member's offset from itself is zero: Constant 0, no strides.
MemberOffsets.emplace_back();
- SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
+ SmallSetVector<const SCEV *, 4> LocalStrideLowerLimits;
+ SmallMapVector<const SCEV *, APInt, 4> LocalStrideUpperLimits;
// Decompose one member's offset (relative to BaseLow) and append it to
// MemberOffsets. Returns false if the offset is not in stencil form (so
@@ -1393,9 +1441,26 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return false;
}
- for (const auto &[Stride, Coeff] : DLow->Coefficients)
+ auto UpperLimit = getStencilStrideUpperLimit(
+ *DLow, SE->getTypeSizeInBits(LowOffset->getType()));
+ if (!UpperLimit)
+ return false;
+
+ for (const auto &[Stride, Coeff] : DLow->Coefficients) {
+ // Require a positive stride if needed. The lower limit is always 1.
if (!SE->isKnownPositive(Stride))
- LocalStridesNeedingPreds.insert(Stride);
+ LocalStrideLowerLimits.insert(Stride);
+
+ // Check whether an upper-limit predicate is needed.
+ if (!Coeff || SE->isKnownPredicate(ICmpInst::ICMP_SLE, Stride,
+ SE->getConstant(*UpperLimit)))
+ continue;
+ auto [It, Inserted] =
+ LocalStrideUpperLimits.insert({Stride, *UpperLimit});
+ // Keep the smallest limit seen so far.
+ if (!Inserted && UpperLimit->ult(It->second))
+ It->second = *UpperLimit;
+ }
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< ": Const=" << DLow->Constant
@@ -1427,10 +1492,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// single merged group actually reduces the number of runtime checks. Run
// it before building the merged bounds: a rejected DepSet then creates no
// umin/umax expressions that would only be thrown away.
- auto [ChecksBefore, ChecksAfter] =
- computeStencilMergeCost(*this, GroupIndices, MergedGroupIndices,
- LocalStridesNeedingPreds.getArrayRef(),
- CommittedStridePredicates, NumBoundOperands);
+ auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
+ *this, GroupIndices, MergedGroupIndices,
+ LocalStrideLowerLimits.getArrayRef(), LocalStrideUpperLimits,
+ StrideLowerLimits, StrideUpperLimits, NumBoundOperands);
if (ChecksAfter >= ChecksBefore) {
LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
@@ -1465,10 +1530,19 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Build the merged group and register its positive-stride predicates.
NewMergedGroups.push_back(buildMergedStencilGroup(
*this, PSE, AllMembers, MergedLow, MergedHigh, GroupIndices,
- LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates));
+ LocalStrideLowerLimits.getArrayRef(), StrideLowerLimits));
+ for (const auto &[Stride, UpperLimit] : LocalStrideUpperLimits) {
+ auto [It, Inserted] = StrideUpperLimits.insert({Stride, UpperLimit});
+ if (!Inserted && UpperLimit.ult(It->second))
+ It->second = UpperLimit;
+ }
MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
}
+ for (const auto &[Stride, UpperLimit] : StrideUpperLimits)
+ PSE.addPredicate(*SE->getComparePredicate(ICmpInst::ICMP_SLE, Stride,
+ SE->getConstant(UpperLimit)));
+
// Rebuild CheckingGroups if we merged anything.
if (!NewMergedGroups.empty()) {
SmallVector<RuntimeCheckingPtrGroup, 2> FinalGroups;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
index bc0b7d2023210..ae5e88d4dcb5e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-auto.ll
@@ -34,6 +34,7 @@ define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; AUTOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; AUTOMERGE-NEXT: SCEV assumptions:
; AUTOMERGE-NEXT: Compare predicate: %cdj sgt) 0
+; AUTOMERGE-NEXT: Compare predicate: %cdj sle) 1537228672809129301
; AUTOMERGE-EMPTY:
; AUTOMERGE-NEXT: Expressions re-written:
;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index 17b2326f200d7..45da8011b3abf 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --function nested_multiplier_ok --version 6
; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,MERGE %s
; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,NOMERGE %s
@@ -6,7 +6,7 @@
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
;; relative to base = %a + 8*iv. Store to %out.
;; With merge: all 6 loads form 1 merged group with bounds spanning [-3*cdj, +3*cdj]
-;; relative to base, producing 1 runtime check and 1 stride predicate (cdj > 0).
+;; relative to base, producing 1 runtime check and lower/upper predicates (0 < cdj <= INT64_MAX / 6).
;; Without merge: 6 separate groups (each load alone), 6 checks, no predicates.
define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-LABEL: 'stencil_merge_single_stride'
@@ -40,6 +40,7 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 1537228672809129301
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -383,6 +384,7 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -645,9 +647,9 @@ exit:
;; Test 8: Shared stride predicate deduplication across two DepSets.
;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
-;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
-;; should be emitted, not 2.
-;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 1 predicate.
+;; Both DepSets share the same stride limits, so each stride predicate
+;; should be emitted once.
+;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 2 predicates.
;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
; MERGE-LABEL: 'shared_stride_predicate_dedup'
@@ -687,6 +689,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -796,12 +799,13 @@ exit:
}
-;; Test 9: Known-positive stride skips predicate emission.
-;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
+;; Test 9: A known-positive stride with a known range needs no predicates.
+;; smax(%cdj_in, 1) proves positivity; the parameter range leaves enough
+;; headroom for the offsets.
;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
-;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
+;; With merge: 1 merged group, 1 check, no stride predicates.
;; Without merge: 3 separate groups + 1 %out, 3 checks, no predicates.
-define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %cdj_in) {
+define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 range(i64 -2147483648, 2147483648) %cdj_in) {
; MERGE-LABEL: 'known_positive_stride_no_predicate'
; MERGE-NEXT: loop:
; MERGE-NEXT: Memory dependences are safe with run-time checks
@@ -1175,9 +1179,9 @@ declare i64 @llvm.smax.i64(i64, i64)
;; One member offset is (-1 * (s1 + s2)). decomposeStencilOffset distributes
;; the -1, so that member is keyed on s1 and s2 like the others. It then
;; defines the low bound alone: Low is plain %p with no umin. Both strides
-;; are known positive, so no predicate is emitted.
-;; Checks: 3 before the merge, 1 after.
-define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
+;; are known positive and have signed-i32 parameter ranges, so no stride
+;; predicates are needed. Checks: 3 before the merge, 1 after.
+define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 range(i64 -2147483648, 2147483648) %s1in, i64 range(i64 -2147483648, 2147483648) %s2in, i64 %n) {
; MERGE-LABEL: 'stencil_merge_summed_stride'
; MERGE-NEXT: loop:
; MERGE-NEXT: Memory dependences are safe with run-time checks
@@ -1321,6 +1325,8 @@ define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-NEXT: Compare predicate: %s1 sle) 614891469123651720
+; MERGE-NEXT: Compare predicate: %s2 sle) 614891469123651720
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -1446,7 +1452,7 @@ exit:
;; The strides are unrelated, so all four -s_i members can be the lowest
;; address and all four +s_i members can be the highest. The merged bounds
;; would need a 4-way umin and a 4-way umax.
-;; Cost: 8 checks before. After: 1 check + 4 stride predicates + 3 umin
+;; Cost: 8 checks before. After: 1 check + 4 strides needing predicates + 3 umin
;; operands + 3 umax operands = 11. 11 >= 8, so no merge, even with
;; -stencil-runtime-check-merge=force.
define void @too_many_candidates_no_merge(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2, i64 %s3, i64 %s4) {
@@ -1589,7 +1595,7 @@ exit:
;; and the member is keyed on s1 and s2 like the others. That is enough to
;; prove it is the highest member and %p is the lowest. So the merged bounds
;; are simple: Low is %p, High is the End of the 64*(s1+s2) member, no umin
-;; or umax, and only two predicates, s1 > 0 and s2 > 0.
+;; or umax, and lower/upper predicates for each of the two strides.
;; Two stores, so the cost model accepts the merge: 8 checks before, 4 after.
define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i64 %s2, i64 %n) {
; MERGE-LABEL: 'stencil_merge_factored_diagonal'
@@ -1642,6 +1648,8 @@ define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i
; MERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-NEXT: Compare predicate: %s1 sle) 72057594037927935
+; MERGE-NEXT: Compare predicate: %s2 sle) 72057594037927935
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -1766,7 +1774,7 @@ done:
;; 3 loads from %p at byte offsets {0, 64*s1, 8 + 64*(s1 + s2 + 4*s3)}.
;; Stores to %q, %q2 and %q3.
;; In the last offset, 4*s3 sits at depth 3, the cap. It is not split into
-;; 4 times s3. The keys are s1, s2 and (4 * s3), each with a > 0 predicate,
+;; 4 times s3. The keys are s1, s2 and (4 * s3), each with lower/upper predicates,
;; and the group still merges.
;; Three stores, so the cost model accepts the merge: 9 checks before, 6 after.
define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1, i64 %s2, i64 %s3, i64 %n) {
@@ -1838,6 +1846,9 @@ define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1,
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
; MERGE-NEXT: Compare predicate: (4 * %s3) sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-NEXT: Compare predicate: %s1 sle) 48038396025285290
+; MERGE-NEXT: Compare predicate: (4 * %s3) sle) 48038396025285290
+; MERGE-NEXT: Compare predicate: %s2 sle) 48038396025285290
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2027,6 +2038,7 @@ define void @stencil_merge_constant_and_stride_candidates(ptr %a, ptr %out, ptr
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2202,6 +2214,9 @@ define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n
; MERGE-NEXT: Compare predicate: %s3 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: %s3 sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: %s2 sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: %s1 sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2374,7 +2389,7 @@ exit:
;; no inbounds. The merge does not need inbounds facts of its own: the merged
;; bounds reuse the members' Start/End values, the same values the unmerged
;; checks would compare, so no new address is computed. The group merges as
-;; usual with the cdj > 0 predicate.
+;; usual with predicates limiting the positive stride.
define void @stencil_merge_no_inbounds_member(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-LABEL: 'stencil_merge_no_inbounds_member'
; MERGE-NEXT: loop:
@@ -2401,6 +2416,7 @@ define void @stencil_merge_no_inbounds_member(ptr %a, ptr %out, i64 %n, i64 %cdj
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2553,7 +2569,7 @@ exit:
;; LAA turns the select load into two members, one per arm, each with its
;; own Start and End. The merge treats them like any other member. High is
;; the End of the 3*cdj arm, so both arms are inside the merged bounds.
-;; With merge: 1 group, 1 check, 1 predicate (cdj > 0).
+;; With merge: 1 group, 1 check, 2 predicates (0 < cdj <= INT64_MAX / 2).
;; Without merge: 4 groups, 4 checks, no predicates.
define void @forked_pointer_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
; MERGE-LABEL: 'forked_pointer_merge'
@@ -2583,6 +2599,7 @@ define void @forked_pointer_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2670,10 +2687,10 @@ exit:
;; Stores to %q, %q2, %q3 and %q4.
;; In the last offset, (s1 + s2) sits at depth 3, the cap. It is not split
;; into s1 and s2: it becomes one stride key, like a new stride. So the keys
-;; are s1 (from the second load), s3, s4 and (s1 + s2), each with a > 0
-;; predicate. The merge cannot tell that 64*s1 is below the deep member,
-;; because s1 and (s1 + s2) are different keys to it. So High is a umax over
-;; those two members, and Low is %p. Still correct, just less precise.
+;; are s1 (from the second load), s3, s4 and (s1 + s2), each with lower
+;; and upper limit predicates. The merge cannot tell that 64*s1 is below the
+;; deep member, because s1 and (s1 + s2) are different keys to it. So High is
+;; a umax over those two members, and Low is %p. Still correct, just less precise.
;; The inner sum (s1 + s2) must not share a stride with the outer sum:
;; SCEV would combine them and nothing would be left at depth 3. The deep
;; load comes first in the loop on purpose, so that %p is the base for the
@@ -2776,6 +2793,10 @@ define void @stencil_merge_depth_cap_sum(ptr %p, ptr %q, ptr %q2, ptr %q3, ptr %
; MERGE-NEXT: Compare predicate: (%s1 + %s2) sgt) 0
; MERGE-NEXT: Compare predicate: %s3 sgt) 0
; MERGE-NEXT: Compare predicate: %s4 sgt) 0
+; MERGE-NEXT: Compare predicate: %s1 sle) 144115188075855871
+; MERGE-NEXT: Compare predicate: (%s1 + %s2) sle) 2150972956356057
+; MERGE-NEXT: Compare predicate: %s3 sle) 2150972956356057
+; MERGE-NEXT: Compare predicate: %s4 sle) 2150972956356057
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -2950,11 +2971,12 @@ done:
;; Test 25: A nested multiplier product that overflows int64.
;; 6 loads from %a at offsets {0, +s1, -s1, +2*s1, -2*s1, C * (s2 + s4 + C*s3)}.
;; The last offset stays factored, so decomposing it multiplies C * C.
-;; @nested_multiplier_ok: C = 1024. The product fits, the merge fires.
+;; @nested_multiplier_ok: C = 1024. Positive strides up to INT32_MAX leave
+;; enough headroom for the offsets. The product fits, and the merge fires.
;; Checks: 6 before the merge, 3 after.
;; @nested_multiplier_overflow: C = 2^32. The product is 2^64, it does not
;; fit int64_t, the offset is not decomposable and the DepSet is skipped.
-define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in, i64 %s4in) {
+define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 range(i64 1, 2147483648) %s1, i64 range(i64 1, 2147483648) %s2, i64 range(i64 1, 2147483648) %s3, i64 range(i64 1, 2147483648) %s4) {
; MERGE-LABEL: 'nested_multiplier_ok'
; MERGE-NEXT: loop:
; MERGE-NEXT: Memory dependences are safe with run-time checks
@@ -2975,17 +2997,16 @@ define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in
; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: (((-2 * (1 smax %s1in)) + %a) umin %a) High: (((2 * (1 smax %s1in))<nuw> + (8 * (1 smax %n)) + %a) umax ((8 * (1 smax %n)) + (1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a)))
-; MERGE-NEXT: Member: {((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {((-2 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {((2 * (1 smax %s1in))<nuw> + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: (Low: (((-2 * %s1)<nsw> + %a) umin %a) High: (((2 * %s1)<nuw><nsw> + (8 * (1 smax %n)) + %a) umax ((8 * (1 smax %n)) + (1024 * ((1024 * %s3)<nuw><nsw> + %s2 + %s4))<nuw><nsw> + %a)))
+; MERGE-NEXT: Member: {((1024 * ((1024 * %s3)<nuw><nsw> + %s2 + %s4))<nuw><nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-2 * %s1)<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((2 * %s1)<nuw><nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * %s1)<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(%s1 + %a),+,8}<nw><%loop>
; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
-; MERGE-NEXT: Compare predicate: (1024 * (1 smax %s3in)) sgt) 0
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -3029,20 +3050,20 @@ define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in
; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a) High: ((8 * (1 smax %n)) + (1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a))
-; NOMERGE-NEXT: Member: {((1024 * ((1024 * (1 smax %s3in)) + (1 smax %s2in) + (1 smax %s4in))) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: ((1024 * ((1024 * %s3)<nuw><nsw> + %s2 + %s4))<nuw><nsw> + %a) High: ((8 * (1 smax %n)) + (1024 * ((1024 * %s3)<nuw><nsw> + %s2 + %s4))<nuw><nsw> + %a))
+; NOMERGE-NEXT: Member: {((1024 * ((1024 * %s3)<nuw><nsw> + %s2 + %s4))<nuw><nsw> + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((-2 * (1 smax %s1in)) + %a) High: ((8 * (1 smax %n)) + (-2 * (1 smax %s1in)) + %a))
-; NOMERGE-NEXT: Member: {((-2 * (1 smax %s1in)) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: ((-2 * %s1)<nsw> + %a) High: ((8 * (1 smax %n)) + (-2 * %s1)<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-2 * %s1)<nsw> + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: ((2 * (1 smax %s1in))<nuw> + %a) High: ((2 * (1 smax %s1in))<nuw> + (8 * (1 smax %n)) + %a))
-; NOMERGE-NEXT: Member: {((2 * (1 smax %s1in))<nuw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: ((2 * %s1)<nuw><nsw> + %a) High: ((2 * %s1)<nuw><nsw> + (8 * (1 smax %n)) + %a))
+; NOMERGE-NEXT: Member: {((2 * %s1)<nuw><nsw> + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
-; NOMERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: ((-1 * %s1)<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * %s1)<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * %s1)<nsw> + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP5:
-; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
-; NOMERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: (%s1 + %a) High: ((8 * (1 smax %n)) + %s1 + %a))
+; NOMERGE-NEXT: Member: {(%s1 + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP6:
; NOMERGE-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
@@ -3053,10 +3074,6 @@ define void @nested_multiplier_ok(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in
; NOMERGE-NEXT: Expressions re-written:
;
entry:
- %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
- %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
- %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
- %s4 = call i64 @llvm.smax.i64(i64 %s4in, i64 1)
%ns1 = sub i64 0, %s1
%s1x2 = shl i64 %s1, 1
%ns1x2 = sub i64 0, %s1x2
@@ -3364,6 +3381,9 @@ define void @corner_sum_no_overflow(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: (1 smax %s3in) sle) 2
+; MERGE-NEXT: Compare predicate: (1 smax %s2in) sle) 2
+; MERGE-NEXT: Compare predicate: (1 smax %s1in) sle) 2
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
>From 5b1f9c9d62346a0aca6a34fd2b16e1a4eda2decc Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 11 Sep 2026 12:57:43 +0000
Subject: [PATCH 30/33] Prevent merging depsets with forked pointers
---
.../llvm/Analysis/LoopAccessAnalysis.h | 9 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 38 +++--
.../stencil-group-merging.ll | 134 +++++++-----------
3 files changed, 81 insertions(+), 100 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 8cd98c03675b4..914283b35b36a 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -568,13 +568,16 @@ class RuntimePointerChecking {
const SCEV *Expr;
/// True if the pointer expressions needs to be frozen after expansion.
bool NeedsFreeze;
+ /// True if this entry represents one arm of a forked pointer.
+ bool IsForked;
PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End,
bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId,
- const SCEV *Expr, bool NeedsFreeze)
+ const SCEV *Expr, bool NeedsFreeze, bool IsForked)
: PointerValue(PointerValue), Start(Start), End(End),
IsWritePtr(IsWritePtr), DependencySetId(DependencySetId),
- AliasSetId(AliasSetId), Expr(Expr), NeedsFreeze(NeedsFreeze) {}
+ AliasSetId(AliasSetId), Expr(Expr), NeedsFreeze(NeedsFreeze),
+ IsForked(IsForked) {}
};
RuntimePointerChecking(MemoryDepChecker &DC, ScalarEvolution *SE,
@@ -600,7 +603,7 @@ class RuntimePointerChecking {
LLVM_ABI bool insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
Type *AccessTy, bool WritePtr, unsigned DepSetId,
unsigned ASId, PredicatedScalarEvolution &PSE,
- bool NeedsFreeze);
+ bool NeedsFreeze, bool IsForked);
/// No run-time memory checking is necessary.
bool empty() const { return Pointers.empty(); }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index b8dec07e29a14..7a0d6cdd5ebf8 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -528,7 +528,7 @@ bool RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
Type *AccessTy, bool WritePtr,
unsigned DepSetId, unsigned ASId,
PredicatedScalarEvolution &PSE,
- bool NeedsFreeze) {
+ bool NeedsFreeze, bool IsForked) {
const SCEV *SymbolicMaxBTC = PSE.getSymbolicMaxBackedgeTakenCount();
const SCEV *BTC = PSE.getBackedgeTakenCount();
const auto &[ScStart, ScEnd] = getStartAndEndForAccess(
@@ -537,7 +537,7 @@ bool RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
if (isa<SCEVCouldNotCompute>(ScStart) || isa<SCEVCouldNotCompute>(ScEnd))
return false;
Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, PtrExpr,
- NeedsFreeze);
+ NeedsFreeze, IsForked);
return true;
}
@@ -1208,10 +1208,15 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// range also covers the gaps between the members, so the merged check
// can report a conflict where the per-group checks would not.
//
+ // We only merge ranges for reads that happen on every loop iteration.
+ // These reads must stay inside the array; otherwise, the original loop
+ // already has undefined behaviour. We choose the merged bounds from
+ // these ranges.
+ //
// We use the following algorithm to construct a merged stencil group:
// - collect checking groups that share both DependencySetId and AliasSetId;
- // - reject groups with writes, predicated accesses, different access
- // ranges, or different recurrence steps;
+ // - reject groups with writes, predicated accesses, forked pointers,
+ // different access ranges, or different recurrence steps;
// - use one member as the base and decompose each other member's offset
// from that base as C + sum(Coeff[Stride] * Stride), where Stride is
// loop-invariant;
@@ -1301,21 +1306,31 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// merged group complicates the cost model and doesn't match known stencil
// patterns, so stop and skip the whole DepSet as soon as we see a write.
SmallVector<unsigned, 8> AllMembers;
- bool HasWrite = false;
+ bool CanMerge = true;
for (unsigned GI : GroupIndices) {
ArrayRef<unsigned> Members = CheckingGroups[GI].Members;
if (any_of(Members,
[&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
- HasWrite = true;
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with write access\n");
+ CanMerge = false;
+ break;
+ }
+ // For a forked pointer, LAA considers both possible addresses, even if
+ // the loop only uses one of them. The unused address can be outside the
+ // array. Its bounds can underflow or overflow, so merging them can hide
+ // an overlap and allow unsafe vectorization.
+ if (any_of(Members,
+ [&](unsigned Idx) { return Pointers[Idx].IsForked; })) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with forked pointer\n");
+ CanMerge = false;
break;
}
append_range(AllMembers, Members);
}
- if (HasWrite) {
- LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
- << ") with write access\n");
+ if (!CanMerge)
continue;
- }
// We do not allow predicated accesses. They may result in overestimation
// of the boundaries. Imagine a stencil access where we must skip some first
@@ -2184,7 +2199,8 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
bool IsWrite = Access.getInt();
if (!RtCheck.insert(TheLoop, Ptr, PtrExpr, AccessTy, IsWrite, DepId, ASId,
- PSE, NeedsFreeze)) {
+ PSE, NeedsFreeze,
+ /*IsForked=*/RTCheckPtrs.size() == 2)) {
RtCheck.Pointers.truncate(NumPointers);
return false;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index 45da8011b3abf..5d42926c58d08 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -2563,92 +2563,54 @@ exit:
ret void
}
-;; Test 23: A forked pointer: one load's address is a select.
-;; 3 loads from %a at byte offsets {0, cdj, select(2*cdj, 3*cdj)}. Store to
-;; %out.
-;; LAA turns the select load into two members, one per arm, each with its
-;; own Start and End. The merge treats them like any other member. High is
-;; the End of the 3*cdj arm, so both arms are inside the merged bounds.
-;; With merge: 1 group, 1 check, 2 predicates (0 < cdj <= INT64_MAX / 2).
-;; Without merge: 4 groups, 4 checks, no predicates.
-define void @forked_pointer_merge(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
-; MERGE-LABEL: 'forked_pointer_merge'
-; MERGE-NEXT: loop:
-; MERGE-NEXT: Memory dependences are safe with run-time checks
-; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Run-time memory checks:
-; MERGE-NEXT: Check 0:
-; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Grouped accesses:
-; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: %a High: ((3 * %cdj) + (8 * (1 smax %n)) + %a))
-; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {((3 * %cdj) + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; MERGE-EMPTY:
-; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; MERGE-NEXT: SCEV assumptions:
-; MERGE-NEXT: Compare predicate: %cdj sgt) 0
-; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
-; MERGE-EMPTY:
-; MERGE-NEXT: Expressions re-written:
-;
-; NOMERGE-LABEL: 'forked_pointer_merge'
-; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Memory dependences are safe with run-time checks
-; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Run-time memory checks:
-; NOMERGE-NEXT: Check 0:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
-; NOMERGE-NEXT: Check 2:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
-; NOMERGE-NEXT: Check 3:
-; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Grouped accesses:
-; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
-; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((2 * %cdj) + (8 * (1 smax %n)) + %a))
-; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((3 * %cdj) + %a) High: ((3 * %cdj) + (8 * (1 smax %n)) + %a))
-; NOMERGE-NEXT: Member: {((3 * %cdj) + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
-; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
-; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
-; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
-; NOMERGE-NEXT: SCEV assumptions:
-; NOMERGE-EMPTY:
-; NOMERGE-NEXT: Expressions re-written:
+;; Test 23: Reject forked pointers from stencil merging.
+define void @forked_pointer_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i32 %c) {
+; CHECK-LABEL: 'forked_pointer_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %off
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: ((2 * %cdj) + (8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((3 * %cdj) + %a) High: ((3 * %cdj) + (8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {((3 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((8 * (1 smax %n)) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: %a High: ((8 * (1 smax %n)) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
;
entry:
br label %loop
>From 27d87ca4a356ad1d53e698f2bf30b624891af2c3 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 17 Sep 2026 13:40:55 +0100
Subject: [PATCH 31/33] Address comments
Refactor Stride limit related code into `StrideLimits` class
One extra test and one test changed
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 188 ++++++++++--------
.../stencil-group-merging-i128.ll | 77 +++++++
.../stencil-group-merging.ll | 41 ++--
3 files changed, 208 insertions(+), 98 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 7a0d6cdd5ebf8..a7c5e2c5393b8 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -989,6 +989,95 @@ getStencilStrideUpperLimit(const StencilDecomposition &D, unsigned BitWidth) {
return APInt(BitWidth, CoeffSum ? Budget / CoeffSum : SignedMax);
}
+namespace {
+/// The runtime checks the merge needs on each stride.
+/// Example:
+/// s1: {NeedsPositive = true, Max = 1000} means the checks 1 <= s1 <= 1000
+/// s2: {Max = 50} means the check s2 <= 50
+/// The lower limit is always 1, so a flag is enough for it.
+/// Several members can each ask for an upper limit on the same stride, but only
+/// the smallest one is kept.
+class StrideLimits {
+ struct Limit {
+ bool NeedsPositive = false;
+ std::optional<APInt> Max;
+ };
+ SmallMapVector<const SCEV *, Limit, 4> Limits;
+
+public:
+ void requireLowerLimit(const SCEV *Stride) {
+ Limits[Stride].NeedsPositive = true;
+ }
+
+ void requireUpperLimit(const SCEV *Stride, const APInt &Max) {
+ std::optional<APInt> &Current = Limits[Stride].Max;
+ if (!Current || Max.ult(*Current))
+ Current = Max;
+ }
+
+ /// Add every new or more strict check in \p Other to this set.
+ void addFrom(const StrideLimits &Other) {
+ for (const auto &[Stride, L] : Other.Limits) {
+ if (L.NeedsPositive)
+ requireLowerLimit(Stride);
+ if (L.Max)
+ requireUpperLimit(Stride, *L.Max);
+ }
+ }
+
+ /// Count the strides that have no check in \p Committed yet.
+ unsigned countNew(const StrideLimits &Committed) const {
+ return count_if(Limits, [&](const auto &Entry) {
+ return !Committed.Limits.contains(Entry.first);
+ });
+ }
+
+ /// Add the checks to \p PSE as SCEV predicates.
+ void addPredicates(PredicatedScalarEvolution &PSE) const {
+ ScalarEvolution &SE = *PSE.getSE();
+ for (const auto &[Stride, L] : Limits) {
+ if (!L.NeedsPositive)
+ continue;
+ const SCEV *Zero = SE.getZero(Stride->getType());
+ PSE.addPredicate(
+ *SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+ for (const auto &[Stride, L] : Limits) {
+ if (!L.Max)
+ continue;
+ PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SLE, Stride,
+ SE.getConstant(*L.Max)));
+ LLVM_DEBUG(dbgs() << "LAA: Adding stride upper-limit predicate "
+ << *Stride << " <= " << *L.Max << "\n");
+ }
+ }
+};
+} // namespace
+
+/// Add to \p Limits the checks each stride s of \p D needs:
+/// 1 <= s isNeverAbove assumes every stride is 1 or more.
+/// s <= Max Max is from getStencilStrideUpperLimit.
+/// A check is skipped when SCEV already proves it.
+/// Returns false if getStencilStrideUpperLimit finds no Max.
+static bool collectStrideLimits(const StencilDecomposition &D,
+ unsigned BitWidth, ScalarEvolution &SE,
+ StrideLimits &Limits) {
+ std::optional<APInt> UpperLimit = getStencilStrideUpperLimit(D, BitWidth);
+ if (!UpperLimit)
+ return false;
+
+ for (const auto &[Stride, Coeff] : D.Coefficients) {
+ if (!SE.isKnownPositive(Stride))
+ Limits.requireLowerLimit(Stride);
+ if (!SE.isKnownPredicate(ICmpInst::ICMP_SLE, Stride,
+ SE.getConstant(*UpperLimit)))
+ Limits.requireUpperLimit(Stride, *UpperLimit);
+ }
+ return true;
+}
+
/// Return true if offset A is never higher than offset B.
/// A and B are these sums:
/// A = A.Constant + CoefA_1 * stride_1 + CoefA_2 * stride_2 + ...
@@ -1107,10 +1196,7 @@ collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
static std::pair<unsigned, unsigned> computeStencilMergeCost(
const RuntimePointerChecking &RtCheck, ArrayRef<unsigned> GroupIndices,
const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
- ArrayRef<const SCEV *> LocalStrideLowerLimits,
- const SmallMapVector<const SCEV *, APInt, 4> &LocalStrideUpperLimits,
- const SmallDenseSet<const SCEV *, 4> &StrideLowerLimits,
- const SmallMapVector<const SCEV *, APInt, 4> &StrideUpperLimits,
+ const StrideLimits &Local, const StrideLimits &Committed,
unsigned NumBoundOperands) {
ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
unsigned NumGroups = GroupIndices.size();
@@ -1128,19 +1214,7 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
}
}
- // Charge once per stride, even when both limits need runtime predicates.
- // Later DepSets can narrow an upper limit without adding another check.
- const auto NeedsNewCheck = [&](const SCEV *Stride) {
- return !StrideLowerLimits.contains(Stride) &&
- !StrideUpperLimits.contains(Stride);
- };
- unsigned NewPredicates = 0;
- for (const SCEV *Stride : LocalStrideLowerLimits)
- if (NeedsNewCheck(Stride))
- ++NewPredicates;
- for (const auto &[Stride, UpperLimit] : LocalStrideUpperLimits)
- if (!is_contained(LocalStrideLowerLimits, Stride) && NeedsNewCheck(Stride))
- ++NewPredicates;
+ unsigned NewPredicates = Local.countNew(Committed);
LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
<< ", NumExternalChecks=" << NumExternalChecks
@@ -1156,17 +1230,13 @@ static std::pair<unsigned, unsigned> computeStencilMergeCost(
/// Build the merged stencil group for one DepSet, after the cost model has
/// decided the merge is profitable. Constructs the bounding group over
-/// \p AllMembers with bounds [\p MergedLow, \p MergedHigh], and registers with
-/// \p PSE a positive-stride SCEV predicate for each stride in
-/// \p LocalStrideLowerLimits that has not already been committed (tracked in
-/// \p StrideLowerLimits across DepSets). Returns the new group.
+/// \p AllMembers with bounds [\p MergedLow, \p MergedHigh]. Returns the new
+/// group.
static RuntimeCheckingPtrGroup
buildMergedStencilGroup(const RuntimePointerChecking &RtCheck,
- PredicatedScalarEvolution &PSE,
ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
- const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
- ArrayRef<const SCEV *> LocalStrideLowerLimits,
- SmallDenseSet<const SCEV *, 4> &StrideLowerLimits) {
+ const SCEV *MergedHigh,
+ ArrayRef<unsigned> GroupIndices) {
RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], RtCheck);
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
@@ -1174,18 +1244,6 @@ buildMergedStencilGroup(const RuntimePointerChecking &RtCheck,
CandidateGroup.NeedsFreeze = any_of(GroupIndices, [&](unsigned GI) {
return RtCheck.CheckingGroups[GI].NeedsFreeze;
});
-
- // Register the positive-stride SCEV predicates with PSE, skipping any stride
- // an earlier DepSet already added a predicate for.
- ScalarEvolution &SE = *PSE.getSE();
- for (const SCEV *Stride : LocalStrideLowerLimits) {
- if (!StrideLowerLimits.insert(Stride).second)
- continue;
- const SCEV *Zero = SE.getZero(Stride->getType());
- PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
- LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
- << *Stride << "\n");
- }
return CandidateGroup;
}
@@ -1289,11 +1347,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
SmallDenseSet<unsigned, 4> MergedGroupIndices;
SmallVector<RuntimeCheckingPtrGroup, 2> NewMergedGroups;
- // Track lower limits from accepted DepSets. Each is 1, so store only strides.
- SmallDenseSet<const SCEV *, 4> StrideLowerLimits;
- // Accepted DepSets may narrow these limits. Emit their predicates only
- // after all DepSets have been analyzed.
- SmallMapVector<const SCEV *, APInt, 4> StrideUpperLimits;
+ // Stride checks from the accepted DepSets. A later DepSet can lower an
+ // upper limit, so the predicates are added only after the last DepSet.
+ StrideLimits CommittedStrideLimits;
for (auto &[DepAliasKey, GroupIndices] : DepSetToGroups) {
[[maybe_unused]] auto [DepId, ASId] = DepAliasKey;
@@ -1439,8 +1495,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
MemberOffsets.reserve(AllMembers.size());
// The base member's offset from itself is zero: Constant 0, no strides.
MemberOffsets.emplace_back();
- SmallSetVector<const SCEV *, 4> LocalStrideLowerLimits;
- SmallMapVector<const SCEV *, APInt, 4> LocalStrideUpperLimits;
+ // Stride checks this DepSet needs if it is merged.
+ StrideLimits LocalStrideLimits;
// Decompose one member's offset (relative to BaseLow) and append it to
// MemberOffsets. Returns false if the offset is not in stencil form (so
@@ -1455,28 +1511,11 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< " NOT decomposable: " << *LowOffset << "\n");
return false;
}
-
- auto UpperLimit = getStencilStrideUpperLimit(
- *DLow, SE->getTypeSizeInBits(LowOffset->getType()));
- if (!UpperLimit)
+ if (!collectStrideLimits(*DLow,
+ SE->getTypeSizeInBits(LowOffset->getType()), *SE,
+ LocalStrideLimits))
return false;
- for (const auto &[Stride, Coeff] : DLow->Coefficients) {
- // Require a positive stride if needed. The lower limit is always 1.
- if (!SE->isKnownPositive(Stride))
- LocalStrideLowerLimits.insert(Stride);
-
- // Check whether an upper-limit predicate is needed.
- if (!Coeff || SE->isKnownPredicate(ICmpInst::ICMP_SLE, Stride,
- SE->getConstant(*UpperLimit)))
- continue;
- auto [It, Inserted] =
- LocalStrideUpperLimits.insert({Stride, *UpperLimit});
- // Keep the smallest limit seen so far.
- if (!Inserted && UpperLimit->ult(It->second))
- It->second = *UpperLimit;
- }
-
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< ": Const=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
@@ -1508,9 +1547,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// it before building the merged bounds: a rejected DepSet then creates no
// umin/umax expressions that would only be thrown away.
auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
- *this, GroupIndices, MergedGroupIndices,
- LocalStrideLowerLimits.getArrayRef(), LocalStrideUpperLimits,
- StrideLowerLimits, StrideUpperLimits, NumBoundOperands);
+ *this, GroupIndices, MergedGroupIndices, LocalStrideLimits,
+ CommittedStrideLimits, NumBoundOperands);
if (ChecksAfter >= ChecksBefore) {
LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
@@ -1542,21 +1580,13 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Merging, net saving "
<< ChecksBefore - ChecksAfter << "\n");
- // Build the merged group and register its positive-stride predicates.
NewMergedGroups.push_back(buildMergedStencilGroup(
- *this, PSE, AllMembers, MergedLow, MergedHigh, GroupIndices,
- LocalStrideLowerLimits.getArrayRef(), StrideLowerLimits));
- for (const auto &[Stride, UpperLimit] : LocalStrideUpperLimits) {
- auto [It, Inserted] = StrideUpperLimits.insert({Stride, UpperLimit});
- if (!Inserted && UpperLimit.ult(It->second))
- It->second = UpperLimit;
- }
+ *this, AllMembers, MergedLow, MergedHigh, GroupIndices));
+ CommittedStrideLimits.addFrom(LocalStrideLimits);
MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
}
- for (const auto &[Stride, UpperLimit] : StrideUpperLimits)
- PSE.addPredicate(*SE->getComparePredicate(ICmpInst::ICMP_SLE, Stride,
- SE->getConstant(UpperLimit)));
+ CommittedStrideLimits.addPredicates(PSE);
// Rebuild CheckingGroups if we merged anything.
if (!NewMergedGroups.empty()) {
@@ -2200,7 +2230,7 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
bool IsWrite = Access.getInt();
if (!RtCheck.insert(TheLoop, Ptr, PtrExpr, AccessTy, IsWrite, DepId, ASId,
PSE, NeedsFreeze,
- /*IsForked=*/RTCheckPtrs.size() == 2)) {
+ /*IsForked=*/RTCheckPtrs.size() > 1)) {
RtCheck.Pointers.truncate(NumPointers);
return false;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
index 8dae803c5b6ba..679c5e2f2510f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging-i128.ll
@@ -78,3 +78,80 @@ loop:
exit:
ret void
}
+
+; The same loop with small coefficients {1, 2, 3}. The offsets decompose, but
+; the index type is 128 bits wide, so the merge must skip the DepSet.
+define void @stencil_wide_index_small_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj) {
+; CHECK-LABEL: 'stencil_wide_index_small_coeff'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((3 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (3 * %cdj) + %a))
+; CHECK-NEXT: Member: {((3 * %cdj) + %a),+,1}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
+; CHECK-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %ivx = zext i64 %iv to i128
+
+ %o0 = mul i128 %cdj, 1
+ %t0 = add i128 %o0, %ivx
+ %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+ %v0 = load i8, ptr %p0
+
+ %o1 = mul i128 %cdj, 2
+ %t1 = add i128 %o1, %ivx
+ %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+ %v1 = load i8, ptr %p1
+
+ %o2 = mul i128 %cdj, 3
+ %t2 = add i128 %o2, %ivx
+ %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+ %v2 = load i8, ptr %p2
+
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+ store i8 %s, ptr %op
+
+ %iv.next = add i64 %iv, 1
+ %c = icmp slt i64 %iv.next, %n
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index 5d42926c58d08..6a9e424ba1d62 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --function nested_multiplier_ok --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,MERGE %s
; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,NOMERGE %s
@@ -646,9 +646,10 @@ exit:
;; Test 8: Shared stride predicate deduplication across two DepSets.
-;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
-;; Both DepSets share the same stride limits, so each stride predicate
-;; should be emitted once.
+;; %a is read at offsets {-cdj, 0, +cdj}, %b at {-2*cdj, 0, +2*cdj}.
+;; Store to %out.
+;; Both DepSets need limits on %cdj. Each predicate is emitted once.
+;; %b needs a smaller upper limit than %a, and the smaller one is kept.
;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 2 predicates.
;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
@@ -668,9 +669,9 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj2
; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
-; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj2
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -681,15 +682,15 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + %cdj + %b))
-; MERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; MERGE-NEXT: (Low: (24 + (-2 * %cdj) + %b) High: (-24 + (2 * %cdj) + (8 * %n) + %b))
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + %b),+,8}<nw><%loop>
; MERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
-; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-2 * %cdj) + %b),+,8}<nw><%loop>
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %cdj sgt) 0
-; MERGE-NEXT: Compare predicate: %cdj sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: %cdj sle) 2305843009213693951
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
;
@@ -717,7 +718,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj2
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -727,7 +728,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj2
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -742,14 +743,14 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP4:
-; NOMERGE-NEXT: (Low: (24 + %cdj + %b) High: (-24 + (8 * %n) + %cdj + %b))
-; NOMERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + %b) High: (-24 + (2 * %cdj) + (8 * %n) + %b))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %b),+,8}<nw><%loop>
; NOMERGE-NEXT: Group GRP5:
; NOMERGE-NEXT: (Low: (24 + %b) High: (-24 + (8 * %n) + %b))
; NOMERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP6:
-; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + (-1 * %cdj) + %b))
-; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: (Low: (24 + (-2 * %cdj) + %b) High: (-24 + (8 * %n) + (-2 * %cdj) + %b))
+; NOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %b),+,8}<nw><%loop>
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; NOMERGE-NEXT: SCEV assumptions:
@@ -772,12 +773,14 @@ loop:
%pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
%va2 = load double, ptr %pa2, align 8
- ; Reads from %b at {-cdj, 0, +cdj}
+ ; Reads from %b at {-2*cdj, 0, +2*cdj}
%baseb = getelementptr inbounds double, ptr %b, i64 %iv
- %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+ %negcdj2 = mul nsw i64 %cdj, -2
+ %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj2
%vb0 = load double, ptr %pb0, align 8
%vb1 = load double, ptr %baseb, align 8
- %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+ %cdj2 = mul nsw i64 %cdj, 2
+ %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj2
%vb2 = load double, ptr %pb2, align 8
%s0 = fadd double %va0, %va1
>From 6c119d8339cf329c88a8cae7323048f3f07e4a31 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 18 Sep 2026 13:16:40 +0100
Subject: [PATCH 32/33] Add tests
---
.../stencil-group-merging.ll | 290 +++++++++++++++++-
1 file changed, 285 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index 6a9e424ba1d62..18245c4bebaf6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -3185,12 +3185,12 @@ exit:
ret void
}
-;; Test 26: A member whose corner sum overflows int64 in isNeverAbove.
+;; Test 26: A member whose coefficients are too large for any stride limit.
;; 7 loads from %a at offsets {-C*(s1+s2+s3), +-s1, +-s2, +-s3}.
-;; @corner_sum_overflow: C = 2^62. The far member's corner sum (its offset
-;; at s1 = s2 = s3 = 1) is -3 * 2^62, below int64 min, so isNeverAbove gives
-;; up and keeps the member as a candidate on both sides: 4 candidates for
-;; Low, 4 for High, 6 bound operands, and the cost model rejects the merge.
+;; @corner_sum_overflow: C = 2^62. The far member's absolute coefficients add
+;; up to 3 * 2^62, more than int64 max. Even s1 = s2 = s3 = 1 gives an offset
+;; below int64 min. So getStencilStrideUpperLimit finds no stride limit and
+;; the DepSet is skipped.
;; @corner_sum_no_overflow: C = 2^60. The corner sums fit, the far member
;; alone is Low and High is a 3-way umax. Checks: 7 before the merge, 3 after.
define void @corner_sum_overflow(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
@@ -3545,3 +3545,283 @@ outer.latch:
exit:
ret void
}
+
+;; Test 28: A member whose constant offset is too far from the base.
+;; 7 loads from %a at offsets {C - s1 - s2 - s3, +-s1, +-s2, +-s3}.
+;; @constant_offset_too_large: C = -2^63. abs(C) is 2^63, more than int64
+;; max, so getStencilStrideUpperLimit finds no stride limit and the DepSet is
+;; skipped. Checks: 7 before, 7 after.
+;; @constant_offset_fits: C = -2^62. The stride limit is (2^63 - 1 - 2^62) / 2
+;; and the merge fires. Checks: 7 before the merge, 1 after.
+define void @constant_offset_too_large(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
+; CHECK-LABEL: 'constant_offset_too_large'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP5:
+; CHECK-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; CHECK-NEXT: Check 5:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP6:
+; CHECK-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; CHECK-NEXT: Check 6:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP7:
+; CHECK-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s3in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s3in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((1 smax %s3in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s3in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s2in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: ((1 smax %s2in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s2in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP5:
+; CHECK-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; CHECK-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP6:
+; CHECK-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; CHECK-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP7:
+; CHECK-NEXT: (Low: (-9223372036854775808 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a) High: (-9223372036854775808 + (8 * (1 smax %n)) + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a))
+; CHECK-NEXT: Member: {(-9223372036854775808 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %ns2 = sub i64 0, %s2
+ %ns3 = sub i64 0, %s3
+ %s12 = add i64 %s1, %s2
+ %s123 = add i64 %s12, %s3
+ %boff = sub i64 -9223372036854775808, %s123 ; -2^63 - s1 - s2 - s3
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %pb = getelementptr i8, ptr %base, i64 %boff
+ %vb = load double, ptr %pb, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+ %p6 = getelementptr i8, ptr %base, i64 %ns3
+ %v6 = load double, ptr %p6, align 8
+ %t1 = fadd double %vb, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %t6 = fadd double %t5, %v6
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Same loop with C = -2^62.
+define void @constant_offset_fits(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
+; MERGE-LABEL: 'constant_offset_fits'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; MERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; MERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; MERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; MERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; MERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; MERGE-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (-4611686018427387904 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a) High: (((8 * (1 smax %n)) + (1 smax %s3in) + %a) umax ((8 * (1 smax %n)) + (1 smax %s2in) + %a) umax ((8 * (1 smax %n)) + (1 smax %s1in) + %a)))
+; MERGE-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(-4611686018427387904 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: (1 smax %s3in) sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: (1 smax %s2in) sle) 2305843009213693951
+; MERGE-NEXT: Compare predicate: (1 smax %s1in) sle) 2305843009213693951
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'constant_offset_fits'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p6 = getelementptr i8, ptr %base, i64 %ns3
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p5 = getelementptr i8, ptr %base, i64 %s3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p4 = getelementptr i8, ptr %base, i64 %ns2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p3 = getelementptr i8, ptr %base, i64 %s2
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p2 = getelementptr i8, ptr %base, i64 %ns1
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p1 = getelementptr i8, ptr %base, i64 %s1
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP7:
+; NOMERGE-NEXT: %pb = getelementptr i8, ptr %base, i64 %boff
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * (1 smax %n)) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s3in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s3in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s3in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((1 smax %s3in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s3in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s3in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s2in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s2in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((1 smax %s2in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s2in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s2in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: ((-1 * (1 smax %s1in))<nsw> + %a) High: ((8 * (1 smax %n)) + (-1 * (1 smax %s1in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {((-1 * (1 smax %s1in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %a) High: ((8 * (1 smax %n)) + (1 smax %s1in) + %a))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP7:
+; NOMERGE-NEXT: (Low: (-4611686018427387904 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a) High: (-4611686018427387904 + (8 * (1 smax %n)) + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a))
+; NOMERGE-NEXT: Member: {(-4611686018427387904 + (-1 * ((1 smax %s1in) + (1 smax %s2in) + (1 smax %s3in))) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s3 = call i64 @llvm.smax.i64(i64 %s3in, i64 1)
+ %ns1 = sub i64 0, %s1
+ %ns2 = sub i64 0, %s2
+ %ns3 = sub i64 0, %s3
+ %s12 = add i64 %s1, %s2
+ %s123 = add i64 %s12, %s3
+ %boff = sub i64 -4611686018427387904, %s123 ; -2^62 - s1 - s2 - s3
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %pb = getelementptr i8, ptr %base, i64 %boff
+ %vb = load double, ptr %pb, align 8
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+ %p2 = getelementptr i8, ptr %base, i64 %ns1
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+ %p4 = getelementptr i8, ptr %base, i64 %ns2
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+ %p6 = getelementptr i8, ptr %base, i64 %ns3
+ %v6 = load double, ptr %p6, align 8
+ %t1 = fadd double %vb, %v1
+ %t2 = fadd double %t1, %v2
+ %t3 = fadd double %t2, %v3
+ %t4 = fadd double %t3, %v4
+ %t5 = fadd double %t4, %v5
+ %t6 = fadd double %t5, %v6
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
>From 9572d91baff993684fbe3659e60b0595702ec0c3 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Mon, 21 Sep 2026 14:40:17 +0000
Subject: [PATCH 33/33] Address comments
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 28 +++++++++----------
.../stencil-group-merging.ll | 20 ++++++-------
2 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index a7c5e2c5393b8..e00056fbfaeb6 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1036,21 +1036,19 @@ class StrideLimits {
void addPredicates(PredicatedScalarEvolution &PSE) const {
ScalarEvolution &SE = *PSE.getSE();
for (const auto &[Stride, L] : Limits) {
- if (!L.NeedsPositive)
- continue;
- const SCEV *Zero = SE.getZero(Stride->getType());
- PSE.addPredicate(
- *SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
- LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
- << *Stride << "\n");
- }
- for (const auto &[Stride, L] : Limits) {
- if (!L.Max)
- continue;
- PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SLE, Stride,
- SE.getConstant(*L.Max)));
- LLVM_DEBUG(dbgs() << "LAA: Adding stride upper-limit predicate "
- << *Stride << " <= " << *L.Max << "\n");
+ if (L.NeedsPositive) {
+ const SCEV *Zero = SE.getZero(Stride->getType());
+ PSE.addPredicate(
+ *SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+ if (L.Max) {
+ PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SLE, Stride,
+ SE.getConstant(*L.Max)));
+ LLVM_DEBUG(dbgs() << "LAA: Adding stride upper-limit predicate "
+ << *Stride << " <= " << *L.Max << "\n");
+ }
}
}
};
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
index 18245c4bebaf6..8ab2cb952fbb1 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/stencil-group-merging.ll
@@ -1327,8 +1327,8 @@ define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
-; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sle) 614891469123651720
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sle) 614891469123651720
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
@@ -1650,8 +1650,8 @@ define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i
; MERGE-NEXT: {((64 * %s2) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
-; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sle) 72057594037927935
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sle) 72057594037927935
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
@@ -1847,10 +1847,10 @@ define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1,
; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
-; MERGE-NEXT: Compare predicate: (4 * %s3) sgt) 0
-; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sle) 48038396025285290
+; MERGE-NEXT: Compare predicate: (4 * %s3) sgt) 0
; MERGE-NEXT: Compare predicate: (4 * %s3) sle) 48038396025285290
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sle) 48038396025285290
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
@@ -2215,10 +2215,10 @@ define void @stencil_merge_three_stride_star(ptr %a, ptr %out, ptr %out2, i64 %n
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
; MERGE-NEXT: Compare predicate: %s3 sgt) 0
-; MERGE-NEXT: Compare predicate: %s2 sgt) 0
-; MERGE-NEXT: Compare predicate: %s1 sgt) 0
; MERGE-NEXT: Compare predicate: %s3 sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
; MERGE-NEXT: Compare predicate: %s2 sle) 4611686018427387903
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sle) 4611686018427387903
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
@@ -2755,12 +2755,12 @@ define void @stencil_merge_depth_cap_sum(ptr %p, ptr %q, ptr %q2, ptr %q3, ptr %
; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
; MERGE-NEXT: Compare predicate: %s1 sgt) 0
-; MERGE-NEXT: Compare predicate: (%s1 + %s2) sgt) 0
-; MERGE-NEXT: Compare predicate: %s3 sgt) 0
-; MERGE-NEXT: Compare predicate: %s4 sgt) 0
; MERGE-NEXT: Compare predicate: %s1 sle) 144115188075855871
+; MERGE-NEXT: Compare predicate: (%s1 + %s2) sgt) 0
; MERGE-NEXT: Compare predicate: (%s1 + %s2) sle) 2150972956356057
+; MERGE-NEXT: Compare predicate: %s3 sgt) 0
; MERGE-NEXT: Compare predicate: %s3 sle) 2150972956356057
+; MERGE-NEXT: Compare predicate: %s4 sgt) 0
; MERGE-NEXT: Compare predicate: %s4 sle) 2150972956356057
; MERGE-EMPTY:
; MERGE-NEXT: Expressions re-written:
@@ -3550,7 +3550,7 @@ exit:
;; 7 loads from %a at offsets {C - s1 - s2 - s3, +-s1, +-s2, +-s3}.
;; @constant_offset_too_large: C = -2^63. abs(C) is 2^63, more than int64
;; max, so getStencilStrideUpperLimit finds no stride limit and the DepSet is
-;; skipped. Checks: 7 before, 7 after.
+;; skipped.
;; @constant_offset_fits: C = -2^62. The stride limit is (2^63 - 1 - 2^62) / 2
;; and the merge fires. Checks: 7 before the merge, 1 after.
define void @constant_offset_too_large(ptr %a, ptr %out, i64 %n, i64 %s1in, i64 %s2in, i64 %s3in) {
More information about the llvm-commits
mailing list