[llvm] [LAA] Add stencil group merging to reduce runtime pointer checks (PR #187252)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 08:10:43 PDT 2026
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/187252
>From 69ad00eb847ace4fa82a5afdcfd2fcfc7906678a Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 12:17:21 +0000
Subject: [PATCH 01/14] [LAA] Add stencil group merging to reduce runtime
pointer checks
Add a post-processing pass in RuntimePointerChecking that merges
checking groups whose pointers share a base object and differ only
by linear combinations of loop-invariant strides. This reduces the
number of runtime overlap checks for stencil-like access patterns,
enabling vectorization of loops that would otherwise be rejected
due to exceeding the runtime-check threshold.
The primary motivation is the Einstein Toolkit / Cactus CCZ4
numerical relativity stencil code, where 3D stencils with two
runtime strides (cdj, cdk) produce thousands of runtime checks.
The optimization:
- Decomposes member offsets into constant + sum(coeff * stride)
- Computes a bounding box over the coefficient space
- Applies a cost model: only merges when checks_after < checks_before
- Adds stride > 0 predicates for non-provably-positive strides
- Rejects predicated accesses and write groups conservatively
Gated behind -enable-stencil-runtime-check-merge (default: off).
Also activates automatically when check count exceeds
-stencil-merge-check-threshold (default: 128).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 8 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 380 ++++-
.../affine-group-merging.ll | 1406 +++++++++++++++++
3 files changed, 1791 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 85901ebddc7f0..644c1b24788e3 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -567,7 +567,8 @@ class RuntimePointerChecking {
/// Generate the checks and store it. This also performs the grouping
/// of pointers to reduce the number of memchecks necessary.
- LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands);
+ LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands,
+ PredicatedScalarEvolution &PSE, Loop &L);
/// Returns the checks that generateChecks created. They can be used to ensure
/// no read/write accesses overlap across all loop iterations.
@@ -637,6 +638,11 @@ class RuntimePointerChecking {
/// and re-compute it.
void groupChecks(MemoryDepChecker::DepCandidates &DepCands);
+ /// Attempt to merge checking groups that share a base pointer and differ
+ /// by stencil functions of loop-invariant strides. This reduces runtime
+ /// checks for multi-dimensional stencil-like access patterns.
+ void mergeStencilGroups(PredicatedScalarEvolution &PSE, Loop &L);
+
/// Generate the checks and return them.
SmallVector<RuntimePointerCheck, 4> generateChecks();
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 1a1f458d00253..5015be674fe18 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
@@ -99,6 +100,18 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable merging of runtime check groups with stencil stride "
+ "patterns (default = false)"),
+ cl::init(false));
+
+static cl::opt<unsigned> StencilMergeCheckThreshold(
+ "stencil-merge-check-threshold", cl::Hidden,
+ cl::desc("Trigger stencil group merging when runtime check count "
+ "exceeds this threshold (default = 128)"),
+ cl::init(128));
+
/// Maximum SIMD width.
const unsigned VectorizerParams::MaxVectorWidth = 64;
@@ -549,9 +562,11 @@ SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() {
}
void RuntimePointerChecking::generateChecks(
- MemoryDepChecker::DepCandidates &DepCands) {
+ MemoryDepChecker::DepCandidates &DepCands, PredicatedScalarEvolution &PSE,
+ Loop &L) {
assert(Checks.empty() && "Checks is not empty");
groupChecks(DepCands);
+ mergeStencilGroups(PSE, L);
Checks = generateChecks();
}
@@ -734,6 +749,367 @@ void RuntimePointerChecking::groupChecks(
}
}
+/// Result of decomposing a SCEV expression into stencil offset form:
+/// Offset = Constant + sum(Coefficients[stride] * stride)
+/// where each stride is a loop-invariant SCEV expression.
+struct StencilDecomposition {
+ int64_t Constant = 0;
+ /// Map from loop-invariant stride SCEV to its integer coefficient.
+ SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+};
+
+/// Try to decompose \p Expr into a stencil offset function of loop-invariant
+/// strides: C + a1*s1 + a2*s2 + ...
+/// Relies on SCEV's canonical form: AddExpr operands are flattened,
+/// MulExpr has the constant operand first.
+/// Returns std::nullopt if the expression contains non-stencil terms.
+static std::optional<StencilDecomposition>
+decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
+ StencilDecomposition D;
+
+ // Collect top-level additive terms.
+ SmallVector<const SCEV *, 4> Terms;
+ if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
+ Terms.append(Add->operands().begin(), Add->operands().end());
+ else
+ Terms.push_back(Expr);
+
+ for (const SCEV *Term : Terms) {
+ if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ D.Constant += C->getAPInt().getSExtValue();
+ } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
+ // SCEV canonical form: constant is operand 0 in a MulExpr.
+ if (Mul->getNumOperands() != 2)
+ return std::nullopt;
+ auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
+ if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
+ return std::nullopt;
+ D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ } else if (SE.isLoopInvariant(Term, &L)) {
+ D.Coefficients[Term] += 1;
+ } else {
+ return std::nullopt;
+ }
+ }
+ return D;
+}
+
+void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
+ Loop &L) {
+ LLVM_DEBUG(dbgs() << "LAA: Attempting stencil group merging on "
+ << CheckingGroups.size() << " groups\n");
+
+ if (CheckingGroups.size() < 2)
+ return;
+
+ // Count the total checks from current grouping.
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks, proceeding with stencil merge\n");
+
+ // Group CheckingGroups by (DependencySetId, AliasSetId) pair.
+ // DependencySetId alone is not unique: it resets per alias set, so
+ // pointers in different alias sets can share the same DependencySetId.
+ // Use MapVector for deterministic iteration order across platforms.
+ using DepAliasKey = std::pair<unsigned, unsigned>;
+ MapVector<DepAliasKey, SmallVector<unsigned, 4>> DepSetToGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ const auto &P = Pointers[CheckingGroups[I].Members[0]];
+ DepSetToGroups[{P.DependencySetId, P.AliasSetId}].push_back(I);
+ }
+
+ SmallDenseSet<unsigned, 4> MergedGroupIndices;
+ SmallVector<RuntimeCheckingPtrGroup, 2> NewMergedGroups;
+ // Track strides that already have committed predicates (across all DepSets).
+ SmallDenseSet<const SCEV *, 4> CommittedStridePredicates;
+
+ for (auto &[DepAliasKey, GroupIndices] : DepSetToGroups) {
+ [[maybe_unused]] auto [DepId, ASId] = DepAliasKey;
+ if (GroupIndices.size() < 2)
+ continue;
+
+ // Collect all member pointers across these groups.
+ SmallVector<unsigned, 8> AllMembers;
+ for (unsigned GI : GroupIndices)
+ AllMembers.append(CheckingGroups[GI].Members.begin(),
+ CheckingGroups[GI].Members.end());
+
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ bool HasPredicatedAccess = false;
+ for (unsigned Idx : AllMembers) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ if (auto *I = dyn_cast<Instruction>(PtrVal)) {
+ if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT())) {
+ HasPredicatedAccess = true;
+ break;
+ }
+ }
+ }
+ if (HasPredicatedAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
+ // Only merge read-only groups. Stencil patterns read an array at
+ // multiple offsets and write to a different array (different DepSet).
+ // Mixing reads and writes within a merged group complicates the cost
+ // model and doesn't match known stencil patterns.
+ bool HasWriteAccess = false;
+ for (unsigned Idx : AllMembers) {
+ if (Pointers[Idx].IsWritePtr) {
+ HasWriteAccess = true;
+ break;
+ }
+ }
+ if (HasWriteAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with write access\n");
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
+ << ") with " << AllMembers.size() << " members, base: "
+ << *Pointers[AllMembers[0]].Start << "\n");
+
+ // Use the first member as the reference for decomposition. All offsets
+ // are computed relative to BaseLow, and MergedHigh is built from
+ // BaseHigh (see merged-bounds computation below).
+ const SCEV *BaseLow = Pointers[AllMembers[0]].Start;
+ const SCEV *BaseHigh = Pointers[AllMembers[0]].End;
+
+ const SCEV *BaseStep = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[AllMembers[0]].Expr))
+ if (AR->getLoop() == &L)
+ BaseStep = AR->getStepRecurrence(*SE);
+
+ if (!BaseStep)
+ continue;
+
+ // Verify all members have the same access range (End - Start).
+ // MergedHigh = BaseHigh + max_offsets is only correct when every
+ // member's range equals BaseHigh - BaseLow. We check by subtracting
+ // ranges and testing for zero, which lets SCEV simplify algebraically
+ // even when the individual range SCEVs aren't pointer-identical.
+ const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ bool RangeMismatch = false;
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different access range, skipping DepSet\n");
+ RangeMismatch = true;
+ break;
+ }
+ }
+ if (RangeMismatch)
+ continue;
+
+ bool DecompOk = true;
+
+ // Verify all members have the same recurrence step w.r.t. the analyzed
+ // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
+ // correct when every member's High-Low range equals BaseHigh - BaseLow.
+ // Different steps (e.g., 8 vs 16) produce different ranges.
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+
+ if (Step != BaseStep) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different step, skipping DepSet\n");
+ DecompOk = false;
+ break;
+ }
+ }
+ if (!DecompOk)
+ continue;
+ SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
+ int64_t CMin = 0, CMax = 0;
+ SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+
+ for (unsigned Idx : AllMembers) {
+ const SCEV *PtrStart = Pointers[Idx].Start;
+
+ const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset)) {
+ DecompOk = false;
+ break;
+ }
+ auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
+ if (!DLow) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " NOT decomposable: " << *LowOffset << "\n");
+ DecompOk = false;
+ break;
+ }
+
+ CMin = std::min(CMin, DLow->Constant);
+ CMax = std::max(CMax, DLow->Constant);
+
+ for (const auto &[Stride, Coeff] : DLow->Coefficients) {
+ auto MinIt = MinCoeff.find(Stride);
+ if (MinIt == MinCoeff.end()) {
+ MinCoeff[Stride] = Coeff;
+ MaxCoeff[Stride] = Coeff;
+ } else {
+ MinIt->second = std::min(MinIt->second, Coeff);
+ MaxCoeff[Stride] = std::max(MaxCoeff[Stride], Coeff);
+ }
+
+ if (!SE->isKnownPositive(Stride))
+ LocalStridesNeedingPreds.insert(Stride);
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
+ << ", strides=" << DLow->Coefficients.size() << "\n");
+ }
+
+ if (!DecompOk)
+ continue;
+
+ // The base member (offset = 0) implicitly has coefficient 0 for every
+ // stride. Members that lack a particular stride term also have an
+ // implicit coefficient of 0 for that stride. Clamp so that the
+ // bounding-box always includes coefficient 0, which is guaranteed to
+ // exist (at least from the base member).
+ for (auto &[Stride, MinC] : MinCoeff)
+ MinC = std::min(MinC, (int64_t)0);
+ for (auto &[Stride, MaxC] : MaxCoeff)
+ MaxC = std::max(MaxC, (int64_t)0);
+
+ // MergedLow = BaseLow + CMin + sum(MinCoeff[s] * s)
+ const SCEV *MergedLow = BaseLow;
+ if (CMin != 0)
+ MergedLow =
+ SE->getAddExpr(MergedLow, SE->getConstant(BaseLow->getType(), CMin,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MinC] : MinCoeff) {
+ if (MinC != 0)
+ MergedLow = SE->getAddExpr(
+ MergedLow, SE->getMulExpr(SE->getConstant(Stride->getType(), MinC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ // MergedHigh = BaseHigh + CMax + sum(MaxCoeff[s] * s)
+ // BaseHigh = BaseLow + Range, so this gives max(Low_j) + Range.
+ const SCEV *MergedHigh = BaseHigh;
+ if (CMax != 0)
+ MergedHigh =
+ SE->getAddExpr(MergedHigh, SE->getConstant(BaseHigh->getType(), CMax,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MaxC] : MaxCoeff) {
+ if (MaxC != 0)
+ MergedHigh = SE->getAddExpr(
+ MergedHigh, SE->getMulExpr(SE->getConstant(Stride->getType(), MaxC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
+ << ", High=" << *MergedHigh << "\n");
+
+ // Local cost model.
+ // G = number of groups in this DepSet, C = number of external groups
+ // that need checking against any member of this DepSet.
+ // Before merge: G * C checks. After merge: C + predicates.
+ unsigned G = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned C = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ // Check if this external group needs checking against any member
+ // of our DepSet.
+ for (unsigned GI : GroupIndices) {
+ if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++C;
+ break;
+ }
+ }
+ }
+
+ // Only count predicates we haven't already committed as cost.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ unsigned ChecksBefore = G * C;
+ unsigned ChecksAfter = C + NewPredicates;
+
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: G=" << G << ", C=" << C
+ << ", predicates=" << NewPredicates << ", checks "
+ << ChecksBefore << "->" << ChecksAfter);
+
+ if (ChecksAfter >= ChecksBefore) {
+ LLVM_DEBUG(dbgs() << " (skipping, not beneficial)\n");
+ continue;
+ }
+ LLVM_DEBUG(dbgs() << " (merging, net saving " << ChecksBefore - ChecksAfter
+ << ")\n");
+
+ // Build the merged group (after deciding to merge).
+ RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
+ CandidateGroup.Low = MergedLow;
+ CandidateGroup.High = MergedHigh;
+ for (unsigned I = 1; I < AllMembers.size(); ++I)
+ CandidateGroup.Members.push_back(AllMembers[I]);
+ for (unsigned GI : GroupIndices)
+ CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
+
+ // COMMIT: add stride predicates (skip already-committed ones).
+ for (const SCEV *Stride : LocalStridesNeedingPreds) {
+ if (!CommittedStridePredicates.insert(Stride).second)
+ continue;
+ const SCEV *Zero = SE->getZero(Stride->getType());
+ PSE.addPredicate(
+ *SE->getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+
+ for (unsigned GI : GroupIndices)
+ MergedGroupIndices.insert(GI);
+ NewMergedGroups.push_back(std::move(CandidateGroup));
+ }
+
+ // Rebuild CheckingGroups if we merged anything.
+ if (!NewMergedGroups.empty()) {
+ SmallVector<RuntimeCheckingPtrGroup, 2> FinalGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ if (!MergedGroupIndices.contains(I))
+ FinalGroups.push_back(std::move(CheckingGroups[I]));
+ FinalGroups.append(std::make_move_iterator(NewMergedGroups.begin()),
+ std::make_move_iterator(NewMergedGroups.end()));
+ CheckingGroups = std::move(FinalGroups);
+
+ LLVM_DEBUG(dbgs() << "LAA: After stencil merging: " << CheckingGroups.size()
+ << " groups\n");
+ }
+}
+
bool RuntimePointerChecking::arePointersInSamePartition(
const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
unsigned PtrIdx2) {
@@ -1511,7 +1887,7 @@ bool AccessAnalysis::canCheckPtrAtRT(
}
if (MayNeedRTCheck && (CanDoRT || AllowPartial))
- RtCheck.generateChecks(DepCands);
+ RtCheck.generateChecks(DepCands, PSE, *TheLoop);
LLVM_DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
<< " pointer comparisons.\n");
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
new file mode 100644
index 0000000000000..fb6b116a5b300
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
@@ -0,0 +1,1406 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+;; Test 1: Basic stencil merge with one runtime stride.
+;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
+;; relative to base = %a + 8*iv. Store to %out.
+;; With merge: all 6 loads form 1 merged group with bounds spanning [-3*cdj, +3*cdj]
+;; relative to base, producing 1 runtime check and 1 stride predicate (cdj > 0).
+;; Without merge: 6 separate groups (each load alone), 6 checks, no predicates.
+define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_single_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_single_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 2: Two runtime strides with constant offsets.
+;; 5 loads from %vol at offsets: +8+cdj-cdk, -16-2*cdj, +24+3*cdj+2*cdk,
+;; -8+2*cdj-3*cdk, +32-4*cdj+cdk. Store to %out.
+;; With merge: 1 merged group with bounds spanning the bounding box over
+;; both cdj and cdk coefficients, 1 check, 2 stride predicates.
+;; Without merge: 5 separate groups, 5 checks, no predicates.
+define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64 %cdk) {
+; MERGE-LABEL: 'stencil_merge_two_strides'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-4 * %cdj) + (-3 * %cdk) + %vol) High: ((2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; MERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdk sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_two_strides'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (64 + (-4 * %cdj) + %cdk + %vol) High: ((8 * %n) + (-4 * %cdj) + %cdk + %vol))
+; NOMERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + (-3 * %cdk) + %vol) High: (-40 + (2 * %cdj) + (8 * %n) + (-3 * %cdk) + %vol))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (56 + (2 * %cdk) + (3 * %cdj) + %vol) High: (-8 + (2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; NOMERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %vol) High: (-48 + (8 * %n) + (-2 * %cdj) + %vol))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (40 + (-1 * %cdk) + %cdj + %vol) High: (-24 + (8 * %n) + (-1 * %cdk) + %cdj + %vol))
+; NOMERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %vol, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; +8 + cdj - cdk
+ %off0a = add nsw i64 8, %cdj
+ %negcdk = sub nsw i64 0, %cdk
+ %off0 = add nsw i64 %off0a, %negcdk
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %v0 = load double, ptr %p0, align 8
+
+ ; -16 - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %off1 = add nsw i64 -16, %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %v1 = load double, ptr %p1, align 8
+
+ ; +24 + 3*cdj + 2*cdk
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %pos2cdk = mul nsw i64 %cdk, 2
+ %off2a = add nsw i64 24, %pos3cdj
+ %off2 = add nsw i64 %off2a, %pos2cdk
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %v2 = load double, ptr %p2, align 8
+
+ ; -8 + 2*cdj - 3*cdk
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %neg3cdk = mul nsw i64 %cdk, -3
+ %off3a = add nsw i64 -8, %pos2cdj
+ %off3 = add nsw i64 %off3a, %neg3cdk
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %v3 = load double, ptr %p3, align 8
+
+ ; +32 - 4*cdj + cdk
+ %neg4cdj = mul nsw i64 %cdj, -4
+ %off4a = add nsw i64 32, %neg4cdj
+ %off4 = add nsw i64 %off4a, %cdk
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %v4 = load double, ptr %p4, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s3, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 3: Constant offsets only — existing grouping handles this.
+;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
+;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
+;; Both with and without flag: 1 check, 2 groups, no predicates.
+define void @constant_offsets_only(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'constant_offsets_only'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'constant_offsets_only'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s2, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 4: Cost model rejection — only 2 loads with runtime stride.
+;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
+;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
+define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'cost_model_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'cost_model_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %v1 = load double, ptr %base, align 8
+
+ %s = fadd double %v0, %v1
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 5: Invariant + strided accesses to same base -> different steps.
+;; A strided store {%a,+,8} and an invariant store to %a have different
+;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Same result with and without flag: 2 checks, 3 separate groups.
+define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'different_steps_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Unknown:
+; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; MERGE-EMPTY:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %a
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'different_steps_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Unknown:
+; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %a
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; GRP0: invariant store to %a (single address, step=0):
+; GRP1: strided store to %a (step=8, different range from GRP0):
+; GRP2: load from %out:
+entry:
+ %cmp = icmp sgt i64 %n, 2
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+
+ ; Strided store: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ store double 1.0, ptr %gep.a, align 8
+
+ ; Invariant store to %a (step = 0, different range)
+ store double 2.0, ptr %a, align 8
+
+ ; Read from %out
+ %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+ %v = load double, ptr %gep.out, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 6: Coefficient clamping in merged bounds.
+;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. Members that lack a stride
+;; term have an implicit coefficient of 0, which must be included in the
+;; min/max computation via clamping.
+define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'coefficient_clamping_regression'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'coefficient_clamping_regression'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base + 0 (no stride offset -- this is the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; load at base - cdj
+ %negcdj = sub nsw i64 0, %cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; load at base - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 7: Predicated accesses are rejected from stencil merging.
+;; Models the dilateKernel pattern (llvm-test-suite MicroBenchmarks/ImageProcessing/Dilate):
+;; 3 loads at {-cdj, 0, +cdj} where the -cdj and +cdj loads are conditional.
+;; Predicated loads have overapproximated SCEV bounds; merging would widen
+;; them further, causing false runtime overlap detection.
+;; Both modes: 3 checks (groups stay separate), no predicates.
+define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c1, i1 %c2) {
+; MERGE-LABEL: 'predicated_access_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 3 checks: each temp group separately vs %out (not merged due to predication):
+; 4 groups: 1 for %out, 3 separate temp groups:
+entry:
+ %cmp = icmp sgt i64 %n, 0
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Unconditional load at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Conditional load at base + cdj (predicated block)
+ br i1 %c1, label %if.then1, label %if.end1
+
+if.then1:
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+ br label %if.end1
+
+if.end1:
+ %m1 = phi double [ %v1, %if.then1 ], [ %v0, %loop ]
+
+ ; Conditional load at base - cdj (predicated block)
+ %negcdj = sub nsw i64 0, %cdj
+ br i1 %c2, label %if.then2, label %if.end2
+
+if.then2:
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+ br label %if.end2
+
+if.end2:
+ %m2 = phi double [ %v2, %if.then2 ], [ %m1, %if.end1 ]
+
+ %s = fadd double %m1, %m2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 8: Write pointer in a group prevents stencil merging.
+;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
+;; All three share the same base %a (same DepSet). The write prevents merging.
+;; Both modes: groups stay separate.
+define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'write_in_group_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'write_in_group_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 5 checks: write group vs all others, %out vs the two read groups:
+; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
+; Both modes produce identical output (write prevents merging).
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Read at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Read at base + cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; Write at base + 2*cdj (same DepSet as reads — prevents merging)
+ %cdj2 = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+ %s = fadd double %v0, %v1
+ store double %s, ptr %p2, align 8
+
+ ; Store to %out (different DepSet)
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 9: Mixed constant offsets + runtime stride.
+;; 4 constant-offset loads at {-16, -8, +8, +16} + 2 runtime-stride loads
+;; at {-cdj, +cdj}. Store to %out.
+;; Without merge: standard grouping merges the 4 constant-offset loads into
+;; 1 group but the 2 runtime-stride loads remain separate -> 3 checks.
+;; With merge: all 6 loads form 1 merged group -> 1 check + 1 predicate.
+;; Merged Low includes -cdj, Merged High includes +cdj:
+define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; -16 (constant)
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ ; -8 (constant)
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ ; +8 (constant)
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ ; +16 (constant)
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ ; -cdj (runtime)
+ %negcdj = sub nsw i64 0, %cdj
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v4 = load double, ptr %p4, align 8
+
+ ; +cdj (runtime)
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 10: Shared stride predicate deduplication across two DepSets.
+;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
+;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
+;; should be emitted, not 2.
+;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 1 predicate.
+;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
+define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'shared_stride_predicate_dedup'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; MERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'shared_stride_predicate_dedup'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + %a) High: (-24 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + %b) High: (-24 + (8 * %n) + %b))
+; NOMERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + (-1 * %cdj) + %b))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+
+ ; Reads from %a at {-cdj, 0, +cdj}
+ %basea = getelementptr inbounds double, ptr %a, i64 %iv
+ %basea.i8 = bitcast ptr %basea to ptr
+ %negcdj = sub nsw i64 0, %cdj
+ %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %va0 = load double, ptr %pa0, align 8
+ %va1 = load double, ptr %basea, align 8
+ %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %va2 = load double, ptr %pa2, align 8
+
+ ; Reads from %b at {-cdj, 0, +cdj}
+ %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+ %baseb.i8 = bitcast ptr %baseb to ptr
+ %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %vb0 = load double, ptr %pb0, align 8
+ %vb1 = load double, ptr %baseb, align 8
+ %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %vb2 = load double, ptr %pb2, align 8
+
+ %s0 = fadd double %va0, %va1
+ %s1 = fadd double %s0, %va2
+ %s2 = fadd double %s1, %vb0
+ %s3 = fadd double %s2, %vb1
+ %s4 = fadd double %s3, %vb2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 11: Known-positive stride skips predicate emission.
+;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
+;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
+;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
+;; Without merge: 3 separate groups + 1 %out, 3 checks, no predicates.
+define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %cdj_in) {
+; MERGE-LABEL: 'known_positive_stride_no_predicate'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; MERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'known_positive_stride_no_predicate'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (1 smax %cdj_in) + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; NOMERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (-1 * (1 smax %cdj_in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base - stride
+ %negcdj = sub nsw i64 0, %cdj
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v0 = load double, ptr %p0, align 8
+
+ ; load at base
+ %v1 = load double, ptr %base, align 8
+
+ ; load at base + stride
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
>From 08a209dc44fa351fb9f24a3a8c6166b0ff6f809f Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 15:13:42 +0000
Subject: [PATCH 02/14] Use determenistic structure Rename test file
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 4 ++--
...affine-group-merging.ll => runtime-check-group-merging.ll} | 0
2 files changed, 2 insertions(+), 2 deletions(-)
rename llvm/test/Analysis/LoopAccessAnalysis/{affine-group-merging.ll => runtime-check-group-merging.ll} (100%)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 5015be674fe18..6b37c8e076321 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -755,7 +755,7 @@ void RuntimePointerChecking::groupChecks(
struct StencilDecomposition {
int64_t Constant = 0;
/// Map from loop-invariant stride SCEV to its integer coefficient.
- SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+ SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
};
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
@@ -943,7 +943,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
- SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+ SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
for (unsigned Idx : AllMembers) {
const SCEV *PtrStart = Pointers[Idx].Start;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
similarity index 100%
rename from llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
>From 2a14a3a7f7a0a39b6e79b996a628d0b24cc5891c Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 09:22:37 +0100
Subject: [PATCH 03/14] Address some comments (nit related and about the flag)
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 193 ++++++++----------
.../runtime-check-group-merging.ll | 174 ++++++++--------
2 files changed, 175 insertions(+), 192 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 6b37c8e076321..8d320a04d8f0b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,15 +100,15 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable merging of runtime check groups with stencil stride "
- "patterns (default = false)"),
+static cl::opt<bool> ForceStencilMerge(
+ "force-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Force merging of runtime check groups with stencil stride "
+ "patterns regardless of check count (default = false)"),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
- cl::desc("Trigger stencil group merging when runtime check count "
+ cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128)"),
cl::init(128));
@@ -760,8 +760,8 @@ struct StencilDecomposition {
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
-/// Relies on SCEV's canonical form: AddExpr operands are flattened,
-/// MulExpr has the constant operand first.
+/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
+/// MulExpr has the constant operand first when present.
/// Returns std::nullopt if the expression contains non-stencil terms.
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
@@ -770,21 +770,20 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
- Terms.append(Add->operands().begin(), Add->operands().end());
+ append_range(Terms, Add->operands());
else
Terms.push_back(Expr);
+ const SCEVConstant *C;
+ const SCEV *Stride;
for (const SCEV *Term : Terms) {
- if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ if (match(Term, m_SCEVConstant(C))) {
D.Constant += C->getAPInt().getSExtValue();
- } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
- // SCEV canonical form: constant is operand 0 in a MulExpr.
- if (Mul->getNumOperands() != 2)
+ } else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
+ // Canonical 2-operand pattern (constant * loop-invariant).
+ if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
- if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
- return std::nullopt;
- D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ D.Coefficients[Stride] += C->getAPInt().getSExtValue();
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
@@ -802,20 +801,30 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
- // Count the total checks from current grouping.
- unsigned TotalChecks = 0;
- for (unsigned I = 0; I < CheckingGroups.size(); ++I)
- for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
- if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
- ++TotalChecks;
-
- if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks <= threshold, skipping stencil merge\n");
- return;
+ // Stencil merging runs when either:
+ // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
+ // - the current check count exceeds the auto-trigger threshold, where the
+ // vectorizer would otherwise reject the loop for having too many runtime
+ // checks. In that case the merge can only improve things: at worst we
+ // decline to merge and behave as before.
+ if (!ForceStencilMerge) {
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(
+ dbgs() << "LAA: " << TotalChecks
+ << " checks > threshold, proceeding with stencil merge\n");
+ } else {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge forced via flag\n");
}
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks, proceeding with stencil merge\n");
// Group CheckingGroups by (DependencySetId, AliasSetId) pair.
// DependencySetId alone is not unique: it resets per alias set, so
@@ -841,47 +850,34 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Collect all member pointers across these groups.
SmallVector<unsigned, 8> AllMembers;
for (unsigned GI : GroupIndices)
- AllMembers.append(CheckingGroups[GI].Members.begin(),
- CheckingGroups[GI].Members.end());
-
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
- bool HasPredicatedAccess = false;
- for (unsigned Idx : AllMembers) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- if (auto *I = dyn_cast<Instruction>(PtrVal)) {
- if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT())) {
- HasPredicatedAccess = true;
- break;
- }
- }
- }
- if (HasPredicatedAccess) {
- LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
- << ") with predicated access\n");
- continue;
- }
+ append_range(AllMembers, CheckingGroups[GI].Members);
// Only merge read-only groups. Stencil patterns read an array at
// multiple offsets and write to a different array (different DepSet).
// Mixing reads and writes within a merged group complicates the cost
// model and doesn't match known stencil patterns.
- bool HasWriteAccess = false;
- for (unsigned Idx : AllMembers) {
- if (Pointers[Idx].IsWritePtr) {
- HasWriteAccess = true;
- break;
- }
- }
- if (HasWriteAccess) {
+ if (any_of(AllMembers,
+ [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with write access\n");
continue;
}
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ auto *I = dyn_cast<Instruction>(PtrVal);
+ return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
<< ") with " << AllMembers.size() << " members, base: "
<< *Pointers[AllMembers[0]].Start << "\n");
@@ -906,59 +902,49 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// ranges and testing for zero, which lets SCEV simplify algebraically
// even when the individual range SCEVs aren't pointer-identical.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
- bool RangeMismatch = false;
- for (unsigned Idx : AllMembers) {
- const SCEV *Range =
- SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different access range, skipping DepSet\n");
- RangeMismatch = true;
- break;
- }
- }
- if (RangeMismatch)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ return Range != BaseRange &&
+ !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
+ "skipping DepSet\n");
continue;
-
- bool DecompOk = true;
+ }
// Verify all members have the same recurrence step w.r.t. the analyzed
// loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
// correct when every member's High-Low range equals BaseHigh - BaseLow.
// Different steps (e.g., 8 vs 16) produce different ranges.
- for (unsigned Idx : AllMembers) {
- const SCEV *Step = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
- if (AR->getLoop() == &L)
- Step = AR->getStepRecurrence(*SE);
-
- if (Step != BaseStep) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different step, skipping DepSet\n");
- DecompOk = false;
- break;
- }
- }
- if (!DecompOk)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+ return Step != BaseStep;
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
+ "skipping DepSet\n");
continue;
+ }
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
- for (unsigned Idx : AllMembers) {
- const SCEV *PtrStart = Pointers[Idx].Start;
-
- const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
- if (isa<SCEVCouldNotCompute>(LowOffset)) {
- DecompOk = false;
- break;
- }
+ // Decompose one member's offset (relative to BaseLow) and fold its
+ // constant and per-stride coefficients into the running bounding box.
+ // Returns false if the offset is not in stencil form (so the whole
+ // DepSet is skipped).
+ const auto AccumulateOffset = [&](unsigned Idx) -> bool {
+ const SCEV *LowOffset = SE->getMinusSCEV(Pointers[Idx].Start, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset))
+ return false;
auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
if (!DLow) {
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< " NOT decomposable: " << *LowOffset << "\n");
- DecompOk = false;
- break;
+ return false;
}
CMin = std::min(CMin, DLow->Constant);
@@ -980,9 +966,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
- }
+ return true;
+ };
- if (!DecompOk)
+ if (!all_of(AllMembers, AccumulateOffset))
continue;
// The base member (offset = 0) implicitly has coefficient 0 for every
@@ -1074,8 +1061,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
- for (unsigned I = 1; I < AllMembers.size(); ++I)
- CandidateGroup.Members.push_back(AllMembers[I]);
+ append_range(CandidateGroup.Members, drop_begin(AllMembers));
for (unsigned GI : GroupIndices)
CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
@@ -1090,8 +1076,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< *Stride << "\n");
}
- for (unsigned GI : GroupIndices)
- MergedGroupIndices.insert(GI);
+ MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
NewMergedGroups.push_back(std::move(CandidateGroup));
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index fb6b116a5b300..c6b9507d84752 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -18,12 +18,12 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -52,32 +52,32 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 5:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -105,6 +105,7 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -112,29 +113,28 @@ entry:
loop:
%iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
%neg3cdj = mul nsw i64 %cdj, -3
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
%v0 = load double, ptr %p0, align 8
%neg2cdj = mul nsw i64 %cdj, -2
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v1 = load double, ptr %p1, align 8
%negcdj = sub nsw i64 0, %cdj
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v2 = load double, ptr %p2, align 8
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v3 = load double, ptr %p3, align 8
%pos2cdj = mul nsw i64 %cdj, 2
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
%v4 = load double, ptr %p4, align 8
%pos3cdj = mul nsw i64 %cdj, 3
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -172,11 +172,11 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -205,27 +205,27 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -250,6 +250,7 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 8
br i1 %cmp, label %loop, label %exit
@@ -257,19 +258,18 @@ entry:
loop:
%iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %vol, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; +8 + cdj - cdk
%off0a = add nsw i64 8, %cdj
%negcdk = sub nsw i64 0, %cdk
%off0 = add nsw i64 %off0a, %negcdk
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
%v0 = load double, ptr %p0, align 8
; -16 - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
%off1 = add nsw i64 -16, %neg2cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
%v1 = load double, ptr %p1, align 8
; +24 + 3*cdj + 2*cdk
@@ -277,7 +277,7 @@ loop:
%pos2cdk = mul nsw i64 %cdk, 2
%off2a = add nsw i64 24, %pos3cdj
%off2 = add nsw i64 %off2a, %pos2cdk
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
%v2 = load double, ptr %p2, align 8
; -8 + 2*cdj - 3*cdk
@@ -285,14 +285,14 @@ loop:
%neg3cdk = mul nsw i64 %cdk, -3
%off3a = add nsw i64 -8, %pos2cdj
%off3 = add nsw i64 %off3a, %neg3cdk
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
%v3 = load double, ptr %p3, align 8
; +32 - 4*cdj + cdk
%neg4cdj = mul nsw i64 %cdj, -4
%off4a = add nsw i64 32, %neg4cdj
%off4 = add nsw i64 %off4a, %cdk
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
%v4 = load double, ptr %p4, align 8
%s0 = fadd double %v0, %v1
@@ -431,7 +431,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -462,7 +462,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -486,9 +486,8 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v0 = load double, ptr %p0, align 8
%v1 = load double, ptr %base, align 8
@@ -629,8 +628,8 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
@@ -657,12 +656,12 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -686,6 +685,7 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -693,19 +693,18 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base + 0 (no stride offset -- this is the base member)
%v0 = load double, ptr %base, align 8
; load at base - cdj
%negcdj = sub nsw i64 0, %cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v1 = load double, ptr %p1, align 8
; load at base - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
@@ -1019,12 +1018,12 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1053,20 +1052,20 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1088,6 +1087,7 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -1095,31 +1095,30 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; -16 (constant)
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
%v0 = load double, ptr %p0, align 8
; -8 (constant)
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
%v1 = load double, ptr %p1, align 8
; +8 (constant)
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
%v2 = load double, ptr %p2, align 8
; +16 (constant)
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
%v3 = load double, ptr %p3, align 8
; -cdj (runtime)
%negcdj = sub nsw i64 0, %cdj
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v4 = load double, ptr %p4, align 8
; +cdj (runtime)
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -1157,16 +1156,16 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; MERGE-NEXT: Check 1:
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
-; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1197,7 +1196,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1207,12 +1206,12 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1222,7 +1221,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1250,6 +1249,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -1259,21 +1259,19 @@ loop:
; Reads from %a at {-cdj, 0, +cdj}
%basea = getelementptr inbounds double, ptr %a, i64 %iv
- %basea.i8 = bitcast ptr %basea to ptr
%negcdj = sub nsw i64 0, %cdj
- %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
%va0 = load double, ptr %pa0, align 8
%va1 = load double, ptr %basea, align 8
- %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
%va2 = load double, ptr %pa2, align 8
; Reads from %b at {-cdj, 0, +cdj}
%baseb = getelementptr inbounds double, ptr %b, i64 %iv
- %baseb.i8 = bitcast ptr %baseb to ptr
- %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
%vb0 = load double, ptr %pb0, align 8
%vb1 = load double, ptr %baseb, align 8
- %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
%vb2 = load double, ptr %pb2, align 8
%s0 = fadd double %va0, %va1
@@ -1310,9 +1308,9 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1337,7 +1335,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1347,7 +1345,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1366,6 +1364,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
%cmp = icmp sgt i64 %n, 4
@@ -1374,18 +1373,17 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base - stride
%negcdj = sub nsw i64 0, %cdj
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v0 = load double, ptr %p0, align 8
; load at base
%v1 = load double, ptr %base, align 8
; load at base + stride
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
>From f3846b6d15ece810509803dce79776a28a866f16 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 13:36:21 +0100
Subject: [PATCH 04/14] Address int64_t comment
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 8d320a04d8f0b..737da16c8daa2 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -762,11 +762,20 @@ struct StencilDecomposition {
/// strides: C + a1*s1 + a2*s2 + ...
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
-/// Returns std::nullopt if the expression contains non-stencil terms.
+/// Returns std::nullopt if the expression contains non-stencil terms or any
+/// SCEV constant doesn't fit in int64_t (we commit to the signed
+/// interpretation; values that need more than 64 significant bits are
+/// out of scope).
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
+ if (V.getSignificantBits() > 64)
+ return std::nullopt;
+ return V.getSExtValue();
+ };
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -778,12 +787,18 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- D.Constant += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- D.Coefficients[Stride] += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Coefficients[Stride] += *V;
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
>From f9a9fd6641ff23efdbd00f98794283a00f63d398 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:35:25 +0100
Subject: [PATCH 05/14] Add 128-addrspace test and replaced lambda with
existing trySExtValue
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 10 +-
.../runtime-check-group-merging-i128.ll | 117 ++++++++++++++++++
2 files changed, 119 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 737da16c8daa2..c1bdaf63e18fd 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -770,12 +770,6 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
- auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
- if (V.getSignificantBits() > 64)
- return std::nullopt;
- return V.getSExtValue();
- };
-
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -787,7 +781,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Constant += *V;
@@ -795,7 +789,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
new file mode 100644
index 0000000000000..dfb56233e91d5
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+target datalayout = "e-p:128:128"
+
+define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj) {
+; MERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %ivx = zext i64 %iv to i128
+
+ %o0 = mul i128 %cdj, 1
+ %t0 = add i128 %o0, %ivx
+ %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+ %v0 = load i8, ptr %p0
+
+ %o1 = mul i128 %cdj, 2
+ %t1 = add i128 %o1, %ivx
+ %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+ %v1 = load i8, ptr %p1
+
+ %o2 = mul i128 %cdj, 36893488147419103232 ; 2^65, needs 67 significant bits
+ %t2 = add i128 %o2, %ivx
+ %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+ %v2 = load i8, ptr %p2
+
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+ store i8 %s, ptr %op
+
+ %iv.next = add i64 %iv, 1
+ %c = icmp slt i64 %iv.next, %n
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
>From 89d1aa88310e9e5abe3e4b391e52dea260e55705 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:50:11 +0100
Subject: [PATCH 06/14] Add -enable-stencil-runtime-check-merge switch
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index c1bdaf63e18fd..221386f3cf0c4 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,16 +100,26 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable stencil-pattern merging of runtime memory checks "
+ "(default = false, intended to be flipped to true in a "
+ "follow-up patch once the algorithm has settled)"),
+ cl::init(false));
+
static cl::opt<bool> ForceStencilMerge(
"force-stencil-runtime-check-merge", cl::Hidden,
cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false)"),
+ "patterns regardless of check count (default = false). "
+ "Implies enabling the merge regardless of "
+ "-enable-stencil-runtime-check-merge."),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
- "exceeds this threshold (default = 128)"),
+ "exceeds this threshold (default = 128). Only used when "
+ "-enable-stencil-runtime-check-merge is true."),
cl::init(128));
/// Maximum SIMD width.
@@ -811,12 +821,17 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
- // - the current check count exceeds the auto-trigger threshold, where the
+ // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
+ // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
if (!ForceStencilMerge) {
+ if (!EnableStencilMerge) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
+ return;
+ }
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 06611660f908ce609646e91214c185f7b55658d4 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:20:36 +0000
Subject: [PATCH 07/14] Fix test
---
.../runtime-check-group-merging.ll | 79 +++++++++----------
1 file changed, 36 insertions(+), 43 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index c6b9507d84752..822be3812b3a6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -507,41 +507,38 @@ exit:
}
-;; Test 5: Invariant + strided accesses to same base -> different steps.
-;; A strided store {%a,+,8} and an invariant store to %a have different
-;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Test 5: Invariant + strided reads from same base -> different access ranges.
+;; A strided read {%a,+,8} and an invariant read from %a have different
+;; access ranges (8*n vs 8), so merging must NOT combine them.
+;; Both reads are in the same DepSet (same underlying object, both reads),
+;; so they pass the write-access guard and reach the access-range check.
;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; MERGE-LABEL: 'different_steps_no_merge'
; MERGE-NEXT: loop:
-; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Memory dependences are safe with run-time checks
; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Unknown:
-; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; MERGE-EMPTY:
; MERGE-NEXT: Run-time memory checks:
; MERGE-NEXT: Check 0:
; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: ptr %a
-; MERGE-NEXT: Against group GRP2:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: Against group GRP1:
; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %a
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: %a
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
@@ -550,43 +547,38 @@ define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
;
; NOMERGE-LABEL: 'different_steps_no_merge'
; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Unknown:
-; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; NOMERGE-EMPTY:
; NOMERGE-NEXT: Run-time memory checks:
; NOMERGE-NEXT: Check 0:
; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: ptr %a
-; NOMERGE-NEXT: Against group GRP2:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: Against group GRP1:
; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %a
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
;
-; GRP0: invariant store to %a (single address, step=0):
-; GRP1: strided store to %a (step=8, different range from GRP0):
-; GRP2: load from %out:
+; GRP0: store to %out (write, different DepSet):
+; GRP1: strided read from %a (step=8):
+; GRP2: invariant read from %a (step=0, different from GRP1):
entry:
%cmp = icmp sgt i64 %n, 2
br i1 %cmp, label %loop, label %exit
@@ -594,16 +586,17 @@ entry:
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- ; Strided store: {%a,+,8}
- %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
- store double 1.0, ptr %gep.a, align 8
+ ; Invariant read from %a (step = 0, different from strided)
+ %v.inv = load double, ptr %a, align 8
- ; Invariant store to %a (step = 0, different range)
- store double 2.0, ptr %a, align 8
+ ; Strided read: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ %v.strided = load double, ptr %gep.a, align 8
- ; Read from %out
+ ; Store to %out (different DepSet, triggers runtime checks)
+ %sum = fadd double %v.strided, %v.inv
%gep.out = getelementptr inbounds double, ptr %out, i64 %iv
- %v = load double, ptr %gep.out, align 8
+ store double %sum, ptr %gep.out, align 8
%iv.next = add nuw nsw i64 %iv, 1
%cond = icmp slt i64 %iv.next, %n
>From c1bccb5f7970b9e23d08198b459537dd9a4c9895 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:40:06 +0000
Subject: [PATCH 08/14] Refactor flags
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 46 ++++++++++++------------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 221386f3cf0c4..3f9bdf4af3c67 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,26 +100,27 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable stencil-pattern merging of runtime memory checks "
- "(default = false, intended to be flipped to true in a "
- "follow-up patch once the algorithm has settled)"),
- cl::init(false));
-
-static cl::opt<bool> ForceStencilMerge(
- "force-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false). "
- "Implies enabling the merge regardless of "
- "-enable-stencil-runtime-check-merge."),
- cl::init(false));
+enum class StencilMergePolicy { Off, Auto, Force };
+
+static cl::opt<StencilMergePolicy> StencilMerge(
+ "stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Control stencil-pattern merging of runtime memory checks"),
+ cl::init(StencilMergePolicy::Off),
+ cl::values(
+ clEnumValN(StencilMergePolicy::Off, "off",
+ "Disable stencil merge (default)"),
+ clEnumValN(StencilMergePolicy::Auto, "auto",
+ "Enable stencil merge when runtime check count exceeds "
+ "the threshold"),
+ clEnumValN(StencilMergePolicy::Force, "force",
+ "Always attempt stencil merge regardless of check "
+ "count")));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128). Only used when "
- "-enable-stencil-runtime-check-merge is true."),
+ "-stencil-runtime-check-merge is auto."),
cl::init(128));
/// Maximum SIMD width.
@@ -821,17 +822,18 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
- // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
+ // - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
// current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
- if (!ForceStencilMerge) {
- if (!EnableStencilMerge) {
- LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
- return;
- }
+ if (StencilMerge == StencilMergePolicy::Off) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled\n");
+ return;
+ }
+
+ if (StencilMerge == StencilMergePolicy::Auto) {
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 25ee2843cced31188415bbcbee786f239700ef40 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 3 Jun 2026 15:29:11 +0100
Subject: [PATCH 09/14] Fix predicated-access check, turn some conditions into
asserts, add tests
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 52 ++-
.../runtime-check-group-merging-auto.ll | 150 +++++++++
.../runtime-check-group-merging-i128.ll | 20 +-
.../runtime-check-group-merging.ll | 314 +++++++++++++++++-
4 files changed, 506 insertions(+), 30 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 3f9bdf4af3c67..3605908d6c286 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -781,6 +781,11 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ // The only caller passes a difference of two pointer Starts, and a Start is
+ // always loop-invariant (getStartAndEndForAccess asserts it). So Expr is
+ // loop-invariant and every term below is loop-invariant too.
+ assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -797,17 +802,14 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return std::nullopt;
D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
- // Canonical 2-operand pattern (constant * loop-invariant).
- if (!SE.isLoopInvariant(Stride, &L))
- return std::nullopt;
+ assert(SE.isLoopInvariant(Stride, &L) && "stride must be loop-invariant");
auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
- } else if (SE.isLoopInvariant(Term, &L)) {
- D.Coefficients[Term] += 1;
} else {
- return std::nullopt;
+ assert(SE.isLoopInvariant(Term, &L) && "term must be loop-invariant");
+ D.Coefficients[Term] += 1;
}
}
return D;
@@ -889,15 +891,26 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
+ // We do not allow predicated accesses. They may result in overestimation
+ // of the boundaries. Imagine a stencil access where we must skip some first
+ // or last iterations because the stencil does not fit the array and has to
+ // go from 1..N-2 although the array is [0..N-1] (for example the dilate
+ // kernel from llvm-test-suite ImageProcessing/Dilate, which reads the
+ // neighbours of every pixel and guards the borders with conditions).
+ // Merging such bounds would widen the already overestimated range further.
+ // This is not necessary in StencilMergePolicy::Auto mode, but skipping it
+ // in StencilMergePolicy::Force mode causes a regression on that benchmark.
+ //
+ // Look at the block of the actual load/store, not of the pointer: a
+ // loop-invariant address is computed in the preheader, outside the loop.
if (any_of(AllMembers, [&](unsigned Idx) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- auto *I = dyn_cast<Instruction>(PtrVal);
- return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT());
+ const PointerInfo &P = Pointers[Idx];
+ return any_of(
+ DC.getInstructionsForAccess(P.PointerValue, P.IsWritePtr),
+ [&](Instruction *I) {
+ return LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ });
})) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with predicated access\n");
@@ -939,10 +952,13 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Verify all members have the same recurrence step w.r.t. the analyzed
- // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
- // correct when every member's High-Low range equals BaseHigh - BaseLow.
- // Different steps (e.g., 8 vs 16) produce different ranges.
+ // Require all members to have the same recurrence step. Equal ranges
+ // (checked above) are what the merged bounds actually need, and a different
+ // step usually means a different range. But ranges can be equal by accident
+ // - e.g. an invariant access whose range matches the stride, or a loop with
+ // a single iteration. The base member is picked arbitrarily, so together
+ // with the BaseStep check above this keeps the decision the same no matter
+ // which member comes first: we only merge recurrences with one common step.
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Step = nullptr;
if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
new file mode 100644
index 0000000000000..6855e80d4cffa
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1000 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
+
+define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; AUTOMERGE-LABEL: 'stencil_auto_threshold'
+; AUTOMERGE-NEXT: loop:
+; AUTOMERGE-NEXT: Memory dependences are safe with run-time checks
+; AUTOMERGE-NEXT: Dependences:
+; AUTOMERGE-NEXT: Run-time memory checks:
+; AUTOMERGE-NEXT: Check 0:
+; AUTOMERGE-NEXT: Comparing group GRP0:
+; AUTOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOMERGE-NEXT: Against group GRP1:
+; AUTOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOMERGE-NEXT: Grouped accesses:
+; AUTOMERGE-NEXT: Group GRP0:
+; AUTOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOMERGE-NEXT: Group GRP1:
+; AUTOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOMERGE-NEXT: SCEV assumptions:
+; AUTOMERGE-NEXT: Compare predicate: %cdj sgt) 0
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Expressions re-written:
+;
+; AUTOSKIP-LABEL: 'stencil_auto_threshold'
+; AUTOSKIP-NEXT: loop:
+; AUTOSKIP-NEXT: Memory dependences are safe with run-time checks
+; AUTOSKIP-NEXT: Dependences:
+; AUTOSKIP-NEXT: Run-time memory checks:
+; AUTOSKIP-NEXT: Check 0:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP1:
+; AUTOSKIP-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOSKIP-NEXT: Check 1:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP2:
+; AUTOSKIP-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOSKIP-NEXT: Check 2:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP3:
+; AUTOSKIP-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOSKIP-NEXT: Check 3:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP4:
+; AUTOSKIP-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOSKIP-NEXT: Check 4:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP5:
+; AUTOSKIP-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOSKIP-NEXT: Check 5:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP6:
+; AUTOSKIP-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOSKIP-NEXT: Grouped accesses:
+; AUTOSKIP-NEXT: Group GRP0:
+; AUTOSKIP-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOSKIP-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOSKIP-NEXT: Group GRP1:
+; AUTOSKIP-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP2:
+; AUTOSKIP-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP3:
+; AUTOSKIP-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; AUTOSKIP-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP4:
+; AUTOSKIP-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP5:
+; AUTOSKIP-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP6:
+; AUTOSKIP-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOSKIP-NEXT: SCEV assumptions:
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
index dfb56233e91d5..0799360bf346e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
target datalayout = "e-p:128:128"
@@ -27,16 +27,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -66,16 +66,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 822be3812b3a6..3ef1bd18a3ab2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -1394,4 +1394,314 @@ exit:
ret void
}
+;; Test 12: loop-invariant pointer is computed in the preheader.
+;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
+;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
+;; read, so we reach the predicated-access check. We must look at the load,
+;; which is in the loop, not at the pointer, which is in the preheader -
+;; otherwise we crash.
+define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'invariant_pointer_in_preheader'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (64 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: (%cdj + %a)
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'invariant_pointer_in_preheader'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (64 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: (%cdj + %a)
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+ %sv = load i8, ptr %sp
+ %sv64 = zext i8 %sv to i64
+ %iv8 = load i64, ptr %inv.ptr
+ %sum = add i64 %sv64, %iv8
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+
+;; Test 13: predicated access whose address is computed in the header.
+;; The stride GEPs are in the header, which dominates the latch, but the loads
+;; are in a conditional block guarded by %c. So the accesses are predicated and
+;; must not be merged. We must look at the load block, which is conditional,
+;; not at the GEP block, which is the header. Both modes: groups stay separate.
+define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c) {
+; MERGE-LABEL: 'predicated_access_hoisted_address'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; MERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP4:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_hoisted_address'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+ %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+ %v0 = load i64, ptr %p0
+ %i1 = add i64 %iv, %cdj
+ %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+ %i2 = sub i64 %iv, %cdj
+ %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+ %i3 = add i64 %i1, %cdj
+ %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+ br i1 %c, label %then, label %backedge
+then:
+ %v1 = load i64, ptr %p1
+ %v2 = load i64, ptr %p2
+ %v3 = load i64, ptr %p3
+ %t = add i64 %v1, %v2
+ %u = add i64 %t, %v3
+ br label %backedge
+backedge:
+ %vp = phi i64 [ %u, %then ], [ 0, %loop ]
+ %sum = add i64 %v0, %vp
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+;; Test 14: equal access ranges but different recurrence steps -> no merge.
+;; In a single-iteration loop every access spans just its element size, so the
+;; ranges are equal even though the steps differ (8 vs 16). The access-range
+;; check passes here, so the step check is the guard that stops the merge. Both
+;; modes: groups stay separate.
+define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'equal_range_different_step_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (8 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'equal_range_different_step_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (8 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %s8 = mul i64 %iv, 8
+ %b0 = getelementptr inbounds i8, ptr %a, i64 %s8
+ %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+ %v0 = load i64, ptr %p0
+ %s16 = mul i64 %iv, 16
+ %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+ %v1 = load i64, ptr %p1
+ %sum = add i64 %v0, %v1
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 91a369dad51a872b3a634162f3dd06358a37dd99 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 10:41:15 +0000
Subject: [PATCH 10/14] Add SCEVCouldNotCompute test
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 26 ++++--
.../runtime-check-group-merging.ll | 85 +++++++++++++++++++
2 files changed, 103 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 3605908d6c286..f482e88276323 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -936,19 +936,29 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
// Verify all members have the same access range (End - Start).
- // MergedHigh = BaseHigh + max_offsets is only correct when every
- // member's range equals BaseHigh - BaseLow. We check by subtracting
- // ranges and testing for zero, which lets SCEV simplify algebraically
- // even when the individual range SCEVs aren't pointer-identical.
+ // MergedHigh = BaseHigh + max_offsets is only correct when every member's
+ // range equals BaseHigh - BaseLow. Compute the ranges first and compare
+ // them by subtracting, so algebraically equal but non-identical SCEVs still
+ // match. Bail out if any subtraction produces SCEVCouldNotCompute.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ if (isa<SCEVCouldNotCompute>(BaseRange)) {
+ LLVM_DEBUG(dbgs() << "LAA: Base access range not computable, "
+ "skipping DepSet\n");
+ continue;
+ }
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Range =
SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- return Range != BaseRange &&
- !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ if (isa<SCEVCouldNotCompute>(Range))
+ return true;
+ if (Range == BaseRange)
+ return false;
+ const SCEV *RangeDiff = SE->getMinusSCEV(Range, BaseRange);
+ return isa<SCEVCouldNotCompute>(RangeDiff) || !RangeDiff->isZero();
})) {
- LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
- "skipping DepSet\n");
+ LLVM_DEBUG(dbgs()
+ << "LAA: Member with different or not computable access "
+ "range, skipping DepSet\n");
continue;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 3ef1bd18a3ab2..fada2d09d0509 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1704,4 +1704,89 @@ exit:
ret void
}
+;; Test 15: %gep Start/End expressions are min/max expressions.
+;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
+define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
+; MERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %p
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + %q))
+; MERGE-NEXT: Member: %q
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; MERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %p High: (1 + %p))
+; MERGE-NEXT: Member: %p
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %p
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + %q))
+; NOMERGE-NEXT: Member: %q
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; NOMERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %p High: (1 + %p))
+; NOMERGE-NEXT: Member: %p
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %step = add i64 %m, 1
+ br label %loop
+
+loop:
+ %off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
+ %i = phi i1 [ false, %entry ], [ true, %loop ]
+ load i8, ptr %p
+ %gep = getelementptr i8, ptr %p, i64 %off
+ load i8, ptr %gep
+ store i8 0, ptr %q
+ %off.next = add i64 %off, %step
+ br i1 %i, label %done, label %loop
+
+done:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 21f35de5e153004dcaa1c38d46b6b8470a729e8a Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 11:11:00 +0000
Subject: [PATCH 11/14] Add high level explanation to mergeStencilGroups like
in groupChecks
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index f482e88276323..175b2c3a21e0e 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -823,6 +823,28 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
+ // We try to merge groups produced by groupChecks when their pointers follow
+ // a stencil access pattern. groupChecks intentionally only merges pointers
+ // whose min/max bounds differ by constants, because that keeps each runtime
+ // check precise. Stencil kernels often read the same underlying object at
+ // several loop-invariant stride offsets, so those groups remain separate and
+ // can produce too many checks. Here we trade some precision for fewer
+ // checks by replacing a set of same-dependence, same-alias read groups with
+ // one conservative bounding group.
+ //
+ // We use the following algorithm to construct a merged stencil group:
+ // - collect checking groups that share both DependencySetId and AliasSetId;
+ // - reject groups with writes, predicated accesses, different access
+ // ranges, or different recurrence steps;
+ // - use one member as the base and decompose each other member's offset
+ // from that base as C + sum(Coeff[Stride] * Stride), where Stride is
+ // loop-invariant;
+ // - build one bounding range by taking the min/max constant offset and the
+ // min/max coefficient for each stride, adding predicates for strides
+ // that are not already known positive;
+ // - commit the merge only if the local cost model reduces the number of
+ // checks after accounting for any new predicates.
+
// Stencil merging runs when either:
// - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
// - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
>From e3bbaba01133f1c23fc53761a8fd56c2817b299c Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 12:26:54 +0000
Subject: [PATCH 12/14] Clang format
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 175b2c3a21e0e..ef06baf1699eb 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -978,8 +978,8 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
const SCEV *RangeDiff = SE->getMinusSCEV(Range, BaseRange);
return isa<SCEVCouldNotCompute>(RangeDiff) || !RangeDiff->isZero();
})) {
- LLVM_DEBUG(dbgs()
- << "LAA: Member with different or not computable access "
+ LLVM_DEBUG(
+ dbgs() << "LAA: Member with different or not computable access "
"range, skipping DepSet\n");
continue;
}
>From 49d3db1f5adb94934270eae45217b9f17b42eaf1 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 18 Jun 2026 16:29:25 +0100
Subject: [PATCH 13/14] Address comments
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 264 ++++++++++++------
.../runtime-check-group-merging.ll | 107 +++++++
2 files changed, 290 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index ef06baf1699eb..f4661e5d31d46 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -123,6 +123,13 @@ static cl::opt<unsigned> StencilMergeCheckThreshold(
"-stencil-runtime-check-merge is auto."),
cl::init(128));
+static cl::opt<unsigned> StencilMergeMaxGroups(
+ "stencil-merge-max-groups", cl::Hidden,
+ cl::desc(
+ "Skip stencil group merging when the number of runtime checking groups "
+ "exceeds this limit, to bound compile time (default =4096)."),
+ cl::init(4096));
+
/// Maximum SIMD width.
const unsigned VectorizerParams::MaxVectorWidth = 64;
@@ -771,6 +778,11 @@ struct StencilDecomposition {
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
+/// \p Expr is the difference of two access "Start" SCEVs (Start_member -
+/// Start_base). A "Start" is the low bound of a memory access range as computed
+/// by getStartAndEndForAccess: the address of the first byte the access can
+/// touch. The result describes where one member's range sits relative to the
+/// base member's range.
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
/// Returns std::nullopt if the expression contains non-stencil terms or any
@@ -781,9 +793,9 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
- // The only caller passes a difference of two pointer Starts, and a Start is
- // always loop-invariant (getStartAndEndForAccess asserts it). So Expr is
- // loop-invariant and every term below is loop-invariant too.
+ // A "Start" is always loop-invariant (getStartAndEndForAccess asserts it), so
+ // the difference Expr passed in by the caller is loop-invariant too, and so
+ // is every additive term we pull out of it below.
assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
// Collect top-level additive terms.
@@ -848,15 +860,28 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Stencil merging runs when either:
// - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
// - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
- // current check count exceeds the auto-trigger threshold, where the
- // vectorizer would otherwise reject the loop for having too many runtime
- // checks. In that case the merge can only improve things: at worst we
- // decline to merge and behave as before.
+ // current check count exceeds the auto-trigger threshold, which defaults
+ // to the vectorizer's own runtime-check cutoff
+ // (-vectorize-memory-check-threshold). Above it the vectorizer would
+ // otherwise reject the loop for having too many runtime checks. In that
+ // case the merge can only improve things: at worst we decline to merge
+ // and behave as before.
if (StencilMerge == StencilMergePolicy::Off) {
LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled\n");
return;
}
+ // For each checking group this pass decomposes each member's offset into
+ // stencil form and builds a merged bounding box. That extra SCEV work adds up
+ // on a loop with very many groups, so bail out above a configurable limit as
+ // a safety net against pathological inputs.
+ if (CheckingGroups.size() > StencilMergeMaxGroups) {
+ LLVM_DEBUG(
+ dbgs() << "LAA: " << CheckingGroups.size()
+ << " groups exceeds stencil-merge-max-groups, skipping\n");
+ return;
+ }
+
if (StencilMerge == StencilMergePolicy::Auto) {
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
@@ -897,17 +922,23 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (GroupIndices.size() < 2)
continue;
- // Collect all member pointers across these groups.
+ // Collect all member pointers across these groups. Only merge read-only
+ // groups: stencil patterns read an array at multiple offsets and write to a
+ // different array (a different DepSet). Mixing reads and writes within a
+ // merged group complicates the cost model and doesn't match known stencil
+ // patterns, so stop and skip the whole DepSet as soon as we see a write.
SmallVector<unsigned, 8> AllMembers;
- for (unsigned GI : GroupIndices)
- append_range(AllMembers, CheckingGroups[GI].Members);
-
- // Only merge read-only groups. Stencil patterns read an array at
- // multiple offsets and write to a different array (different DepSet).
- // Mixing reads and writes within a merged group complicates the cost
- // model and doesn't match known stencil patterns.
- if (any_of(AllMembers,
- [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
+ bool HasWrite = false;
+ for (unsigned GI : GroupIndices) {
+ ArrayRef<unsigned> Members = CheckingGroups[GI].Members;
+ if (any_of(Members,
+ [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
+ HasWrite = true;
+ break;
+ }
+ append_range(AllMembers, Members);
+ }
+ if (HasWrite) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with write access\n");
continue;
@@ -939,29 +970,37 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
- << ") with " << AllMembers.size() << " members, base: "
- << *Pointers[AllMembers[0]].Start << "\n");
-
- // Use the first member as the reference for decomposition. All offsets
- // are computed relative to BaseLow, and MergedHigh is built from
+ // Use the first member as the reference for decomposition. All offsets are
+ // computed relative to BaseLow, and the merged upper bound is built from
// BaseHigh (see merged-bounds computation below).
- const SCEV *BaseLow = Pointers[AllMembers[0]].Start;
- const SCEV *BaseHigh = Pointers[AllMembers[0]].End;
+ unsigned Member0 = AllMembers[0];
+ const SCEV *BaseLow = Pointers[Member0].Start;
+ const SCEV *BaseHigh = Pointers[Member0].End;
+
+ LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
+ << ") with " << AllMembers.size()
+ << " members, base: " << *BaseLow << "\n");
- const SCEV *BaseStep = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[AllMembers[0]].Expr))
- if (AR->getLoop() == &L)
- BaseStep = AR->getStepRecurrence(*SE);
+ auto GetStepForPointer = [&](unsigned Idx) -> const SCEV * {
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ return AR->getStepRecurrence(*SE);
+ return nullptr;
+ };
+ const SCEV *BaseStep = GetStepForPointer(Member0);
if (!BaseStep)
continue;
- // Verify all members have the same access range (End - Start).
- // MergedHigh = BaseHigh + max_offsets is only correct when every member's
- // range equals BaseHigh - BaseLow. Compute the ranges first and compare
- // them by subtracting, so algebraically equal but non-identical SCEVs still
- // match. Bail out if any subtraction produces SCEVCouldNotCompute.
+ // Verify all members have the same access range (End - Start). The merged
+ // group gets a single upper bound (MergedHigh, built further below) of the
+ // form BaseHigh + max_offsets. That is only correct when every member's
+ // range equals BaseHigh - BaseLow; otherwise a member with a larger range
+ // could reach past MergedHigh.
+ // Compare each member's range (End - Start) and test Range - BaseRange ==
+ // 0, rather than Range == BaseRange, so algebraically equal but
+ // non-identical SCEVs still match. Bail out if any subtraction produces
+ // SCEVCouldNotCompute.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
if (isa<SCEVCouldNotCompute>(BaseRange)) {
LLVM_DEBUG(dbgs() << "LAA: Base access range not computable, "
@@ -992,18 +1031,17 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// with the BaseStep check above this keeps the decision the same no matter
// which member comes first: we only merge recurrences with one common step.
if (any_of(AllMembers, [&](unsigned Idx) {
- const SCEV *Step = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
- if (AR->getLoop() == &L)
- Step = AR->getStepRecurrence(*SE);
- return Step != BaseStep;
+ return GetStepForPointer(Idx) != BaseStep;
})) {
LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
"skipping DepSet\n");
continue;
}
+ // Per-stride min/max coefficient seen across all members, and the min/max
+ // of the constant part of the offset. Together they describe the bounding
+ // box.
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
- int64_t CMin = 0, CMax = 0;
+ int64_t MinConstOffset = 0, MaxConstOffset = 0;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
// Decompose one member's offset (relative to BaseLow) and fold its
@@ -1021,9 +1059,12 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return false;
}
- CMin = std::min(CMin, DLow->Constant);
- CMax = std::max(CMax, DLow->Constant);
+ MinConstOffset = std::min(MinConstOffset, DLow->Constant);
+ MaxConstOffset = std::max(MaxConstOffset, DLow->Constant);
+ // Fold this member's coefficient for each stride into the running min and
+ // max for that stride. MinCoeff and MaxCoeff are kept in lock-step: a
+ // stride is always present in both or neither.
for (const auto &[Stride, Coeff] : DLow->Coefficients) {
auto MinIt = MinCoeff.find(Stride);
if (MinIt == MinCoeff.end()) {
@@ -1031,14 +1072,16 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
MaxCoeff[Stride] = Coeff;
} else {
MinIt->second = std::min(MinIt->second, Coeff);
- MaxCoeff[Stride] = std::max(MaxCoeff[Stride], Coeff);
+ auto MaxIt = MaxCoeff.find(Stride);
+ MaxIt->second = std::max(MaxIt->second, Coeff);
}
if (!SE->isKnownPositive(Stride))
LocalStridesNeedingPreds.insert(Stride);
}
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << ": Const=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
return true;
};
@@ -1046,43 +1089,74 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (!all_of(AllMembers, AccumulateOffset))
continue;
- // The base member (offset = 0) implicitly has coefficient 0 for every
- // stride. Members that lack a particular stride term also have an
- // implicit coefficient of 0 for that stride. Clamp so that the
- // bounding-box always includes coefficient 0, which is guaranteed to
- // exist (at least from the base member).
- for (auto &[Stride, MinC] : MinCoeff)
- MinC = std::min(MinC, (int64_t)0);
- for (auto &[Stride, MaxC] : MaxCoeff)
- MaxC = std::max(MaxC, (int64_t)0);
-
- // MergedLow = BaseLow + CMin + sum(MinCoeff[s] * s)
+ // A member implicitly has coefficient 0 for every stride it does not
+ // mention. So the bounding box must always include coefficient 0 per
+ // stride: clamp the running min down to 0 and the running max up to 0.
+ //
+ // For example, three pointers with two strides s1, s2:
+ // base = p
+ // m1 = p - s1 + s2
+ // m2 = p + 2*s2
+ // Collecting only the coefficients that appear gives:
+ // s1: [-1] -> range [-1, -1]
+ // s2: [1, 2] -> range [ 1, 2]
+ // which is wrong, because it forgets the implicit zeros.
+ // Including every stride in every member:
+ // base = p + 0*s1 + 0*s2
+ // m1 = p + -1*s1 + 1*s2
+ // m2 = p + 0*s1 + 2*s2
+ // gives the correct ranges:
+ // s1: [-1, 0] -> range [-1, 0]
+ // s2: [0, 2] -> range [ 0, 2]
+ // The clamps below enforce exactly that by folding 0 into each stride's
+ // minimum and maximum coefficient.
+ for (auto &[_, MinCoeffVal] : MinCoeff)
+ MinCoeffVal = std::min(MinCoeffVal, (int64_t)0);
+ for (auto &[_, MaxCoeffVal] : MaxCoeff)
+ MaxCoeffVal = std::max(MaxCoeffVal, (int64_t)0);
+
+ // OffsetTy is the type of a member's offset from the base,
+ // Start_member - BaseLow, which is an integer. The merged bounds below are
+ // built as BaseLow/BaseHigh (pointers) plus integer terms of this type.
+ Type *OffsetTy = SE->getEffectiveSCEVType(BaseLow->getType());
+
+ // Construct the merged lower bound:
+ // MergedLow = BaseLow + MinConstOffset + sum(MinCoeff[s] * s)
+ // This assumes every stride s > 0, so the smallest coefficient gives the
+ // smallest address. The SCEV predicates added below (before the loop runs)
+ // guarantee that for any stride not already known positive.
const SCEV *MergedLow = BaseLow;
- if (CMin != 0)
+ if (MinConstOffset != 0)
MergedLow =
- SE->getAddExpr(MergedLow, SE->getConstant(BaseLow->getType(), CMin,
+ SE->getAddExpr(MergedLow, SE->getConstant(OffsetTy, MinConstOffset,
/*isSigned=*/true));
- for (const auto &[Stride, MinC] : MinCoeff) {
- if (MinC != 0)
+ for (const auto &[Stride, MinCoeffVal] : MinCoeff) {
+ // MinCoeff and MaxCoeff share their keys, so this also covers MaxCoeff.
+ assert(Stride->getType() == OffsetTy &&
+ "stride type must match the offset type");
+ if (MinCoeffVal != 0)
MergedLow = SE->getAddExpr(
- MergedLow, SE->getMulExpr(SE->getConstant(Stride->getType(), MinC,
+ MergedLow, SE->getMulExpr(SE->getConstant(OffsetTy, MinCoeffVal,
/*isSigned=*/true),
Stride));
}
- // MergedHigh = BaseHigh + CMax + sum(MaxCoeff[s] * s)
- // BaseHigh = BaseLow + Range, so this gives max(Low_j) + Range.
+ // Construct the merged upper bound:
+ // MergedHigh = BaseHigh + MaxConstOffset + sum(MaxCoeff[s] * s)
+ // Since BaseHigh = BaseLow + Range and every member shares Range, this is
+ // max(Start_j) + Range, i.e. the highest address any member can reach. As
+ // with MergedLow this assumes s > 0, guaranteed by the predicates below.
const SCEV *MergedHigh = BaseHigh;
- if (CMax != 0)
+ if (MaxConstOffset != 0)
MergedHigh =
- SE->getAddExpr(MergedHigh, SE->getConstant(BaseHigh->getType(), CMax,
+ SE->getAddExpr(MergedHigh, SE->getConstant(OffsetTy, MaxConstOffset,
/*isSigned=*/true));
- for (const auto &[Stride, MaxC] : MaxCoeff) {
- if (MaxC != 0)
+ for (const auto &[Stride, MaxCoeffVal] : MaxCoeff) {
+ if (MaxCoeffVal != 0)
MergedHigh = SE->getAddExpr(
- MergedHigh, SE->getMulExpr(SE->getConstant(Stride->getType(), MaxC,
+ MergedHigh, SE->getMulExpr(SE->getConstant(OffsetTy, MaxCoeffVal,
/*isSigned=*/true),
Stride));
}
@@ -1090,37 +1164,63 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
<< ", High=" << *MergedHigh << "\n");
- // Local cost model.
- // G = number of groups in this DepSet, C = number of external groups
- // that need checking against any member of this DepSet.
- // Before merge: G * C checks. After merge: C + predicates.
- unsigned G = GroupIndices.size();
+ // Local cost model: decide whether replacing this DepSet's groups with the
+ // single merged group actually reduces the number of runtime checks.
+ //
+ // The unit is one runtime check: a single bounds comparison emitted between
+ // two groups (the low bound of one against the high bound of the other).
+ // Two groups only produce a check when needsChecking() says so. We assume
+ // every such check costs the same before and after merging (each is the
+ // same pair of pointer comparisons in the IR), so we can just count them.
+ //
+ // NumGroups - groups in this DepSet; all of them collapse into
+ // the
+ // one merged group.
+ // NumExternalChecks - groups *outside* this DepSet that need a check
+ // against at least one member of it. The merged group
+ // will still be checked against exactly these.
+ //
+ // Before merging, each of the NumGroups groups is checked against each of
+ // the NumExternalChecks external groups, giving NumGroups *
+ // NumExternalChecks checks. Every group in this DepSet is read-only and
+ // shares the same (DependencySetId, AliasSetId), and needsChecking() looks
+ // only at those IDs and at whether a group writes memory. So toward any
+ // given external group either all of these groups need a check or none do,
+ // making the count exactly the product above. After merging only the single
+ // merged group remains, so just NumExternalChecks checks, plus one extra
+ // check (a SCEV predicate) for each stride we must prove positive and have
+ // not already paid for in an earlier DepSet.
+ //
+ // To find the external groups we walk *all* checking groups and keep the
+ // ones that are neither part of this DepSet (GroupIndexSet) nor already
+ // consumed by a previous merge in this same call (MergedGroupIndices).
+ unsigned NumGroups = GroupIndices.size();
SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
GroupIndices.end());
- unsigned C = 0;
+ unsigned NumExternalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
continue;
- // Check if this external group needs checking against any member
- // of our DepSet.
for (unsigned GI : GroupIndices) {
if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
- ++C;
+ ++NumExternalChecks;
break;
}
}
}
- // Only count predicates we haven't already committed as cost.
+ // Each not-yet-committed positive-stride predicate becomes one extra
+ // runtime check, so it counts against the saving.
unsigned NewPredicates = 0;
for (const SCEV *Stride : LocalStridesNeedingPreds)
if (!CommittedStridePredicates.contains(Stride))
++NewPredicates;
- unsigned ChecksBefore = G * C;
- unsigned ChecksAfter = C + NewPredicates;
+ unsigned ChecksBefore = NumGroups * NumExternalChecks;
+ unsigned ChecksAfter = NumExternalChecks + NewPredicates;
- LLVM_DEBUG(dbgs() << "LAA: Cost model: G=" << G << ", C=" << C
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
+ << ", NumExternalChecks=" << NumExternalChecks
<< ", predicates=" << NewPredicates << ", checks "
<< ChecksBefore << "->" << ChecksAfter);
@@ -1139,7 +1239,9 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
for (unsigned GI : GroupIndices)
CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
- // COMMIT: add stride predicates (skip already-committed ones).
+ // We have decided to merge, so now actually register the positive-stride
+ // SCEV predicates with PSE (skipping any stride an earlier DepSet already
+ // added a predicate for).
for (const SCEV *Stride : LocalStridesNeedingPreds) {
if (!CommittedStridePredicates.insert(Stride).second)
continue;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index fada2d09d0509..433bd1e83a319 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1790,3 +1790,110 @@ done:
}
declare i64 @llvm.smax.i64(i64, i64)
+
+;; Test 16: A stride that is itself a sum (s1 + s2).
+;; s1 = smax(s1in, 1), s2 = smax(s2in, 1) (known positive); 3 loads from %p at
+;; offsets {0, s1, s1 + s2}, store to %q.
+;; TODO: Distribute s1 + s2 into s1 and s2 in decomposeStencilOffset
+define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_summed_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * (1 smax %s2in))<nsw> + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; MERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: ((1 smax %s1in) + (1 smax %s2in)) sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_summed_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + (1 smax %s2in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s12 = add i64 %s1, %s2
+ %p1 = getelementptr i8, ptr %p, i64 %s1
+ %p2 = getelementptr i8, ptr %p, i64 %s12
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ load i8, ptr %a0
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ load i8, ptr %a1
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ load i8, ptr %a2
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 0, ptr %aq
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
>From 714c739ad34296def4c89748a939c09f1db68ef2 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 23 Jun 2026 15:54:43 +0100
Subject: [PATCH 14/14] Split mergeStencilGroups loop into helper functions
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 196 +++++++++++++----------
1 file changed, 113 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index f4661e5d31d46..9a987a4ecb7d2 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -827,6 +827,109 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return D;
}
+/// Local cost model: count the runtime checks required before and after
+/// replacing one DepSet's groups (\p GroupIndices) with the single merged
+/// group.
+///
+/// The unit is one runtime check: a single bounds comparison emitted between
+/// two groups (the low bound of one against the high bound of the other). Two
+/// groups only produce a check when needsChecking() says so. We assume every
+/// such check costs the same before and after merging (each is the same pair of
+/// pointer comparisons in the IR), so we can just count them.
+///
+/// NumGroups - groups in this DepSet; all of them collapse into the
+/// one merged group.
+/// NumExternalChecks - groups *outside* this DepSet that need a check against
+/// at least one member of it. The merged group will still
+/// be checked against exactly these.
+///
+/// Before merging, each of the NumGroups groups is checked against each of the
+/// NumExternalChecks external groups, giving NumGroups * NumExternalChecks
+/// checks. Every group in this DepSet is read-only and shares the same
+/// (DependencySetId, AliasSetId), and needsChecking() looks only at those IDs
+/// and at whether a group writes memory. So toward any given external group
+/// either all of these groups need a check or none do, making the count exactly
+/// the product above. After merging only the single merged group remains, so
+/// just NumExternalChecks checks, plus one extra check (a SCEV predicate) for
+/// each stride we must prove positive and have not already paid for in an
+/// earlier DepSet (those already in \p CommittedStridePredicates).
+///
+/// To find the external groups we walk *all* checking groups and keep the ones
+/// that are neither part of this DepSet (\p GroupIndices) nor already consumed
+/// by a previous merge in this same call (\p MergedGroupIndices).
+///
+/// Returns {ChecksBefore, ChecksAfter}.
+static std::pair<unsigned, unsigned>
+computeStencilMergeCost(const RuntimePointerChecking &RtCheck,
+ ArrayRef<unsigned> GroupIndices,
+ const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ const SmallDenseSet<const SCEV *, 4>
+ &CommittedStridePredicates) {
+ ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
+ unsigned NumGroups = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned NumExternalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ for (unsigned GI : GroupIndices) {
+ if (RtCheck.needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++NumExternalChecks;
+ break;
+ }
+ }
+ }
+
+ // Each not-yet-committed positive-stride predicate becomes one extra runtime
+ // check, so it counts against the saving.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
+ << ", NumExternalChecks=" << NumExternalChecks
+ << ", predicates=" << NewPredicates << ", checks "
+ << NumGroups * NumExternalChecks << "->"
+ << NumExternalChecks + NewPredicates << "\n");
+
+ return {NumGroups * NumExternalChecks, NumExternalChecks + NewPredicates};
+}
+
+/// Build the merged stencil group for one DepSet, after the cost model has
+/// decided the merge is profitable. Constructs the bounding group over
+/// \p AllMembers with bounds [\p MergedLow, \p MergedHigh], and registers with
+/// \p PSE a positive-stride SCEV predicate for each stride in
+/// \p LocalStridesNeedingPreds that has not already been committed (tracked in
+/// \p CommittedStridePredicates across DepSets). Returns the new group.
+static RuntimeCheckingPtrGroup buildMergedStencilGroup(
+ const RuntimePointerChecking &RtCheck, PredicatedScalarEvolution &PSE,
+ ScalarEvolution &SE, ArrayRef<unsigned> AllMembers, const SCEV *MergedLow,
+ const SCEV *MergedHigh, ArrayRef<unsigned> GroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates) {
+ RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], RtCheck);
+ CandidateGroup.Low = MergedLow;
+ CandidateGroup.High = MergedHigh;
+ append_range(CandidateGroup.Members, drop_begin(AllMembers));
+ for (unsigned GI : GroupIndices)
+ CandidateGroup.NeedsFreeze |= RtCheck.CheckingGroups[GI].NeedsFreeze;
+
+ // Register the positive-stride SCEV predicates with PSE, skipping any stride
+ // an earlier DepSet already added a predicate for.
+ for (const SCEV *Stride : LocalStridesNeedingPreds) {
+ if (!CommittedStridePredicates.insert(Stride).second)
+ continue;
+ const SCEV *Zero = SE.getZero(Stride->getType());
+ PSE.addPredicate(*SE.getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+ return CandidateGroup;
+}
+
void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
Loop &L) {
LLVM_DEBUG(dbgs() << "LAA: Attempting stencil group merging on "
@@ -1166,94 +1269,21 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Local cost model: decide whether replacing this DepSet's groups with the
// single merged group actually reduces the number of runtime checks.
- //
- // The unit is one runtime check: a single bounds comparison emitted between
- // two groups (the low bound of one against the high bound of the other).
- // Two groups only produce a check when needsChecking() says so. We assume
- // every such check costs the same before and after merging (each is the
- // same pair of pointer comparisons in the IR), so we can just count them.
- //
- // NumGroups - groups in this DepSet; all of them collapse into
- // the
- // one merged group.
- // NumExternalChecks - groups *outside* this DepSet that need a check
- // against at least one member of it. The merged group
- // will still be checked against exactly these.
- //
- // Before merging, each of the NumGroups groups is checked against each of
- // the NumExternalChecks external groups, giving NumGroups *
- // NumExternalChecks checks. Every group in this DepSet is read-only and
- // shares the same (DependencySetId, AliasSetId), and needsChecking() looks
- // only at those IDs and at whether a group writes memory. So toward any
- // given external group either all of these groups need a check or none do,
- // making the count exactly the product above. After merging only the single
- // merged group remains, so just NumExternalChecks checks, plus one extra
- // check (a SCEV predicate) for each stride we must prove positive and have
- // not already paid for in an earlier DepSet.
- //
- // To find the external groups we walk *all* checking groups and keep the
- // ones that are neither part of this DepSet (GroupIndexSet) nor already
- // consumed by a previous merge in this same call (MergedGroupIndices).
- unsigned NumGroups = GroupIndices.size();
- SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
- GroupIndices.end());
- unsigned NumExternalChecks = 0;
- for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
- if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
- continue;
- for (unsigned GI : GroupIndices) {
- if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
- ++NumExternalChecks;
- break;
- }
- }
- }
-
- // Each not-yet-committed positive-stride predicate becomes one extra
- // runtime check, so it counts against the saving.
- unsigned NewPredicates = 0;
- for (const SCEV *Stride : LocalStridesNeedingPreds)
- if (!CommittedStridePredicates.contains(Stride))
- ++NewPredicates;
-
- unsigned ChecksBefore = NumGroups * NumExternalChecks;
- unsigned ChecksAfter = NumExternalChecks + NewPredicates;
-
- LLVM_DEBUG(dbgs() << "LAA: Cost model: NumGroups=" << NumGroups
- << ", NumExternalChecks=" << NumExternalChecks
- << ", predicates=" << NewPredicates << ", checks "
- << ChecksBefore << "->" << ChecksAfter);
-
+ auto [ChecksBefore, ChecksAfter] = computeStencilMergeCost(
+ *this, GroupIndices, MergedGroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates);
if (ChecksAfter >= ChecksBefore) {
- LLVM_DEBUG(dbgs() << " (skipping, not beneficial)\n");
+ LLVM_DEBUG(dbgs() << "LAA: Not beneficial, skipping DepSet\n");
continue;
}
- LLVM_DEBUG(dbgs() << " (merging, net saving " << ChecksBefore - ChecksAfter
- << ")\n");
-
- // Build the merged group (after deciding to merge).
- RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
- CandidateGroup.Low = MergedLow;
- CandidateGroup.High = MergedHigh;
- append_range(CandidateGroup.Members, drop_begin(AllMembers));
- for (unsigned GI : GroupIndices)
- CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
-
- // We have decided to merge, so now actually register the positive-stride
- // SCEV predicates with PSE (skipping any stride an earlier DepSet already
- // added a predicate for).
- for (const SCEV *Stride : LocalStridesNeedingPreds) {
- if (!CommittedStridePredicates.insert(Stride).second)
- continue;
- const SCEV *Zero = SE->getZero(Stride->getType());
- PSE.addPredicate(
- *SE->getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
- LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
- << *Stride << "\n");
- }
+ LLVM_DEBUG(dbgs() << "LAA: Merging, net saving "
+ << ChecksBefore - ChecksAfter << "\n");
+ // Build the merged group and register its positive-stride predicates.
+ NewMergedGroups.push_back(buildMergedStencilGroup(
+ *this, PSE, *SE, AllMembers, MergedLow, MergedHigh, GroupIndices,
+ LocalStridesNeedingPreds.getArrayRef(), CommittedStridePredicates));
MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
- NewMergedGroups.push_back(std::move(CandidateGroup));
}
// Rebuild CheckingGroups if we merged anything.
More information about the llvm-commits
mailing list