[llvm] [LAA] Add stencil group merging to reduce runtime pointer checks (PR #187252)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 5 05:16:53 PDT 2026
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/187252
>From f1717dff57a6ee09021f9b4867ccd7fd68bc1d8a Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 12:17:21 +0000
Subject: [PATCH 01/11] [LAA] Add stencil group merging to reduce runtime
pointer checks
Add a post-processing pass in RuntimePointerChecking that merges
checking groups whose pointers share a base object and differ only
by linear combinations of loop-invariant strides. This reduces the
number of runtime overlap checks for stencil-like access patterns,
enabling vectorization of loops that would otherwise be rejected
due to exceeding the runtime-check threshold.
The primary motivation is the Einstein Toolkit / Cactus CCZ4
numerical relativity stencil code, where 3D stencils with two
runtime strides (cdj, cdk) produce thousands of runtime checks.
The optimization:
- Decomposes member offsets into constant + sum(coeff * stride)
- Computes a bounding box over the coefficient space
- Applies a cost model: only merges when checks_after < checks_before
- Adds stride > 0 predicates for non-provably-positive strides
- Rejects predicated accesses and write groups conservatively
Gated behind -enable-stencil-runtime-check-merge (default: off).
Also activates automatically when check count exceeds
-stencil-merge-check-threshold (default: 128).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 8 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 380 ++++-
.../affine-group-merging.ll | 1406 +++++++++++++++++
3 files changed, 1791 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..65354d4e856a6 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -566,7 +566,8 @@ class RuntimePointerChecking {
/// Generate the checks and store it. This also performs the grouping
/// of pointers to reduce the number of memchecks necessary.
- LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands);
+ LLVM_ABI void generateChecks(MemoryDepChecker::DepCandidates &DepCands,
+ PredicatedScalarEvolution &PSE, Loop &L);
/// Returns the checks that generateChecks created. They can be used to ensure
/// no read/write accesses overlap across all loop iterations.
@@ -636,6 +637,11 @@ class RuntimePointerChecking {
/// and re-compute it.
void groupChecks(MemoryDepChecker::DepCandidates &DepCands);
+ /// Attempt to merge checking groups that share a base pointer and differ
+ /// by stencil functions of loop-invariant strides. This reduces runtime
+ /// checks for multi-dimensional stencil-like access patterns.
+ void mergeStencilGroups(PredicatedScalarEvolution &PSE, Loop &L);
+
/// Generate the checks and return them.
SmallVector<RuntimePointerCheck, 4> generateChecks();
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 60e15b2a5bd82..d7d83f61a05ac 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -15,6 +15,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
@@ -99,6 +100,18 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable merging of runtime check groups with stencil stride "
+ "patterns (default = false)"),
+ cl::init(false));
+
+static cl::opt<unsigned> StencilMergeCheckThreshold(
+ "stencil-merge-check-threshold", cl::Hidden,
+ cl::desc("Trigger stencil group merging when runtime check count "
+ "exceeds this threshold (default = 128)"),
+ cl::init(128));
+
/// Maximum SIMD width.
const unsigned VectorizerParams::MaxVectorWidth = 64;
@@ -552,9 +565,11 @@ SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() {
}
void RuntimePointerChecking::generateChecks(
- MemoryDepChecker::DepCandidates &DepCands) {
+ MemoryDepChecker::DepCandidates &DepCands, PredicatedScalarEvolution &PSE,
+ Loop &L) {
assert(Checks.empty() && "Checks is not empty");
groupChecks(DepCands);
+ mergeStencilGroups(PSE, L);
Checks = generateChecks();
}
@@ -737,6 +752,367 @@ void RuntimePointerChecking::groupChecks(
}
}
+/// Result of decomposing a SCEV expression into stencil offset form:
+/// Offset = Constant + sum(Coefficients[stride] * stride)
+/// where each stride is a loop-invariant SCEV expression.
+struct StencilDecomposition {
+ int64_t Constant = 0;
+ /// Map from loop-invariant stride SCEV to its integer coefficient.
+ SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+};
+
+/// Try to decompose \p Expr into a stencil offset function of loop-invariant
+/// strides: C + a1*s1 + a2*s2 + ...
+/// Relies on SCEV's canonical form: AddExpr operands are flattened,
+/// MulExpr has the constant operand first.
+/// Returns std::nullopt if the expression contains non-stencil terms.
+static std::optional<StencilDecomposition>
+decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
+ StencilDecomposition D;
+
+ // Collect top-level additive terms.
+ SmallVector<const SCEV *, 4> Terms;
+ if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
+ Terms.append(Add->operands().begin(), Add->operands().end());
+ else
+ Terms.push_back(Expr);
+
+ for (const SCEV *Term : Terms) {
+ if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ D.Constant += C->getAPInt().getSExtValue();
+ } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
+ // SCEV canonical form: constant is operand 0 in a MulExpr.
+ if (Mul->getNumOperands() != 2)
+ return std::nullopt;
+ auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
+ if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
+ return std::nullopt;
+ D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ } else if (SE.isLoopInvariant(Term, &L)) {
+ D.Coefficients[Term] += 1;
+ } else {
+ return std::nullopt;
+ }
+ }
+ return D;
+}
+
+void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
+ Loop &L) {
+ LLVM_DEBUG(dbgs() << "LAA: Attempting stencil group merging on "
+ << CheckingGroups.size() << " groups\n");
+
+ if (CheckingGroups.size() < 2)
+ return;
+
+ // Count the total checks from current grouping.
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks, proceeding with stencil merge\n");
+
+ // Group CheckingGroups by (DependencySetId, AliasSetId) pair.
+ // DependencySetId alone is not unique: it resets per alias set, so
+ // pointers in different alias sets can share the same DependencySetId.
+ // Use MapVector for deterministic iteration order across platforms.
+ using DepAliasKey = std::pair<unsigned, unsigned>;
+ MapVector<DepAliasKey, SmallVector<unsigned, 4>> DepSetToGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ const auto &P = Pointers[CheckingGroups[I].Members[0]];
+ DepSetToGroups[{P.DependencySetId, P.AliasSetId}].push_back(I);
+ }
+
+ SmallDenseSet<unsigned, 4> MergedGroupIndices;
+ SmallVector<RuntimeCheckingPtrGroup, 2> NewMergedGroups;
+ // Track strides that already have committed predicates (across all DepSets).
+ SmallDenseSet<const SCEV *, 4> CommittedStridePredicates;
+
+ for (auto &[DepAliasKey, GroupIndices] : DepSetToGroups) {
+ [[maybe_unused]] auto [DepId, ASId] = DepAliasKey;
+ if (GroupIndices.size() < 2)
+ continue;
+
+ // Collect all member pointers across these groups.
+ SmallVector<unsigned, 8> AllMembers;
+ for (unsigned GI : GroupIndices)
+ AllMembers.append(CheckingGroups[GI].Members.begin(),
+ CheckingGroups[GI].Members.end());
+
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ bool HasPredicatedAccess = false;
+ for (unsigned Idx : AllMembers) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ if (auto *I = dyn_cast<Instruction>(PtrVal)) {
+ if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT())) {
+ HasPredicatedAccess = true;
+ break;
+ }
+ }
+ }
+ if (HasPredicatedAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
+ // Only merge read-only groups. Stencil patterns read an array at
+ // multiple offsets and write to a different array (different DepSet).
+ // Mixing reads and writes within a merged group complicates the cost
+ // model and doesn't match known stencil patterns.
+ bool HasWriteAccess = false;
+ for (unsigned Idx : AllMembers) {
+ if (Pointers[Idx].IsWritePtr) {
+ HasWriteAccess = true;
+ break;
+ }
+ }
+ if (HasWriteAccess) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with write access\n");
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
+ << ") with " << AllMembers.size() << " members, base: "
+ << *Pointers[AllMembers[0]].Start << "\n");
+
+ // Use the first member as the reference for decomposition. All offsets
+ // are computed relative to BaseLow, and MergedHigh is built from
+ // BaseHigh (see merged-bounds computation below).
+ const SCEV *BaseLow = Pointers[AllMembers[0]].Start;
+ const SCEV *BaseHigh = Pointers[AllMembers[0]].End;
+
+ const SCEV *BaseStep = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[AllMembers[0]].Expr))
+ if (AR->getLoop() == &L)
+ BaseStep = AR->getStepRecurrence(*SE);
+
+ if (!BaseStep)
+ continue;
+
+ // Verify all members have the same access range (End - Start).
+ // MergedHigh = BaseHigh + max_offsets is only correct when every
+ // member's range equals BaseHigh - BaseLow. We check by subtracting
+ // ranges and testing for zero, which lets SCEV simplify algebraically
+ // even when the individual range SCEVs aren't pointer-identical.
+ const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ bool RangeMismatch = false;
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different access range, skipping DepSet\n");
+ RangeMismatch = true;
+ break;
+ }
+ }
+ if (RangeMismatch)
+ continue;
+
+ bool DecompOk = true;
+
+ // Verify all members have the same recurrence step w.r.t. the analyzed
+ // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
+ // correct when every member's High-Low range equals BaseHigh - BaseLow.
+ // Different steps (e.g., 8 vs 16) produce different ranges.
+ for (unsigned Idx : AllMembers) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+
+ if (Step != BaseStep) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " has different step, skipping DepSet\n");
+ DecompOk = false;
+ break;
+ }
+ }
+ if (!DecompOk)
+ continue;
+ SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
+ int64_t CMin = 0, CMax = 0;
+ SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+
+ for (unsigned Idx : AllMembers) {
+ const SCEV *PtrStart = Pointers[Idx].Start;
+
+ const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset)) {
+ DecompOk = false;
+ break;
+ }
+ auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
+ if (!DLow) {
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
+ << " NOT decomposable: " << *LowOffset << "\n");
+ DecompOk = false;
+ break;
+ }
+
+ CMin = std::min(CMin, DLow->Constant);
+ CMax = std::max(CMax, DLow->Constant);
+
+ for (const auto &[Stride, Coeff] : DLow->Coefficients) {
+ auto MinIt = MinCoeff.find(Stride);
+ if (MinIt == MinCoeff.end()) {
+ MinCoeff[Stride] = Coeff;
+ MaxCoeff[Stride] = Coeff;
+ } else {
+ MinIt->second = std::min(MinIt->second, Coeff);
+ MaxCoeff[Stride] = std::max(MaxCoeff[Stride], Coeff);
+ }
+
+ if (!SE->isKnownPositive(Stride))
+ LocalStridesNeedingPreds.insert(Stride);
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
+ << ", strides=" << DLow->Coefficients.size() << "\n");
+ }
+
+ if (!DecompOk)
+ continue;
+
+ // The base member (offset = 0) implicitly has coefficient 0 for every
+ // stride. Members that lack a particular stride term also have an
+ // implicit coefficient of 0 for that stride. Clamp so that the
+ // bounding-box always includes coefficient 0, which is guaranteed to
+ // exist (at least from the base member).
+ for (auto &[Stride, MinC] : MinCoeff)
+ MinC = std::min(MinC, (int64_t)0);
+ for (auto &[Stride, MaxC] : MaxCoeff)
+ MaxC = std::max(MaxC, (int64_t)0);
+
+ // MergedLow = BaseLow + CMin + sum(MinCoeff[s] * s)
+ const SCEV *MergedLow = BaseLow;
+ if (CMin != 0)
+ MergedLow =
+ SE->getAddExpr(MergedLow, SE->getConstant(BaseLow->getType(), CMin,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MinC] : MinCoeff) {
+ if (MinC != 0)
+ MergedLow = SE->getAddExpr(
+ MergedLow, SE->getMulExpr(SE->getConstant(Stride->getType(), MinC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ // MergedHigh = BaseHigh + CMax + sum(MaxCoeff[s] * s)
+ // BaseHigh = BaseLow + Range, so this gives max(Low_j) + Range.
+ const SCEV *MergedHigh = BaseHigh;
+ if (CMax != 0)
+ MergedHigh =
+ SE->getAddExpr(MergedHigh, SE->getConstant(BaseHigh->getType(), CMax,
+ /*isSigned=*/true));
+
+ for (const auto &[Stride, MaxC] : MaxCoeff) {
+ if (MaxC != 0)
+ MergedHigh = SE->getAddExpr(
+ MergedHigh, SE->getMulExpr(SE->getConstant(Stride->getType(), MaxC,
+ /*isSigned=*/true),
+ Stride));
+ }
+
+ LLVM_DEBUG(dbgs() << "LAA: Merged bounds: Low=" << *MergedLow
+ << ", High=" << *MergedHigh << "\n");
+
+ // Local cost model.
+ // G = number of groups in this DepSet, C = number of external groups
+ // that need checking against any member of this DepSet.
+ // Before merge: G * C checks. After merge: C + predicates.
+ unsigned G = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned C = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ // Check if this external group needs checking against any member
+ // of our DepSet.
+ for (unsigned GI : GroupIndices) {
+ if (needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++C;
+ break;
+ }
+ }
+ }
+
+ // Only count predicates we haven't already committed as cost.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ unsigned ChecksBefore = G * C;
+ unsigned ChecksAfter = C + NewPredicates;
+
+ LLVM_DEBUG(dbgs() << "LAA: Cost model: G=" << G << ", C=" << C
+ << ", predicates=" << NewPredicates << ", checks "
+ << ChecksBefore << "->" << ChecksAfter);
+
+ if (ChecksAfter >= ChecksBefore) {
+ LLVM_DEBUG(dbgs() << " (skipping, not beneficial)\n");
+ continue;
+ }
+ LLVM_DEBUG(dbgs() << " (merging, net saving " << ChecksBefore - ChecksAfter
+ << ")\n");
+
+ // Build the merged group (after deciding to merge).
+ RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
+ CandidateGroup.Low = MergedLow;
+ CandidateGroup.High = MergedHigh;
+ for (unsigned I = 1; I < AllMembers.size(); ++I)
+ CandidateGroup.Members.push_back(AllMembers[I]);
+ for (unsigned GI : GroupIndices)
+ CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
+
+ // COMMIT: add stride predicates (skip already-committed ones).
+ for (const SCEV *Stride : LocalStridesNeedingPreds) {
+ if (!CommittedStridePredicates.insert(Stride).second)
+ continue;
+ const SCEV *Zero = SE->getZero(Stride->getType());
+ PSE.addPredicate(
+ *SE->getComparePredicate(ICmpInst::ICMP_SGT, Stride, Zero));
+ LLVM_DEBUG(dbgs() << "LAA: Adding positive-stride predicate for "
+ << *Stride << "\n");
+ }
+
+ for (unsigned GI : GroupIndices)
+ MergedGroupIndices.insert(GI);
+ NewMergedGroups.push_back(std::move(CandidateGroup));
+ }
+
+ // Rebuild CheckingGroups if we merged anything.
+ if (!NewMergedGroups.empty()) {
+ SmallVector<RuntimeCheckingPtrGroup, 2> FinalGroups;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ if (!MergedGroupIndices.contains(I))
+ FinalGroups.push_back(std::move(CheckingGroups[I]));
+ FinalGroups.append(std::make_move_iterator(NewMergedGroups.begin()),
+ std::make_move_iterator(NewMergedGroups.end()));
+ CheckingGroups = std::move(FinalGroups);
+
+ LLVM_DEBUG(dbgs() << "LAA: After stencil merging: " << CheckingGroups.size()
+ << " groups\n");
+ }
+}
+
bool RuntimePointerChecking::arePointersInSamePartition(
const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
unsigned PtrIdx2) {
@@ -1502,7 +1878,7 @@ bool AccessAnalysis::canCheckPtrAtRT(
}
if (MayNeedRTCheck && (CanDoRT || AllowPartial))
- RtCheck.generateChecks(DepCands);
+ RtCheck.generateChecks(DepCands, PSE, *TheLoop);
LLVM_DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
<< " pointer comparisons.\n");
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
new file mode 100644
index 0000000000000..fb6b116a5b300
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
@@ -0,0 +1,1406 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+;; Test 1: Basic stencil merge with one runtime stride.
+;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
+;; relative to base = %a + 8*iv. Store to %out.
+;; With merge: all 6 loads form 1 merged group with bounds spanning [-3*cdj, +3*cdj]
+;; relative to base, producing 1 runtime check and 1 stride predicate (cdj > 0).
+;; Without merge: 6 separate groups (each load alone), 6 checks, no predicates.
+define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_single_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_single_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 2: Two runtime strides with constant offsets.
+;; 5 loads from %vol at offsets: +8+cdj-cdk, -16-2*cdj, +24+3*cdj+2*cdk,
+;; -8+2*cdj-3*cdk, +32-4*cdj+cdk. Store to %out.
+;; With merge: 1 merged group with bounds spanning the bounding box over
+;; both cdj and cdk coefficients, 1 check, 2 stride predicates.
+;; Without merge: 5 separate groups, 5 checks, no predicates.
+define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64 %cdk) {
+; MERGE-LABEL: 'stencil_merge_two_strides'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-4 * %cdj) + (-3 * %cdk) + %vol) High: ((2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; MERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-NEXT: Compare predicate: %cdk sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_two_strides'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (64 + (-4 * %cdj) + %cdk + %vol) High: ((8 * %n) + (-4 * %cdj) + %cdk + %vol))
+; NOMERGE-NEXT: Member: {(64 + (-4 * %cdj) + %cdk + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + (-3 * %cdk) + %vol) High: (-40 + (2 * %cdj) + (8 * %n) + (-3 * %cdk) + %vol))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + (-3 * %cdk) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (56 + (2 * %cdk) + (3 * %cdj) + %vol) High: (-8 + (2 * %cdk) + (3 * %cdj) + (8 * %n) + %vol))
+; NOMERGE-NEXT: Member: {(56 + (2 * %cdk) + (3 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %vol) High: (-48 + (8 * %n) + (-2 * %cdj) + %vol))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %vol),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (40 + (-1 * %cdk) + %cdj + %vol) High: (-24 + (8 * %n) + (-1 * %cdk) + %cdj + %vol))
+; NOMERGE-NEXT: Member: {(40 + (-1 * %cdk) + %cdj + %vol),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %vol, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; +8 + cdj - cdk
+ %off0a = add nsw i64 8, %cdj
+ %negcdk = sub nsw i64 0, %cdk
+ %off0 = add nsw i64 %off0a, %negcdk
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %v0 = load double, ptr %p0, align 8
+
+ ; -16 - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %off1 = add nsw i64 -16, %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %v1 = load double, ptr %p1, align 8
+
+ ; +24 + 3*cdj + 2*cdk
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %pos2cdk = mul nsw i64 %cdk, 2
+ %off2a = add nsw i64 24, %pos3cdj
+ %off2 = add nsw i64 %off2a, %pos2cdk
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %v2 = load double, ptr %p2, align 8
+
+ ; -8 + 2*cdj - 3*cdk
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %neg3cdk = mul nsw i64 %cdk, -3
+ %off3a = add nsw i64 -8, %pos2cdj
+ %off3 = add nsw i64 %off3a, %neg3cdk
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %v3 = load double, ptr %p3, align 8
+
+ ; +32 - 4*cdj + cdk
+ %neg4cdj = mul nsw i64 %cdj, -4
+ %off4a = add nsw i64 32, %neg4cdj
+ %off4 = add nsw i64 %off4a, %cdk
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %v4 = load double, ptr %p4, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s3, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 3: Constant offsets only — existing grouping handles this.
+;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
+;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
+;; Both with and without flag: 1 check, 2 groups, no predicates.
+define void @constant_offsets_only(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'constant_offsets_only'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'constant_offsets_only'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s2, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 4: Cost model rejection — only 2 loads with runtime stride.
+;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
+;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
+define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'cost_model_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'cost_model_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %v1 = load double, ptr %base, align 8
+
+ %s = fadd double %v0, %v1
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 5: Invariant + strided accesses to same base -> different steps.
+;; A strided store {%a,+,8} and an invariant store to %a have different
+;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Same result with and without flag: 2 checks, 3 separate groups.
+define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
+; MERGE-LABEL: 'different_steps_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Unknown:
+; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; MERGE-EMPTY:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %a
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'different_steps_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Unknown:
+; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
+; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %a
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; GRP0: invariant store to %a (single address, step=0):
+; GRP1: strided store to %a (step=8, different range from GRP0):
+; GRP2: load from %out:
+entry:
+ %cmp = icmp sgt i64 %n, 2
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+
+ ; Strided store: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ store double 1.0, ptr %gep.a, align 8
+
+ ; Invariant store to %a (step = 0, different range)
+ store double 2.0, ptr %a, align 8
+
+ ; Read from %out
+ %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+ %v = load double, ptr %gep.out, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 6: Coefficient clamping in merged bounds.
+;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. Members that lack a stride
+;; term have an implicit coefficient of 0, which must be included in the
+;; min/max computation via clamping.
+define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'coefficient_clamping_regression'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'coefficient_clamping_regression'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base + 0 (no stride offset -- this is the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; load at base - cdj
+ %negcdj = sub nsw i64 0, %cdj
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; load at base - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 7: Predicated accesses are rejected from stencil merging.
+;; Models the dilateKernel pattern (llvm-test-suite MicroBenchmarks/ImageProcessing/Dilate):
+;; 3 loads at {-cdj, 0, +cdj} where the -cdj and +cdj loads are conditional.
+;; Predicated loads have overapproximated SCEV bounds; merging would widen
+;; them further, causing false runtime overlap detection.
+;; Both modes: 3 checks (groups stay separate), no predicates.
+define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c1, i1 %c2) {
+; MERGE-LABEL: 'predicated_access_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 3 checks: each temp group separately vs %out (not merged due to predication):
+; 4 groups: 1 for %out, 3 separate temp groups:
+entry:
+ %cmp = icmp sgt i64 %n, 0
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Unconditional load at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Conditional load at base + cdj (predicated block)
+ br i1 %c1, label %if.then1, label %if.end1
+
+if.then1:
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+ br label %if.end1
+
+if.end1:
+ %m1 = phi double [ %v1, %if.then1 ], [ %v0, %loop ]
+
+ ; Conditional load at base - cdj (predicated block)
+ %negcdj = sub nsw i64 0, %cdj
+ br i1 %c2, label %if.then2, label %if.end2
+
+if.then2:
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+ br label %if.end2
+
+if.end2:
+ %m2 = phi double [ %v2, %if.then2 ], [ %m1, %if.end1 ]
+
+ %s = fadd double %m1, %m2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 8: Write pointer in a group prevents stencil merging.
+;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
+;; All three share the same base %a (same DepSet). The write prevents merging.
+;; Both modes: groups stay separate.
+define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'write_in_group_rejection'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'write_in_group_rejection'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+; 5 checks: write group vs all others, %out vs the two read groups:
+; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
+; Both modes produce identical output (write prevents merging).
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Read at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Read at base + cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; Write at base + 2*cdj (same DepSet as reads — prevents merging)
+ %cdj2 = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+ %s = fadd double %v0, %v1
+ store double %s, ptr %p2, align 8
+
+ ; Store to %out (different DepSet)
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 9: Mixed constant offsets + runtime stride.
+;; 4 constant-offset loads at {-16, -8, +8, +16} + 2 runtime-stride loads
+;; at {-cdj, +cdj}. Store to %out.
+;; Without merge: standard grouping merges the 4 constant-offset loads into
+;; 1 group but the 2 runtime-stride loads remain separate -> 3 checks.
+;; With merge: all 6 loads form 1 merged group -> 1 check + 1 predicate.
+;; Merged Low includes -cdj, Merged High includes +cdj:
+define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_constant_and_runtime_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Member: {(8 + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Member: {%a,+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; -16 (constant)
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ ; -8 (constant)
+ %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ ; +8 (constant)
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ ; +16 (constant)
+ %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ ; -cdj (runtime)
+ %negcdj = sub nsw i64 0, %cdj
+ %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v4 = load double, ptr %p4, align 8
+
+ ; +cdj (runtime)
+ %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 10: Shared stride predicate deduplication across two DepSets.
+;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
+;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
+;; should be emitted, not 2.
+;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 1 predicate.
+;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
+define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'shared_stride_predicate_dedup'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; MERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'shared_stride_predicate_dedup'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + %a) High: (-24 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + %b) High: (-24 + (8 * %n) + %b))
+; NOMERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + (-1 * %cdj) + %b))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+
+ ; Reads from %a at {-cdj, 0, +cdj}
+ %basea = getelementptr inbounds double, ptr %a, i64 %iv
+ %basea.i8 = bitcast ptr %basea to ptr
+ %negcdj = sub nsw i64 0, %cdj
+ %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %va0 = load double, ptr %pa0, align 8
+ %va1 = load double, ptr %basea, align 8
+ %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %va2 = load double, ptr %pa2, align 8
+
+ ; Reads from %b at {-cdj, 0, +cdj}
+ %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+ %baseb.i8 = bitcast ptr %baseb to ptr
+ %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %vb0 = load double, ptr %pb0, align 8
+ %vb1 = load double, ptr %baseb, align 8
+ %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %vb2 = load double, ptr %pb2, align 8
+
+ %s0 = fadd double %va0, %va1
+ %s1 = fadd double %s0, %va2
+ %s2 = fadd double %s1, %vb0
+ %s3 = fadd double %s2, %vb1
+ %s4 = fadd double %s3, %vb2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 11: Known-positive stride skips predicate emission.
+;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
+;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
+;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
+;; Without merge: 3 separate groups + 1 %out, 3 checks, no predicates.
+define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %cdj_in) {
+; MERGE-LABEL: 'known_positive_stride_no_predicate'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; MERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'known_positive_stride_no_predicate'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (1 smax %cdj_in) + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; NOMERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (-1 * (1 smax %cdj_in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+entry:
+ %cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+ %base.i8 = bitcast ptr %base to ptr
+
+ ; load at base - stride
+ %negcdj = sub nsw i64 0, %cdj
+ %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %v0 = load double, ptr %p0, align 8
+
+ ; load at base
+ %v1 = load double, ptr %base, align 8
+
+ ; load at base + stride
+ %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
>From 809bfc07a84a72f5b13506d59493c0461c1fa215 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Mar 2026 15:13:42 +0000
Subject: [PATCH 02/11] Use determenistic structure Rename test file
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 4 ++--
...affine-group-merging.ll => runtime-check-group-merging.ll} | 0
2 files changed, 2 insertions(+), 2 deletions(-)
rename llvm/test/Analysis/LoopAccessAnalysis/{affine-group-merging.ll => runtime-check-group-merging.ll} (100%)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index d7d83f61a05ac..9eba76ede932b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -758,7 +758,7 @@ void RuntimePointerChecking::groupChecks(
struct StencilDecomposition {
int64_t Constant = 0;
/// Map from loop-invariant stride SCEV to its integer coefficient.
- SmallDenseMap<const SCEV *, int64_t, 4> Coefficients;
+ SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
};
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
@@ -946,7 +946,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
- SmallDenseSet<const SCEV *, 4> LocalStridesNeedingPreds;
+ SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
for (unsigned Idx : AllMembers) {
const SCEV *PtrStart = Pointers[Idx].Start;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
similarity index 100%
rename from llvm/test/Analysis/LoopAccessAnalysis/affine-group-merging.ll
rename to llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
>From 65315f01d9dc0c7ae3116ac9388d50c720991c50 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 09:22:37 +0100
Subject: [PATCH 03/11] Address some comments (nit related and about the flag)
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 193 ++++++++----------
.../runtime-check-group-merging.ll | 174 ++++++++--------
2 files changed, 175 insertions(+), 192 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9eba76ede932b..7e967266d7f81 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,15 +100,15 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable merging of runtime check groups with stencil stride "
- "patterns (default = false)"),
+static cl::opt<bool> ForceStencilMerge(
+ "force-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Force merging of runtime check groups with stencil stride "
+ "patterns regardless of check count (default = false)"),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
- cl::desc("Trigger stencil group merging when runtime check count "
+ cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128)"),
cl::init(128));
@@ -763,8 +763,8 @@ struct StencilDecomposition {
/// Try to decompose \p Expr into a stencil offset function of loop-invariant
/// strides: C + a1*s1 + a2*s2 + ...
-/// Relies on SCEV's canonical form: AddExpr operands are flattened,
-/// MulExpr has the constant operand first.
+/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
+/// MulExpr has the constant operand first when present.
/// Returns std::nullopt if the expression contains non-stencil terms.
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
@@ -773,21 +773,20 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
- Terms.append(Add->operands().begin(), Add->operands().end());
+ append_range(Terms, Add->operands());
else
Terms.push_back(Expr);
+ const SCEVConstant *C;
+ const SCEV *Stride;
for (const SCEV *Term : Terms) {
- if (auto *C = dyn_cast<SCEVConstant>(Term)) {
+ if (match(Term, m_SCEVConstant(C))) {
D.Constant += C->getAPInt().getSExtValue();
- } else if (auto *Mul = dyn_cast<SCEVMulExpr>(Term)) {
- // SCEV canonical form: constant is operand 0 in a MulExpr.
- if (Mul->getNumOperands() != 2)
+ } else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
+ // Canonical 2-operand pattern (constant * loop-invariant).
+ if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto *C = dyn_cast<SCEVConstant>(Mul->getOperand(0));
- if (!C || !SE.isLoopInvariant(Mul->getOperand(1), &L))
- return std::nullopt;
- D.Coefficients[Mul->getOperand(1)] += C->getAPInt().getSExtValue();
+ D.Coefficients[Stride] += C->getAPInt().getSExtValue();
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
@@ -805,20 +804,30 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
- // Count the total checks from current grouping.
- unsigned TotalChecks = 0;
- for (unsigned I = 0; I < CheckingGroups.size(); ++I)
- for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
- if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
- ++TotalChecks;
-
- if (!EnableStencilMerge && TotalChecks <= StencilMergeCheckThreshold) {
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks <= threshold, skipping stencil merge\n");
- return;
+ // Stencil merging runs when either:
+ // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
+ // - the current check count exceeds the auto-trigger threshold, where the
+ // vectorizer would otherwise reject the loop for having too many runtime
+ // checks. In that case the merge can only improve things: at worst we
+ // decline to merge and behave as before.
+ if (!ForceStencilMerge) {
+ unsigned TotalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I)
+ for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
+ if (needsChecking(CheckingGroups[I], CheckingGroups[J]))
+ ++TotalChecks;
+
+ if (TotalChecks <= StencilMergeCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
+ << " checks <= threshold, skipping stencil merge\n");
+ return;
+ }
+ LLVM_DEBUG(
+ dbgs() << "LAA: " << TotalChecks
+ << " checks > threshold, proceeding with stencil merge\n");
+ } else {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge forced via flag\n");
}
- LLVM_DEBUG(dbgs() << "LAA: " << TotalChecks
- << " checks, proceeding with stencil merge\n");
// Group CheckingGroups by (DependencySetId, AliasSetId) pair.
// DependencySetId alone is not unique: it resets per alias set, so
@@ -844,47 +853,34 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// Collect all member pointers across these groups.
SmallVector<unsigned, 8> AllMembers;
for (unsigned GI : GroupIndices)
- AllMembers.append(CheckingGroups[GI].Members.begin(),
- CheckingGroups[GI].Members.end());
-
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
- bool HasPredicatedAccess = false;
- for (unsigned Idx : AllMembers) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- if (auto *I = dyn_cast<Instruction>(PtrVal)) {
- if (LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT())) {
- HasPredicatedAccess = true;
- break;
- }
- }
- }
- if (HasPredicatedAccess) {
- LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
- << ") with predicated access\n");
- continue;
- }
+ append_range(AllMembers, CheckingGroups[GI].Members);
// Only merge read-only groups. Stencil patterns read an array at
// multiple offsets and write to a different array (different DepSet).
// Mixing reads and writes within a merged group complicates the cost
// model and doesn't match known stencil patterns.
- bool HasWriteAccess = false;
- for (unsigned Idx : AllMembers) {
- if (Pointers[Idx].IsWritePtr) {
- HasWriteAccess = true;
- break;
- }
- }
- if (HasWriteAccess) {
+ if (any_of(AllMembers,
+ [&](unsigned Idx) { return Pointers[Idx].IsWritePtr; })) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with write access\n");
continue;
}
+ // Skip groups with predicated accesses. For conditional loads/stores
+ // (blocks that do not dominate the loop latch), the SCEV-derived bounds
+ // overapproximate the actually-accessed range. Merging such bounds would
+ // widen the range further and can cause false runtime overlap detection.
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ Value *PtrVal = Pointers[Idx].PointerValue;
+ auto *I = dyn_cast<Instruction>(PtrVal);
+ return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
+ << ") with predicated access\n");
+ continue;
+ }
+
LLVM_DEBUG(dbgs() << "LAA: Analyzing DepSet(" << DepId << "," << ASId
<< ") with " << AllMembers.size() << " members, base: "
<< *Pointers[AllMembers[0]].Start << "\n");
@@ -909,59 +905,49 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
// ranges and testing for zero, which lets SCEV simplify algebraically
// even when the individual range SCEVs aren't pointer-identical.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
- bool RangeMismatch = false;
- for (unsigned Idx : AllMembers) {
- const SCEV *Range =
- SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- if (Range != BaseRange && !SE->getMinusSCEV(Range, BaseRange)->isZero()) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different access range, skipping DepSet\n");
- RangeMismatch = true;
- break;
- }
- }
- if (RangeMismatch)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Range =
+ SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
+ return Range != BaseRange &&
+ !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
+ "skipping DepSet\n");
continue;
-
- bool DecompOk = true;
+ }
// Verify all members have the same recurrence step w.r.t. the analyzed
// loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
// correct when every member's High-Low range equals BaseHigh - BaseLow.
// Different steps (e.g., 8 vs 16) produce different ranges.
- for (unsigned Idx : AllMembers) {
- const SCEV *Step = nullptr;
- if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
- if (AR->getLoop() == &L)
- Step = AR->getStepRecurrence(*SE);
-
- if (Step != BaseStep) {
- LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
- << " has different step, skipping DepSet\n");
- DecompOk = false;
- break;
- }
- }
- if (!DecompOk)
+ if (any_of(AllMembers, [&](unsigned Idx) {
+ const SCEV *Step = nullptr;
+ if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
+ if (AR->getLoop() == &L)
+ Step = AR->getStepRecurrence(*SE);
+ return Step != BaseStep;
+ })) {
+ LLVM_DEBUG(dbgs() << "LAA: Member with different step, "
+ "skipping DepSet\n");
continue;
+ }
SmallDenseMap<const SCEV *, int64_t, 4> MinCoeff, MaxCoeff;
int64_t CMin = 0, CMax = 0;
SmallSetVector<const SCEV *, 4> LocalStridesNeedingPreds;
- for (unsigned Idx : AllMembers) {
- const SCEV *PtrStart = Pointers[Idx].Start;
-
- const SCEV *LowOffset = SE->getMinusSCEV(PtrStart, BaseLow);
- if (isa<SCEVCouldNotCompute>(LowOffset)) {
- DecompOk = false;
- break;
- }
+ // Decompose one member's offset (relative to BaseLow) and fold its
+ // constant and per-stride coefficients into the running bounding box.
+ // Returns false if the offset is not in stencil form (so the whole
+ // DepSet is skipped).
+ const auto AccumulateOffset = [&](unsigned Idx) -> bool {
+ const SCEV *LowOffset = SE->getMinusSCEV(Pointers[Idx].Start, BaseLow);
+ if (isa<SCEVCouldNotCompute>(LowOffset))
+ return false;
auto DLow = decomposeStencilOffset(LowOffset, *SE, L);
if (!DLow) {
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx
<< " NOT decomposable: " << *LowOffset << "\n");
- DecompOk = false;
- break;
+ return false;
}
CMin = std::min(CMin, DLow->Constant);
@@ -983,9 +969,10 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
LLVM_DEBUG(dbgs() << "LAA: Member " << Idx << ": C=" << DLow->Constant
<< ", strides=" << DLow->Coefficients.size() << "\n");
- }
+ return true;
+ };
- if (!DecompOk)
+ if (!all_of(AllMembers, AccumulateOffset))
continue;
// The base member (offset = 0) implicitly has coefficient 0 for every
@@ -1077,8 +1064,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
RuntimeCheckingPtrGroup CandidateGroup(AllMembers[0], *this);
CandidateGroup.Low = MergedLow;
CandidateGroup.High = MergedHigh;
- for (unsigned I = 1; I < AllMembers.size(); ++I)
- CandidateGroup.Members.push_back(AllMembers[I]);
+ append_range(CandidateGroup.Members, drop_begin(AllMembers));
for (unsigned GI : GroupIndices)
CandidateGroup.NeedsFreeze |= CheckingGroups[GI].NeedsFreeze;
@@ -1093,8 +1079,7 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
<< *Stride << "\n");
}
- for (unsigned GI : GroupIndices)
- MergedGroupIndices.insert(GI);
+ MergedGroupIndices.insert(GroupIndices.begin(), GroupIndices.end());
NewMergedGroups.push_back(std::move(CandidateGroup));
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index fb6b116a5b300..c6b9507d84752 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -enable-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -18,12 +18,12 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -52,32 +52,32 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 5:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -105,6 +105,7 @@ define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -112,29 +113,28 @@ entry:
loop:
%iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
%neg3cdj = mul nsw i64 %cdj, -3
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %neg3cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
%v0 = load double, ptr %p0, align 8
%neg2cdj = mul nsw i64 %cdj, -2
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v1 = load double, ptr %p1, align 8
%negcdj = sub nsw i64 0, %cdj
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v2 = load double, ptr %p2, align 8
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v3 = load double, ptr %p3, align 8
%pos2cdj = mul nsw i64 %cdj, 2
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %pos2cdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
%v4 = load double, ptr %p4, align 8
%pos3cdj = mul nsw i64 %cdj, 3
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %pos3cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -172,11 +172,11 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -205,27 +205,27 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP5:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
@@ -250,6 +250,7 @@ define void @stencil_merge_two_strides(ptr %vol, ptr %out, i64 %n, i64 %cdj, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 8
br i1 %cmp, label %loop, label %exit
@@ -257,19 +258,18 @@ entry:
loop:
%iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %vol, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; +8 + cdj - cdk
%off0a = add nsw i64 8, %cdj
%negcdk = sub nsw i64 0, %cdk
%off0 = add nsw i64 %off0a, %negcdk
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %off0
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %off0
%v0 = load double, ptr %p0, align 8
; -16 - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
%off1 = add nsw i64 -16, %neg2cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %off1
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %off1
%v1 = load double, ptr %p1, align 8
; +24 + 3*cdj + 2*cdk
@@ -277,7 +277,7 @@ loop:
%pos2cdk = mul nsw i64 %cdk, 2
%off2a = add nsw i64 24, %pos3cdj
%off2 = add nsw i64 %off2a, %pos2cdk
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %off2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %off2
%v2 = load double, ptr %p2, align 8
; -8 + 2*cdj - 3*cdk
@@ -285,14 +285,14 @@ loop:
%neg3cdk = mul nsw i64 %cdk, -3
%off3a = add nsw i64 -8, %pos2cdj
%off3 = add nsw i64 %off3a, %neg3cdk
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 %off3
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %off3
%v3 = load double, ptr %p3, align 8
; +32 - 4*cdj + cdk
%neg4cdj = mul nsw i64 %cdj, -4
%off4a = add nsw i64 32, %neg4cdj
%off4 = add nsw i64 %off4a, %cdk
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %off4
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %off4
%v4 = load double, ptr %p4, align 8
%s0 = fadd double %v0, %v1
@@ -431,7 +431,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -462,7 +462,7 @@ define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -486,9 +486,8 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v0 = load double, ptr %p0, align 8
%v1 = load double, ptr %base, align 8
@@ -629,8 +628,8 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
@@ -657,12 +656,12 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -686,6 +685,7 @@ define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj)
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -693,19 +693,18 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base + 0 (no stride offset -- this is the base member)
%v0 = load double, ptr %base, align 8
; load at base - cdj
%negcdj = sub nsw i64 0, %cdj
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v1 = load double, ptr %p1, align 8
; load at base - 2*cdj
%neg2cdj = mul nsw i64 %cdj, -2
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %neg2cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
@@ -1019,12 +1018,12 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
-; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
-; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1053,20 +1052,20 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP2:
-; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Check 2:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
-; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1088,6 +1087,7 @@ define void @stencil_merge_constant_and_runtime_stride(ptr %a, ptr %out, i64 %n,
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 4
br i1 %cmp, label %loop, label %exit
@@ -1095,31 +1095,30 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; -16 (constant)
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 -16
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
%v0 = load double, ptr %p0, align 8
; -8 (constant)
- %p1 = getelementptr inbounds i8, ptr %base.i8, i64 -8
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
%v1 = load double, ptr %p1, align 8
; +8 (constant)
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 8
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
%v2 = load double, ptr %p2, align 8
; +16 (constant)
- %p3 = getelementptr inbounds i8, ptr %base.i8, i64 16
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
%v3 = load double, ptr %p3, align 8
; -cdj (runtime)
%negcdj = sub nsw i64 0, %cdj
- %p4 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v4 = load double, ptr %p4, align 8
; +cdj (runtime)
- %p5 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v5 = load double, ptr %p5, align 8
%s0 = fadd double %v0, %v1
@@ -1157,16 +1156,16 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; MERGE-NEXT: Check 1:
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP2:
-; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
-; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1197,7 +1196,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1207,12 +1206,12 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
; NOMERGE-NEXT: Check 3:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP4:
-; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
; NOMERGE-NEXT: Check 4:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1222,7 +1221,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP6:
-; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
@@ -1250,6 +1249,7 @@ define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cmp = icmp sgt i64 %n, 6
br i1 %cmp, label %loop, label %exit
@@ -1259,21 +1259,19 @@ loop:
; Reads from %a at {-cdj, 0, +cdj}
%basea = getelementptr inbounds double, ptr %a, i64 %iv
- %basea.i8 = bitcast ptr %basea to ptr
%negcdj = sub nsw i64 0, %cdj
- %pa0 = getelementptr inbounds i8, ptr %basea.i8, i64 %negcdj
+ %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
%va0 = load double, ptr %pa0, align 8
%va1 = load double, ptr %basea, align 8
- %pa2 = getelementptr inbounds i8, ptr %basea.i8, i64 %cdj
+ %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
%va2 = load double, ptr %pa2, align 8
; Reads from %b at {-cdj, 0, +cdj}
%baseb = getelementptr inbounds double, ptr %b, i64 %iv
- %baseb.i8 = bitcast ptr %baseb to ptr
- %pb0 = getelementptr inbounds i8, ptr %baseb.i8, i64 %negcdj
+ %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
%vb0 = load double, ptr %pb0, align 8
%vb1 = load double, ptr %baseb, align 8
- %pb2 = getelementptr inbounds i8, ptr %baseb.i8, i64 %cdj
+ %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
%vb2 = load double, ptr %pb2, align 8
%s0 = fadd double %va0, %va1
@@ -1310,9 +1308,9 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; MERGE-NEXT: Against group GRP1:
-; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1337,7 +1335,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP1:
-; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
; NOMERGE-NEXT: Check 1:
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
@@ -1347,7 +1345,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
; NOMERGE-NEXT: Against group GRP3:
-; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
@@ -1366,6 +1364,7 @@ define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %c
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
+;
entry:
%cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
%cmp = icmp sgt i64 %n, 4
@@ -1374,18 +1373,17 @@ entry:
loop:
%iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
%base = getelementptr inbounds double, ptr %a, i64 %iv
- %base.i8 = bitcast ptr %base to ptr
; load at base - stride
%negcdj = sub nsw i64 0, %cdj
- %p0 = getelementptr inbounds i8, ptr %base.i8, i64 %negcdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
%v0 = load double, ptr %p0, align 8
; load at base
%v1 = load double, ptr %base, align 8
; load at base + stride
- %p2 = getelementptr inbounds i8, ptr %base.i8, i64 %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
%v2 = load double, ptr %p2, align 8
%s0 = fadd double %v0, %v1
>From 02d96a74d5d46f5348245104500a84344ea33091 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 13:36:21 +0100
Subject: [PATCH 04/11] Address int64_t comment
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 7e967266d7f81..0bbd2219d921e 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -765,11 +765,20 @@ struct StencilDecomposition {
/// strides: C + a1*s1 + a2*s2 + ...
/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
/// MulExpr has the constant operand first when present.
-/// Returns std::nullopt if the expression contains non-stencil terms.
+/// Returns std::nullopt if the expression contains non-stencil terms or any
+/// SCEV constant doesn't fit in int64_t (we commit to the signed
+/// interpretation; values that need more than 64 significant bits are
+/// out of scope).
static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
+ if (V.getSignificantBits() > 64)
+ return std::nullopt;
+ return V.getSExtValue();
+ };
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -781,12 +790,18 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- D.Constant += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- D.Coefficients[Stride] += C->getAPInt().getSExtValue();
+ auto V = ToInt64(C->getAPInt());
+ if (!V)
+ return std::nullopt;
+ D.Coefficients[Stride] += *V;
} else if (SE.isLoopInvariant(Term, &L)) {
D.Coefficients[Term] += 1;
} else {
>From e82b70310056a83993b64b69053290b29b76d321 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:35:25 +0100
Subject: [PATCH 05/11] Add 128-addrspace test and replaced lambda with
existing trySExtValue
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 10 +-
.../runtime-check-group-merging-i128.ll | 117 ++++++++++++++++++
2 files changed, 119 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 0bbd2219d921e..e463237249856 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -773,12 +773,6 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
- auto ToInt64 = [](const APInt &V) -> std::optional<int64_t> {
- if (V.getSignificantBits() > 64)
- return std::nullopt;
- return V.getSExtValue();
- };
-
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -790,7 +784,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
const SCEV *Stride;
for (const SCEV *Term : Terms) {
if (match(Term, m_SCEVConstant(C))) {
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Constant += *V;
@@ -798,7 +792,7 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
// Canonical 2-operand pattern (constant * loop-invariant).
if (!SE.isLoopInvariant(Stride, &L))
return std::nullopt;
- auto V = ToInt64(C->getAPInt());
+ auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
new file mode 100644
index 0000000000000..dfb56233e91d5
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+
+target datalayout = "e-p:128:128"
+
+define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj) {
+; MERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_wide_index_huge_coeff'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %ivx = zext i64 %iv to i128
+
+ %o0 = mul i128 %cdj, 1
+ %t0 = add i128 %o0, %ivx
+ %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
+ %v0 = load i8, ptr %p0
+
+ %o1 = mul i128 %cdj, 2
+ %t1 = add i128 %o1, %ivx
+ %p1 = getelementptr inbounds i8, ptr %a, i128 %t1
+ %v1 = load i8, ptr %p1
+
+ %o2 = mul i128 %cdj, 36893488147419103232 ; 2^65, needs 67 significant bits
+ %t2 = add i128 %o2, %ivx
+ %p2 = getelementptr inbounds i8, ptr %a, i128 %t2
+ %v2 = load i8, ptr %p2
+
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %op = getelementptr inbounds i8, ptr %out, i128 %ivx
+ store i8 %s, ptr %op
+
+ %iv.next = add i64 %iv, 1
+ %c = icmp slt i64 %iv.next, %n
+ br i1 %c, label %loop, label %exit
+exit:
+ ret void
+}
>From 7079151080d2afafae6bd6e8934e72391ac56aba Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 13 May 2026 16:50:11 +0100
Subject: [PATCH 06/11] Add -enable-stencil-runtime-check-merge switch
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e463237249856..e3eb311c1ff77 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,16 +100,26 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
+static cl::opt<bool> EnableStencilMerge(
+ "enable-stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Enable stencil-pattern merging of runtime memory checks "
+ "(default = false, intended to be flipped to true in a "
+ "follow-up patch once the algorithm has settled)"),
+ cl::init(false));
+
static cl::opt<bool> ForceStencilMerge(
"force-stencil-runtime-check-merge", cl::Hidden,
cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false)"),
+ "patterns regardless of check count (default = false). "
+ "Implies enabling the merge regardless of "
+ "-enable-stencil-runtime-check-merge."),
cl::init(false));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
- "exceeds this threshold (default = 128)"),
+ "exceeds this threshold (default = 128). Only used when "
+ "-enable-stencil-runtime-check-merge is true."),
cl::init(128));
/// Maximum SIMD width.
@@ -814,12 +824,17 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the user opted in explicitly (-force-stencil-runtime-check-merge), or
- // - the current check count exceeds the auto-trigger threshold, where the
+ // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
+ // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
if (!ForceStencilMerge) {
+ if (!EnableStencilMerge) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
+ return;
+ }
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 3db292bdd0b5f193f32c87859e1da7bce6920270 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:20:36 +0000
Subject: [PATCH 07/11] Fix test
---
.../runtime-check-group-merging.ll | 79 +++++++++----------
1 file changed, 36 insertions(+), 43 deletions(-)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index c6b9507d84752..822be3812b3a6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -507,41 +507,38 @@ exit:
}
-;; Test 5: Invariant + strided accesses to same base -> different steps.
-;; A strided store {%a,+,8} and an invariant store to %a have different
-;; recurrence steps (8 vs 0), so merging must NOT combine them.
+;; Test 5: Invariant + strided reads from same base -> different access ranges.
+;; A strided read {%a,+,8} and an invariant read from %a have different
+;; access ranges (8*n vs 8), so merging must NOT combine them.
+;; Both reads are in the same DepSet (same underlying object, both reads),
+;; so they pass the write-access guard and reach the access-range check.
;; Same result with and without flag: 2 checks, 3 separate groups.
define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
; MERGE-LABEL: 'different_steps_no_merge'
; MERGE-NEXT: loop:
-; MERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; MERGE-NEXT: Unknown data dependence.
+; MERGE-NEXT: Memory dependences are safe with run-time checks
; MERGE-NEXT: Dependences:
-; MERGE-NEXT: Unknown:
-; MERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; MERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; MERGE-EMPTY:
; MERGE-NEXT: Run-time memory checks:
; MERGE-NEXT: Check 0:
; MERGE-NEXT: Comparing group GRP0:
-; MERGE-NEXT: ptr %a
-; MERGE-NEXT: Against group GRP2:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; MERGE-NEXT: Check 1:
-; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: Against group GRP1:
; MERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
; MERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %a
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %a High: (8 + %a))
-; MERGE-NEXT: Member: %a
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; MERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: %a
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; MERGE-NEXT: SCEV assumptions:
@@ -550,43 +547,38 @@ define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
;
; NOMERGE-LABEL: 'different_steps_no_merge'
; NOMERGE-NEXT: loop:
-; NOMERGE-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; NOMERGE-NEXT: Unknown data dependence.
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
; NOMERGE-NEXT: Dependences:
-; NOMERGE-NEXT: Unknown:
-; NOMERGE-NEXT: store double 1.000000e+00, ptr %gep.a, align 8 ->
-; NOMERGE-NEXT: store double 2.000000e+00, ptr %a, align 8
-; NOMERGE-EMPTY:
; NOMERGE-NEXT: Run-time memory checks:
; NOMERGE-NEXT: Check 0:
; NOMERGE-NEXT: Comparing group GRP0:
-; NOMERGE-NEXT: ptr %a
-; NOMERGE-NEXT: Against group GRP2:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
-; NOMERGE-NEXT: Check 1:
-; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: Against group GRP1:
; NOMERGE-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
-; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
; NOMERGE-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %a
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %a High: (8 + %a))
-; NOMERGE-NEXT: Member: %a
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
; NOMERGE-NEXT: Member: {%a,+,8}<nuw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
-; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: %a
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
; NOMERGE-NEXT: SCEV assumptions:
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Expressions re-written:
;
-; GRP0: invariant store to %a (single address, step=0):
-; GRP1: strided store to %a (step=8, different range from GRP0):
-; GRP2: load from %out:
+; GRP0: store to %out (write, different DepSet):
+; GRP1: strided read from %a (step=8):
+; GRP2: invariant read from %a (step=0, different from GRP1):
entry:
%cmp = icmp sgt i64 %n, 2
br i1 %cmp, label %loop, label %exit
@@ -594,16 +586,17 @@ entry:
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- ; Strided store: {%a,+,8}
- %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
- store double 1.0, ptr %gep.a, align 8
+ ; Invariant read from %a (step = 0, different from strided)
+ %v.inv = load double, ptr %a, align 8
- ; Invariant store to %a (step = 0, different range)
- store double 2.0, ptr %a, align 8
+ ; Strided read: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ %v.strided = load double, ptr %gep.a, align 8
- ; Read from %out
+ ; Store to %out (different DepSet, triggers runtime checks)
+ %sum = fadd double %v.strided, %v.inv
%gep.out = getelementptr inbounds double, ptr %out, i64 %iv
- %v = load double, ptr %gep.out, align 8
+ store double %sum, ptr %gep.out, align 8
%iv.next = add nuw nsw i64 %iv, 1
%cond = icmp slt i64 %iv.next, %n
>From 27dcf6c3dc36a1bab6c51bc9de00d3e985e803f8 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 29 May 2026 14:40:06 +0000
Subject: [PATCH 08/11] Refactor flags
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 46 ++++++++++++------------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e3eb311c1ff77..dc160d93abf7f 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -100,26 +100,27 @@ static cl::opt<unsigned> MemoryCheckMergeThreshold(
"runtime memory checks. (default = 100)"),
cl::init(100));
-static cl::opt<bool> EnableStencilMerge(
- "enable-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Enable stencil-pattern merging of runtime memory checks "
- "(default = false, intended to be flipped to true in a "
- "follow-up patch once the algorithm has settled)"),
- cl::init(false));
-
-static cl::opt<bool> ForceStencilMerge(
- "force-stencil-runtime-check-merge", cl::Hidden,
- cl::desc("Force merging of runtime check groups with stencil stride "
- "patterns regardless of check count (default = false). "
- "Implies enabling the merge regardless of "
- "-enable-stencil-runtime-check-merge."),
- cl::init(false));
+enum class StencilMergePolicy { Off, Auto, Force };
+
+static cl::opt<StencilMergePolicy> StencilMerge(
+ "stencil-runtime-check-merge", cl::Hidden,
+ cl::desc("Control stencil-pattern merging of runtime memory checks"),
+ cl::init(StencilMergePolicy::Off),
+ cl::values(
+ clEnumValN(StencilMergePolicy::Off, "off",
+ "Disable stencil merge (default)"),
+ clEnumValN(StencilMergePolicy::Auto, "auto",
+ "Enable stencil merge when runtime check count exceeds "
+ "the threshold"),
+ clEnumValN(StencilMergePolicy::Force, "force",
+ "Always attempt stencil merge regardless of check "
+ "count")));
static cl::opt<unsigned> StencilMergeCheckThreshold(
"stencil-merge-check-threshold", cl::Hidden,
cl::desc("Auto-trigger stencil group merging when runtime check count "
"exceeds this threshold (default = 128). Only used when "
- "-enable-stencil-runtime-check-merge is true."),
+ "-stencil-runtime-check-merge is auto."),
cl::init(128));
/// Maximum SIMD width.
@@ -824,17 +825,18 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
return;
// Stencil merging runs when either:
- // - the test/debug flag forces it (-force-stencil-runtime-check-merge), or
- // - the feature is enabled (-enable-stencil-runtime-check-merge) AND the
+ // - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
+ // - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
// current check count exceeds the auto-trigger threshold, where the
// vectorizer would otherwise reject the loop for having too many runtime
// checks. In that case the merge can only improve things: at worst we
// decline to merge and behave as before.
- if (!ForceStencilMerge) {
- if (!EnableStencilMerge) {
- LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled by default flag\n");
- return;
- }
+ if (StencilMerge == StencilMergePolicy::Off) {
+ LLVM_DEBUG(dbgs() << "LAA: stencil merge disabled\n");
+ return;
+ }
+
+ if (StencilMerge == StencilMergePolicy::Auto) {
unsigned TotalChecks = 0;
for (unsigned I = 0; I < CheckingGroups.size(); ++I)
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J)
>From 6e4276388f3d2c5a9e986106cd2f0e92080a99d7 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 3 Jun 2026 15:29:11 +0100
Subject: [PATCH 09/11] Fix predicated-access check, turn some conditions into
asserts, add tests
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 52 ++-
.../runtime-check-group-merging-auto.ll | 150 +++++++++
.../runtime-check-group-merging-i128.ll | 20 +-
.../runtime-check-group-merging.ll | 314 +++++++++++++++++-
4 files changed, 506 insertions(+), 30 deletions(-)
create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index dc160d93abf7f..2db41410d28a2 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -784,6 +784,11 @@ static std::optional<StencilDecomposition>
decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
StencilDecomposition D;
+ // The only caller passes a difference of two pointer Starts, and a Start is
+ // always loop-invariant (getStartAndEndForAccess asserts it). So Expr is
+ // loop-invariant and every term below is loop-invariant too.
+ assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
+
// Collect top-level additive terms.
SmallVector<const SCEV *, 4> Terms;
if (auto *Add = dyn_cast<SCEVAddExpr>(Expr))
@@ -800,17 +805,14 @@ decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
return std::nullopt;
D.Constant += *V;
} else if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Stride)))) {
- // Canonical 2-operand pattern (constant * loop-invariant).
- if (!SE.isLoopInvariant(Stride, &L))
- return std::nullopt;
+ assert(SE.isLoopInvariant(Stride, &L) && "stride must be loop-invariant");
auto V = C->getAPInt().trySExtValue();
if (!V)
return std::nullopt;
D.Coefficients[Stride] += *V;
- } else if (SE.isLoopInvariant(Term, &L)) {
- D.Coefficients[Term] += 1;
} else {
- return std::nullopt;
+ assert(SE.isLoopInvariant(Term, &L) && "term must be loop-invariant");
+ D.Coefficients[Term] += 1;
}
}
return D;
@@ -892,15 +894,26 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Skip groups with predicated accesses. For conditional loads/stores
- // (blocks that do not dominate the loop latch), the SCEV-derived bounds
- // overapproximate the actually-accessed range. Merging such bounds would
- // widen the range further and can cause false runtime overlap detection.
+ // We do not allow predicated accesses. They may result in overestimation
+ // of the boundaries. Imagine a stencil access where we must skip some first
+ // or last iterations because the stencil does not fit the array and has to
+ // go from 1..N-2 although the array is [0..N-1] (for example the dilate
+ // kernel from llvm-test-suite ImageProcessing/Dilate, which reads the
+ // neighbours of every pixel and guards the borders with conditions).
+ // Merging such bounds would widen the already overestimated range further.
+ // This is not necessary in StencilMergePolicy::Auto mode, but skipping it
+ // in StencilMergePolicy::Force mode causes a regression on that benchmark.
+ //
+ // Look at the block of the actual load/store, not of the pointer: a
+ // loop-invariant address is computed in the preheader, outside the loop.
if (any_of(AllMembers, [&](unsigned Idx) {
- Value *PtrVal = Pointers[Idx].PointerValue;
- auto *I = dyn_cast<Instruction>(PtrVal);
- return I && LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
- DC.getDT());
+ const PointerInfo &P = Pointers[Idx];
+ return any_of(
+ DC.getInstructionsForAccess(P.PointerValue, P.IsWritePtr),
+ [&](Instruction *I) {
+ return LoopAccessInfo::blockNeedsPredication(I->getParent(), &L,
+ DC.getDT());
+ });
})) {
LLVM_DEBUG(dbgs() << "LAA: Skipping DepSet(" << DepId << "," << ASId
<< ") with predicated access\n");
@@ -942,10 +955,13 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
}
- // Verify all members have the same recurrence step w.r.t. the analyzed
- // loop. MergedHigh is computed as BaseHigh + max_offsets, which is only
- // correct when every member's High-Low range equals BaseHigh - BaseLow.
- // Different steps (e.g., 8 vs 16) produce different ranges.
+ // Require all members to have the same recurrence step. Equal ranges
+ // (checked above) are what the merged bounds actually need, and a different
+ // step usually means a different range. But ranges can be equal by accident
+ // - e.g. an invariant access whose range matches the stride, or a loop with
+ // a single iteration. The base member is picked arbitrarily, so together
+ // with the BaseStep check above this keeps the decision the same no matter
+ // which member comes first: we only merge recurrences with one common step.
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Step = nullptr;
if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Pointers[Idx].Expr))
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
new file mode 100644
index 0000000000000..6855e80d4cffa
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-auto.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=auto -stencil-merge-check-threshold=1000 -disable-output %s 2>&1 | FileCheck --check-prefix=AUTOSKIP %s
+
+define void @stencil_auto_threshold(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; AUTOMERGE-LABEL: 'stencil_auto_threshold'
+; AUTOMERGE-NEXT: loop:
+; AUTOMERGE-NEXT: Memory dependences are safe with run-time checks
+; AUTOMERGE-NEXT: Dependences:
+; AUTOMERGE-NEXT: Run-time memory checks:
+; AUTOMERGE-NEXT: Check 0:
+; AUTOMERGE-NEXT: Comparing group GRP0:
+; AUTOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOMERGE-NEXT: Against group GRP1:
+; AUTOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOMERGE-NEXT: Grouped accesses:
+; AUTOMERGE-NEXT: Group GRP0:
+; AUTOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOMERGE-NEXT: Group GRP1:
+; AUTOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOMERGE-NEXT: SCEV assumptions:
+; AUTOMERGE-NEXT: Compare predicate: %cdj sgt) 0
+; AUTOMERGE-EMPTY:
+; AUTOMERGE-NEXT: Expressions re-written:
+;
+; AUTOSKIP-LABEL: 'stencil_auto_threshold'
+; AUTOSKIP-NEXT: loop:
+; AUTOSKIP-NEXT: Memory dependences are safe with run-time checks
+; AUTOSKIP-NEXT: Dependences:
+; AUTOSKIP-NEXT: Run-time memory checks:
+; AUTOSKIP-NEXT: Check 0:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP1:
+; AUTOSKIP-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; AUTOSKIP-NEXT: Check 1:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP2:
+; AUTOSKIP-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; AUTOSKIP-NEXT: Check 2:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP3:
+; AUTOSKIP-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; AUTOSKIP-NEXT: Check 3:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP4:
+; AUTOSKIP-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; AUTOSKIP-NEXT: Check 4:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP5:
+; AUTOSKIP-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; AUTOSKIP-NEXT: Check 5:
+; AUTOSKIP-NEXT: Comparing group GRP0:
+; AUTOSKIP-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; AUTOSKIP-NEXT: Against group GRP6:
+; AUTOSKIP-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; AUTOSKIP-NEXT: Grouped accesses:
+; AUTOSKIP-NEXT: Group GRP0:
+; AUTOSKIP-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; AUTOSKIP-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; AUTOSKIP-NEXT: Group GRP1:
+; AUTOSKIP-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP2:
+; AUTOSKIP-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP3:
+; AUTOSKIP-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; AUTOSKIP-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP4:
+; AUTOSKIP-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP5:
+; AUTOSKIP-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-NEXT: Group GRP6:
+; AUTOSKIP-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; AUTOSKIP-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; AUTOSKIP-NEXT: SCEV assumptions:
+; AUTOSKIP-EMPTY:
+; AUTOSKIP-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
index dfb56233e91d5..0799360bf346e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging-i128.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
target datalayout = "e-p:128:128"
@@ -27,16 +27,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; MERGE-NEXT: Grouped accesses:
; MERGE-NEXT: Group GRP0:
-; MERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; MERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; MERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; MERGE-NEXT: Group GRP1:
-; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; MERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP2:
-; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; MERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; MERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; MERGE-NEXT: Group GRP3:
-; MERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; MERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; MERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; MERGE-EMPTY:
; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -66,16 +66,16 @@ define void @stencil_wide_index_huge_coeff(ptr %a, ptr %out, i64 %n, i128 %cdj)
; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %a, i128 %t0
; NOMERGE-NEXT: Grouped accesses:
; NOMERGE-NEXT: Group GRP0:
-; NOMERGE-NEXT: (Low: %out High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %out))
+; NOMERGE-NEXT: (Low: %out High: ((zext i64 (1 smax %n) to i128) + %out))
; NOMERGE-NEXT: Member: {%out,+,1}<nuw><%loop>
; NOMERGE-NEXT: Group GRP1:
-; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (36893488147419103232 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((36893488147419103232 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (36893488147419103232 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((36893488147419103232 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP2:
-; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + (2 * %cdj) + %a))
+; NOMERGE-NEXT: (Low: ((2 * %cdj) + %a) High: ((zext i64 (1 smax %n) to i128) + (2 * %cdj) + %a))
; NOMERGE-NEXT: Member: {((2 * %cdj) + %a),+,1}<nw><%loop>
; NOMERGE-NEXT: Group GRP3:
-; NOMERGE-NEXT: (Low: (%cdj + %a) High: (1 + (zext i64 (-1 + (1 smax %n))<nsw> to i128) + %cdj + %a))
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: ((zext i64 (1 smax %n) to i128) + %cdj + %a))
; NOMERGE-NEXT: Member: {(%cdj + %a),+,1}<nw><%loop>
; NOMERGE-EMPTY:
; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 822be3812b3a6..3ef1bd18a3ab2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
-; RUN: opt -passes='print<access-info>' -force-stencil-runtime-check-merge=false -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefix=MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefix=NOMERGE %s
;; Test 1: Basic stencil merge with one runtime stride.
;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
@@ -1394,4 +1394,314 @@ exit:
ret void
}
+;; Test 12: loop-invariant pointer is computed in the preheader.
+;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
+;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
+;; read, so we reach the predicated-access check. We must look at the load,
+;; which is in the loop, not at the pointer, which is in the preheader -
+;; otherwise we crash.
+define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'invariant_pointer_in_preheader'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (64 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: (%cdj + %a)
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'invariant_pointer_in_preheader'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (64 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: (%cdj + %a)
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,1}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+ %sv = load i8, ptr %sp
+ %sv64 = zext i8 %sv to i64
+ %iv8 = load i64, ptr %inv.ptr
+ %sum = add i64 %sv64, %iv8
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+
+;; Test 13: predicated access whose address is computed in the header.
+;; The stride GEPs are in the header, which dominates the latch, but the loads
+;; are in a conditional block guarded by %c. So the accesses are predicated and
+;; must not be merged. We must look at the load block, which is conditional,
+;; not at the GEP block, which is the header. Both modes: groups stay separate.
+define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c) {
+; MERGE-LABEL: 'predicated_access_hoisted_address'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP4:
+; MERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; MERGE-NEXT: Member: {%out,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; MERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; MERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; MERGE-NEXT: Group GRP4:
+; MERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; MERGE-NEXT: Member: {%a,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'predicated_access_hoisted_address'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: ((8 * %n) + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: %a High: ((8 * %n) + %a))
+; NOMERGE-NEXT: Member: {%a,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+ %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+ %v0 = load i64, ptr %p0
+ %i1 = add i64 %iv, %cdj
+ %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+ %i2 = sub i64 %iv, %cdj
+ %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+ %i3 = add i64 %i1, %cdj
+ %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+ br i1 %c, label %then, label %backedge
+then:
+ %v1 = load i64, ptr %p1
+ %v2 = load i64, ptr %p2
+ %v3 = load i64, ptr %p3
+ %t = add i64 %v1, %v2
+ %u = add i64 %t, %v3
+ br label %backedge
+backedge:
+ %vp = phi i64 [ %u, %then ], [ 0, %loop ]
+ %sum = add i64 %v0, %vp
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+;; Test 14: equal access ranges but different recurrence steps -> no merge.
+;; In a single-iteration loop every access spans just its element size, so the
+;; ranges are equal even though the steps differ (8 vs 16). The access-range
+;; check passes here, so the step check is the guard that stops the merge. Both
+;; modes: groups stay separate.
+define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
+; MERGE-LABEL: 'equal_range_different_step_no_merge'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %out High: (8 + %out))
+; MERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %a High: (8 + %a))
+; MERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; MERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'equal_range_different_step_no_merge'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %out High: (8 + %out))
+; NOMERGE-NEXT: Member: {%out,+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %a High: (8 + %a))
+; NOMERGE-NEXT: Member: {%a,+,16}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; NOMERGE-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %s8 = mul i64 %iv, 8
+ %b0 = getelementptr inbounds i8, ptr %a, i64 %s8
+ %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+ %v0 = load i64, ptr %p0
+ %s16 = mul i64 %iv, 16
+ %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+ %v1 = load i64, ptr %p1
+ %sum = add i64 %v0, %v1
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 2dd83879878d82e589c3ec0fc09f21501018c85f Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 10:41:15 +0000
Subject: [PATCH 10/11] Add SCEVCouldNotCompute test
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 26 ++++--
.../runtime-check-group-merging.ll | 85 +++++++++++++++++++
2 files changed, 103 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 2db41410d28a2..bde88709805c6 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -939,19 +939,29 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
continue;
// Verify all members have the same access range (End - Start).
- // MergedHigh = BaseHigh + max_offsets is only correct when every
- // member's range equals BaseHigh - BaseLow. We check by subtracting
- // ranges and testing for zero, which lets SCEV simplify algebraically
- // even when the individual range SCEVs aren't pointer-identical.
+ // MergedHigh = BaseHigh + max_offsets is only correct when every member's
+ // range equals BaseHigh - BaseLow. Compute the ranges first and compare
+ // them by subtracting, so algebraically equal but non-identical SCEVs still
+ // match. Bail out if any subtraction produces SCEVCouldNotCompute.
const SCEV *BaseRange = SE->getMinusSCEV(BaseHigh, BaseLow);
+ if (isa<SCEVCouldNotCompute>(BaseRange)) {
+ LLVM_DEBUG(dbgs() << "LAA: Base access range not computable, "
+ "skipping DepSet\n");
+ continue;
+ }
if (any_of(AllMembers, [&](unsigned Idx) {
const SCEV *Range =
SE->getMinusSCEV(Pointers[Idx].End, Pointers[Idx].Start);
- return Range != BaseRange &&
- !SE->getMinusSCEV(Range, BaseRange)->isZero();
+ if (isa<SCEVCouldNotCompute>(Range))
+ return true;
+ if (Range == BaseRange)
+ return false;
+ const SCEV *RangeDiff = SE->getMinusSCEV(Range, BaseRange);
+ return isa<SCEVCouldNotCompute>(RangeDiff) || !RangeDiff->isZero();
})) {
- LLVM_DEBUG(dbgs() << "LAA: Member with different access range, "
- "skipping DepSet\n");
+ LLVM_DEBUG(dbgs()
+ << "LAA: Member with different or not computable access "
+ "range, skipping DepSet\n");
continue;
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
index 3ef1bd18a3ab2..fada2d09d0509 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-group-merging.ll
@@ -1704,4 +1704,89 @@ exit:
ret void
}
+;; Test 15: %gep Start/End expressions are min/max expressions.
+;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
+define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
+; MERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: ptr %q
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: ptr %p
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + %q))
+; MERGE-NEXT: Member: %q
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; MERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %p High: (1 + %p))
+; MERGE-NEXT: Member: %p
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_range_could_not_compute'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: ptr %q
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: ptr %p
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + %q))
+; NOMERGE-NEXT: Member: %q
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; NOMERGE-NEXT: Member: {%p,+,(1 + %m)}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: %p High: (1 + %p))
+; NOMERGE-NEXT: Member: %p
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%p,+,(1 + %m)}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %step = add i64 %m, 1
+ br label %loop
+
+loop:
+ %off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
+ %i = phi i1 [ false, %entry ], [ true, %loop ]
+ load i8, ptr %p
+ %gep = getelementptr i8, ptr %p, i64 %off
+ load i8, ptr %gep
+ store i8 0, ptr %q
+ %off.next = add i64 %off, %step
+ br i1 %i, label %done, label %loop
+
+done:
+ ret void
+}
+
declare i64 @llvm.smax.i64(i64, i64)
>From 56491b9cb0d1e03f2c2faf8fe10dce74c03c9863 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 5 Jun 2026 11:11:00 +0000
Subject: [PATCH 11/11] Add high level explanation to mergeStencilGroups like
in groupChecks
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index bde88709805c6..d0c2eb6c8ccb8 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -826,6 +826,28 @@ void RuntimePointerChecking::mergeStencilGroups(PredicatedScalarEvolution &PSE,
if (CheckingGroups.size() < 2)
return;
+ // We try to merge groups produced by groupChecks when their pointers follow
+ // a stencil access pattern. groupChecks intentionally only merges pointers
+ // whose min/max bounds differ by constants, because that keeps each runtime
+ // check precise. Stencil kernels often read the same underlying object at
+ // several loop-invariant stride offsets, so those groups remain separate and
+ // can produce too many checks. Here we trade some precision for fewer
+ // checks by replacing a set of same-dependence, same-alias read groups with
+ // one conservative bounding group.
+ //
+ // We use the following algorithm to construct a merged stencil group:
+ // - collect checking groups that share both DependencySetId and AliasSetId;
+ // - reject groups with writes, predicated accesses, different access
+ // ranges, or different recurrence steps;
+ // - use one member as the base and decompose each other member's offset
+ // from that base as C + sum(Coeff[Stride] * Stride), where Stride is
+ // loop-invariant;
+ // - build one bounding range by taking the min/max constant offset and the
+ // min/max coefficient for each stride, adding predicates for strides
+ // that are not already known positive;
+ // - commit the merge only if the local cost model reduces the number of
+ // checks after accounting for any new predicates.
+
// Stencil merging runs when either:
// - the flag is set to 'force' (-stencil-runtime-check-merge=force), or
// - the flag is set to 'auto' (-stencil-runtime-check-merge=auto) AND the
More information about the llvm-commits
mailing list