[llvm] [LAA] Add stencil group merging to reduce runtime pointer checks (PR #187252)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 09:48:52 PDT 2026
================
@@ -747,6 +781,655 @@ void RuntimePointerChecking::groupChecks(
}
}
+/// Result of decomposing a SCEV expression into stencil offset form:
+/// Offset = Constant + sum(Coefficients[stride] * stride)
+/// where each stride is a loop-invariant SCEV expression.
+struct StencilDecomposition {
+ int64_t Constant = 0;
+ /// Map from loop-invariant stride SCEV to its integer coefficient.
+ SmallMapVector<const SCEV *, int64_t, 4> Coefficients;
+};
+
+/// Recursion cap for addScaledStencilTerm. Depth counts how deep a term
+/// sits inside the offset expression. Example, the offset
+/// 8 + (64 * (s1 + s2 + (4 * s3)))
+/// is visited like this:
+/// depth 0: the whole add
+/// depth 1: its operands 8 and (64 * (s1 + s2 + (4 * s3)))
+/// depth 2: (s1 + s2 + (4 * s3)), the operand of the multiply
+/// depth 3: s1, s2 and (4 * s3), the operands of that add
+/// At depth 3 addScaledStencilTerm stops going deeper. s1 and s2 are plain
+/// strides anyway. (4 * s3) is not split into 4 times s3: it becomes one
+/// stride key as it is, with coefficient 64. The result is Constant = 8
+/// and coefficients {s1: 64, s2: 64, (4 * s3): 64}.
+/// Three levels cover the stencil offsets we care about: a top-level add,
+/// a constant times a sum inside it, and the strides in that sum. A deeper
+/// term is kept whole as one stride key. The merge does not care what is
+/// inside a key. It only needs a loop-invariant value with a
+/// positive-stride predicate, and a whole term has both. The only cost is
+/// precision, when another member uses a part of that term, here s3 alone,
+/// as a key of its own. isNeverAbove sees two unrelated keys, so a member
+/// that is in fact always lower or higher may stay a candidate.
+constexpr unsigned MaxStencilDecomposeDepth = 3;
+
+/// Add one term of a stencil offset to \p D. \p Mult is the factor in
+/// front of the term; the top-level call passes 1.
+/// Example: the offset 8 + (-64 * (s1 + s2)) + (-32 * s1), Mult = 1. It is
+/// an add, so each operand is visited in turn with the same Mult = 1:
+/// 8 a constant: D.Constant += 1 * 8
+/// (-64 * (s1 + s2)) a constant times X: visit X = (s1 + s2) with
+/// Mult = 1 * -64. X is an add, so each operand is
+/// visited with Mult = -64:
+/// s1 a stride: D.Coefficients[s1] += -64
+/// s2 a stride: D.Coefficients[s2] += -64
+/// (-32 * s1) a constant times X: visit X = s1 with Mult = -32:
+/// s1 a stride: D.Coefficients[s1] += -32
+/// Result: Constant = 8, Coefficients {s1: -96, s2: -64}. The -64 and the
+/// -32 for s1 come from two different terms and add up in the map.
+/// So, by the kind of term:
+/// constant K D.Constant += Mult * K
+/// (K * X) visit X with Mult * K
+/// (a + b + ...) visit a, b, ... each with this same Mult
+/// anything else a stride key: D.Coefficients[Term] += Mult
+/// The two recursive cases only fire while Depth is below
+/// MaxStencilDecomposeDepth. At the cap, (K * X) and (a + b + ...) are
+/// stride keys like anything else; that is not a bailout.
+/// Returns false when a constant does not fit in int64_t or an update
+/// overflows. The caller then drops the whole decomposition.
+static bool addScaledStencilTerm(const SCEV *Term, int64_t Mult, unsigned Depth,
+ StencilDecomposition &D) {
+ const SCEVConstant *C;
+ // A constant folds into the running constant at any depth.
+ if (match(Term, m_SCEVConstant(C))) {
+ std::optional<int64_t> V = C->getAPInt().trySExtValue();
+ int64_t Scaled;
+ return V && !MulOverflow(Mult, *V, Scaled) &&
+ !AddOverflow(D.Constant, Scaled, D.Constant);
+ }
+
+ if (Depth < MaxStencilDecomposeDepth) {
+ const SCEV *Inner;
+ if (match(Term, m_scev_Mul(m_SCEVConstant(C), m_SCEV(Inner)))) {
+ std::optional<int64_t> V = C->getAPInt().trySExtValue();
+ int64_t NewMult;
+ return V && !MulOverflow(Mult, *V, NewMult) &&
+ addScaledStencilTerm(Inner, NewMult, Depth + 1, D);
+ }
+ if (auto *Add = dyn_cast<SCEVAddExpr>(Term))
+ return all_of(Add->operands(), [&](const SCEV *Op) {
+ return addScaledStencilTerm(Op, Mult, Depth + 1, D);
+ });
+ }
+
+ // Anything else is one stride key.
+ int64_t &Coeff = D.Coefficients[Term];
+ return !AddOverflow(Coeff, Mult, Coeff);
+}
+
+/// Try to decompose \p Expr into a stencil offset function of loop-invariant
+/// strides: C + a1*s1 + a2*s2 + ...
+/// \p Expr is the difference of two access "Start" SCEVs (Start_member -
+/// Start_base). A "Start" is the low bound of a memory access range as computed
+/// by getStartAndEndForAccess: the address of the first byte the access can
+/// touch. The result describes where one member's range sits relative to the
+/// base member's range.
+/// Constant factors are distributed over sums. SCEV can keep a factored form:
+/// -64*s1 + -64*s2 is stored as (-64 * (s1 + s2)). Distributing the -64 gives
+/// the coefficients {s1: -64, s2: -64}, so every member of a group is keyed
+/// on the same base strides.
+/// Relies on SCEV's canonical form: AddExpr operands are flattened (N-ary),
+/// MulExpr has the constant operand first when present.
+/// Returns std::nullopt if a constant does not fit in int64_t or a multiplier
+/// or coefficient update overflows (we commit to the signed interpretation;
+/// values that need more than 64 significant bits are out of scope).
+static std::optional<StencilDecomposition>
+decomposeStencilOffset(const SCEV *Expr, ScalarEvolution &SE, const Loop &L) {
+ // A "Start" is always loop-invariant (getStartAndEndForAccess asserts it), so
+ // the difference Expr passed in by the caller is loop-invariant too, and so
+ // is every term addScaledStencilTerm visits.
+ assert(SE.isLoopInvariant(Expr, &L) && "expected a loop-invariant offset");
+
+ StencilDecomposition D;
+ if (!addScaledStencilTerm(Expr, /*Mult=*/1, /*Depth=*/0, D))
+ return std::nullopt;
+ return D;
+}
+
+/// Return true if A's address is never higher than B's address.
+/// A and B are these sums:
+/// A = A.Constant + CoefA_1 * stride_1 + CoefA_2 * stride_2 + ...
+/// B = B.Constant + CoefB_1 * stride_1 + CoefB_2 * stride_2 + ...
+/// A stride missing from a member's map has coefficient 0. Every stride
+/// is 1 or more: strides are integers, and the merge adds an "s > 0"
+/// predicate for each stride that is not already known positive.
+/// Example:
+/// A: 0 - 80*s1
+/// B: -40 - 40*s1
+/// At s1 = 1 both are -80. For bigger s1, A goes down faster. So A is
+/// never above B.
+/// The rule checks two things:
+/// 1. CoefA_i <= CoefB_i for every stride. So when a stride grows, B - A
+/// grows too, or stays the same.
+/// 2. B - A >= 0 when every stride is 1. That is ACorner <= BCorner, with
+/// ACorner = A.Constant + the sum of all CoefA_i, same for BCorner.
+/// B - A starts at or above zero and never goes down, so B - A >= 0 for
+/// all stride values.
+/// Offsets are signed and addresses are unsigned, but both members read
+/// one object, and an object does not wrap around the address space, so
+/// the smaller offset is the smaller address.
+/// Returns false when ACorner or BCorner overflows int64_t. The caller
+/// then keeps the member, which is the safe side.
+static bool isNeverAbove(const StencilDecomposition &A,
+ const StencilDecomposition &B) {
+ int64_t ACorner = A.Constant, BCorner = B.Constant;
+ for (const auto &[Stride, ACoeff] : A.Coefficients) {
+ if (ACoeff > B.Coefficients.lookup(Stride))
+ return false;
+ if (AddOverflow(ACorner, ACoeff, ACorner))
+ return false;
+ }
+ for (const auto &[Stride, BCoeff] : B.Coefficients) {
+ if (A.Coefficients.lookup(Stride) > BCoeff)
+ return false;
+ if (AddOverflow(BCorner, BCoeff, BCorner))
+ return false;
+ }
+ return ACorner <= BCorner;
+}
+
+/// Find the members that can define the merged bound on one side.
+/// Example for the minimum side (\p ForMin == true), two members:
+/// A: 0 - 80*s1
+/// B: -40 - 40*s1
+/// For every s1 >= 1, A sits at or below B, so B can never be the lowest
+/// member: A beats B. The members nobody beats are the candidates.
+/// The maximum side works the same way with the comparison flipped.
+/// When two members have equal offsets, only the first one is kept.
+/// Returns indices into \p Offsets.
+/// TODO: Worst case compares every pair of members: O(N^2). Fine for real
+/// stencils.
+static SmallVector<unsigned, 4>
+collectCandidateMembers(ArrayRef<StencilDecomposition> Offsets, bool ForMin) {
+ // A beats B when A always bounds at least as well as B: for the minimum
+ // side A is never above B, for the maximum side A is never below B.
+ auto Beats = [&](unsigned A, unsigned B) {
+ return ForMin ? isNeverAbove(Offsets[A], Offsets[B])
+ : isNeverAbove(Offsets[B], Offsets[A]);
+ };
+ // Skipping a beaten member loses nothing: Beats is transitive, so
+ // whoever beat it also beats anyone it would have beaten.
+ BitVector Beaten(Offsets.size());
+ for (unsigned K = 0; K < Offsets.size(); ++K) {
+ if (Beaten.test(K))
+ continue;
+ for (unsigned J = K + 1; J < Offsets.size(); ++J) {
+ if (Beaten.test(J))
+ continue;
+ // Checking K first settles ties: on equal offsets K survives.
+ if (Beats(K, J)) {
+ Beaten.set(J);
+ } else if (Beats(J, K)) {
+ Beaten.set(K);
+ break;
+ }
+ }
+ }
+ SmallVector<unsigned, 4> Candidates;
+ for (unsigned K = 0; K < Offsets.size(); ++K)
+ if (!Beaten.test(K))
+ Candidates.push_back(K);
+ return Candidates;
+}
+
+/// Local cost model: count the runtime checks required before and after
+/// replacing one DepSet's groups (\p GroupIndices) with the single merged
+/// group. Everything is counted in the same unit, one check, even though a
+/// stride predicate or an extra umin/umax operand is cheaper at runtime
+/// than a full group-pair check. The cheaper items only appear on the
+/// After side, and we merge only when After < Before, so the rounding
+/// always errs toward not merging.
+///
+/// Before = NumGroups * NumExternalChecks, where NumExternalChecks is the
+/// number of groups outside this DepSet that need a check against it. The
+/// product is exact: needsChecking() looks only at (DependencySetId,
+/// AliasSetId) and at whether a group writes, and all groups in this
+/// DepSet agree on those, so an external group is checked against all of
+/// them or against none.
+///
+/// After = NumExternalChecks + NewPredicates + NumBoundOperands:
+/// - the merged group keeps the same IDs, so it is checked against exactly
+/// the same external groups;
+/// - one predicate per stride we must prove positive, unless an earlier
+/// DepSet already paid for it (\p CommittedStridePredicates);
+/// - a umin over k members costs k-1 compare+selects, same for the umax
+/// (\p NumMinCandidates - 1 plus \p NumMaxCandidates - 1). A single
+/// candidate costs nothing: the bound is that member's own address.
+///
+/// Returns {ChecksBefore, ChecksAfter}.
+static std::pair<unsigned, unsigned> computeStencilMergeCost(
+ const RuntimePointerChecking &RtCheck, ArrayRef<unsigned> GroupIndices,
+ const SmallDenseSet<unsigned, 4> &MergedGroupIndices,
+ ArrayRef<const SCEV *> LocalStridesNeedingPreds,
+ const SmallDenseSet<const SCEV *, 4> &CommittedStridePredicates,
+ unsigned NumMinCandidates, unsigned NumMaxCandidates) {
+ ArrayRef<RuntimeCheckingPtrGroup> CheckingGroups = RtCheck.CheckingGroups;
+ unsigned NumGroups = GroupIndices.size();
+ SmallDenseSet<unsigned, 4> GroupIndexSet(GroupIndices.begin(),
+ GroupIndices.end());
+ unsigned NumExternalChecks = 0;
+ for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+ if (GroupIndexSet.contains(I) || MergedGroupIndices.contains(I))
+ continue;
+ for (unsigned GI : GroupIndices) {
+ if (RtCheck.needsChecking(CheckingGroups[GI], CheckingGroups[I])) {
+ ++NumExternalChecks;
+ break;
+ }
+ }
+ }
+
+ // Each not-yet-committed positive-stride predicate becomes one extra runtime
+ // check, so it counts against the saving.
+ unsigned NewPredicates = 0;
+ for (const SCEV *Stride : LocalStridesNeedingPreds)
+ if (!CommittedStridePredicates.contains(Stride))
+ ++NewPredicates;
+
+ // Extra bound operands: one compare-and-select per operand past the first
+ // in the merged umin, and the same for the umax.
+ unsigned NumBoundOperands = (NumMinCandidates - 1) + (NumMaxCandidates - 1);
----------------
david-arm wrote:
You could just pass this argument (`NumBoundOperands`) to `computeStencilMergeCost` instead, which saves an argument.
https://github.com/llvm/llvm-project/pull/187252
More information about the llvm-commits
mailing list