[llvm] [InterleavedLoadCombine] Index candidates to avoid quadratic matching (PR #213053)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 08:54:12 PDT 2026
https://github.com/madhur13490 created https://github.com/llvm/llvm-project/pull/213053
Matching scanned every candidate against every other candidate, which got very slow on functions with many interleaved loads. Index by address offset and look up neighbors instead.
llc's Compile-time drops from 212s to ~2s on a Grace machine.
Fixes #162299.
>From d2267fb322ae37ed08e60ef92f47f4ebbdd06c0a Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Thu, 30 Jul 2026 03:24:24 -0700
Subject: [PATCH] [InterleavedLoadCombine] Index candidates to avoid quadratic
matching
Matching scanned every candidate against every other candidate, which
got very slow on functions with many interleaved loads. Index by
address offset and look up neighbors instead.
Fixes #162299.
---
.../CodeGen/InterleavedLoadCombinePass.cpp | 186 ++++++++++--------
...nterleaved-load-combine-many-candidates.ll | 83 ++++++++
2 files changed, 191 insertions(+), 78 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index aaba0d14994e5..4d75255df1510 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -18,6 +18,10 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
@@ -43,6 +47,7 @@
#include <algorithm>
#include <cassert>
#include <list>
+#include <unordered_map>
using namespace llvm;
@@ -95,14 +100,8 @@ struct InterleavedLoadCombineImpl {
/// Replace interleaved load candidates. It does additional
/// analyses if this makes sense. Returns true on success and false
/// of nothing has been changed.
- bool combine(std::list<VectorInfo> &InterleavedLoad,
+ bool combine(ArrayRef<VectorInfo *> InterleavedLoad,
OptimizationRemarkEmitter &ORE);
-
- /// Given a set of VectorInfo containing candidates for a given interleave
- /// factor, find a set that represents a 'factor' interleaved load.
- bool findPattern(std::list<VectorInfo> &Candidates,
- std::list<VectorInfo> &InterleavedLoad, unsigned Factor,
- const DataLayout &DL);
}; // InterleavedLoadCombine
/// First Order Polynomial on an n-Bit Integer Value
@@ -566,12 +565,26 @@ class Polynomial {
}
/// Returns true if it can be proven that two Polynomials are equal.
- bool isProvenEqualTo(const Polynomial &o) {
+ bool isProvenEqualTo(const Polynomial &o) const {
// Subtract both polynomials and test if it is fully defined and zero.
Polynomial r = *this - o;
return (r.ErrorMSBs == 0) && (!r.isFirstOrder()) && (r.A.isZero());
}
+ /// Returns true if every bit of the polynomial is provably exact. An inexact
+ /// polynomial can never be proven equal to another, so it is never a valid
+ /// match candidate.
+ bool isProvenExact() const { return ErrorMSBs == 0; }
+
+ /// Hash the identity checked by isProvenEqualTo. Only meaningful for exact
+ /// polynomials; two exact, proven-equal polynomials hash identically.
+ friend hash_code hash_value(const Polynomial &P) {
+ hash_code H = hash_combine(P.A.getBitWidth(), P.V);
+ for (const auto &BO : P.B)
+ H = hash_combine(H, BO.first, hash_value(BO.second));
+ return hash_combine(H, hash_value(P.A));
+ }
+
/// Print the polynomial into a stream.
void print(raw_ostream &OS) const {
OS << "[{#ErrBits:" << ErrorMSBs << "} ";
@@ -625,6 +638,29 @@ static raw_ostream &operator<<(raw_ostream &OS, const Polynomial &S) {
}
#endif
+/// Address key of a candidate's first vector element: the block, the common
+/// base pointer, the vector type and the offset polynomial. Two candidates
+/// belong to the same interleaved group iff their keys agree on everything but
+/// the constant offset, so consecutive elements are located by building the
+/// neighbouring keys and looking them up.
+struct OffsetKey {
+ BasicBlock *BB;
+ Value *PV;
+ FixedVectorType *VTy;
+ Polynomial Ofs;
+
+ bool operator==(const OffsetKey &O) const {
+ return BB == O.BB && PV == O.PV && VTy == O.VTy &&
+ Ofs.isProvenEqualTo(O.Ofs);
+ }
+};
+
+struct OffsetKeyHash {
+ size_t operator()(const OffsetKey &K) const {
+ return hash_combine(K.BB, K.PV, K.VTy, hash_value(K.Ofs));
+ }
+};
+
/// VectorInfo stores abstract the following information for each vector
/// element:
///
@@ -1053,54 +1089,6 @@ struct VectorInfo {
} // anonymous namespace
-bool InterleavedLoadCombineImpl::findPattern(
- std::list<VectorInfo> &Candidates, std::list<VectorInfo> &InterleavedLoad,
- unsigned Factor, const DataLayout &DL) {
- for (auto C0 = Candidates.begin(), E0 = Candidates.end(); C0 != E0; ++C0) {
- unsigned i;
- // Try to find an interleaved load using the front of Worklist as first line
- unsigned Size = DL.getTypeAllocSize(C0->VTy->getElementType());
-
- // List containing iterators pointing to the VectorInfos of the candidates
- std::vector<std::list<VectorInfo>::iterator> Res(Factor, Candidates.end());
-
- for (auto C = Candidates.begin(), E = Candidates.end(); C != E; C++) {
- if (C->VTy != C0->VTy)
- continue;
- if (C->BB != C0->BB)
- continue;
- if (C->PV != C0->PV)
- continue;
-
- // Check the current value matches any of factor - 1 remaining lines
- for (i = 1; i < Factor; i++) {
- if (C->EI[0].Ofs.isProvenEqualTo(C0->EI[0].Ofs + i * Size)) {
- Res[i] = C;
- }
- }
-
- for (i = 1; i < Factor; i++) {
- if (Res[i] == Candidates.end())
- break;
- }
- if (i == Factor) {
- Res[0] = C0;
- break;
- }
- }
-
- if (Res[0] != Candidates.end()) {
- // Move the result into the output
- for (unsigned i = 0; i < Factor; i++) {
- InterleavedLoad.splice(InterleavedLoad.end(), Candidates, Res[i]);
- }
-
- return true;
- }
- }
- return false;
-}
-
LoadInst *
InterleavedLoadCombineImpl::findFirstLoad(const std::set<LoadInst *> &LIs) {
assert(!LIs.empty() && "No load instructions given.");
@@ -1114,14 +1102,14 @@ InterleavedLoadCombineImpl::findFirstLoad(const std::set<LoadInst *> &LIs) {
return cast<LoadInst>(FLI);
}
-bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
+bool InterleavedLoadCombineImpl::combine(ArrayRef<VectorInfo *> InterleavedLoad,
OptimizationRemarkEmitter &ORE) {
LLVM_DEBUG(dbgs() << "Checking interleaved load\n");
// The insertion point is the LoadInst which loads the first values. The
// following tests are used to proof that the combined load can be inserted
// just before InsertionPoint.
- LoadInst *InsertionPoint = InterleavedLoad.front().EI[0].LI;
+ LoadInst *InsertionPoint = InterleavedLoad.front()->EI[0].LI;
// Test if the offset is computed
if (!InsertionPoint)
@@ -1139,17 +1127,17 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
unsigned Factor = InterleavedLoad.size();
// Merge all input sets used in analysis
- for (auto &VI : InterleavedLoad) {
+ for (const VectorInfo *VI : InterleavedLoad) {
// Generate a set of all load instructions to be combined
- LIs.insert(VI.LIs.begin(), VI.LIs.end());
+ LIs.insert(VI->LIs.begin(), VI->LIs.end());
// Generate a set of all instructions taking part in load
// interleaved. This list excludes the instructions necessary for the
// polynomial construction.
- Is.insert(VI.Is.begin(), VI.Is.end());
+ Is.insert(VI->Is.begin(), VI->Is.end());
// Generate the set of the final ShuffleVectorInst.
- SVIs.insert(VI.SVI);
+ SVIs.insert(VI->SVI);
}
// There is nothing to combine.
@@ -1195,17 +1183,17 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
assert(!LIs.empty() && "There are no LoadInst to combine");
// It is necessary that insertion point dominates all final ShuffleVectorInst.
- for (auto &VI : InterleavedLoad) {
- if (!DT.dominates(InsertionPoint, VI.SVI))
+ for (const VectorInfo *VI : InterleavedLoad) {
+ if (!DT.dominates(InsertionPoint, VI->SVI))
return false;
}
// All checks are done. Add instructions detectable by InterleavedAccessPass
// The old instruction will are left dead.
IRBuilder<> Builder(InsertionPoint);
- Type *ETy = InterleavedLoad.front().SVI->getType()->getElementType();
+ Type *ETy = InterleavedLoad.front()->SVI->getType()->getElementType();
unsigned ElementsPerSVI =
- cast<FixedVectorType>(InterleavedLoad.front().SVI->getType())
+ cast<FixedVectorType>(InterleavedLoad.front()->SVI->getType())
->getNumElements();
FixedVectorType *ILTy = FixedVectorType::get(ETy, Factor * ElementsPerSVI);
@@ -1229,14 +1217,14 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
// Create the final SVIs and replace all uses.
int i = 0;
- for (auto &VI : InterleavedLoad) {
+ for (const VectorInfo *VI : InterleavedLoad) {
SmallVector<int, 4> Mask;
for (unsigned j = 0; j < ElementsPerSVI; j++)
Mask.push_back(i + j * Factor);
- Builder.SetInsertPoint(VI.SVI);
+ Builder.SetInsertPoint(VI->SVI);
auto SVI = Builder.CreateShuffleVector(LI, Mask, "interleaved.shuffle");
- VI.SVI->replaceAllUsesWith(SVI);
+ VI->SVI->replaceAllUsesWith(SVI);
i++;
}
@@ -1282,18 +1270,60 @@ bool InterleavedLoadCombineImpl::run() {
}
}
- std::list<VectorInfo> InterleavedLoad;
- while (findPattern(Candidates, InterleavedLoad, Factor, DL)) {
- if (combine(InterleavedLoad, ORE)) {
+ // Index every candidate whose first element has a provably exact offset by
+ // its address key. Finding an interleaved group then only needs lookups of
+ // the neighbouring keys. The key embeds a Polynomial, which has no natural
+ // empty/tombstone value, so use std::unordered_map rather than DenseMap.
+ std::unordered_map<OffsetKey, SmallVector<VectorInfo *, 1>, OffsetKeyHash>
+ OffsetMap;
+ for (VectorInfo &C : Candidates) {
+ if (!C.EI[0].Ofs.isProvenExact())
+ continue;
+ OffsetMap[{C.BB, C.PV, C.VTy, C.EI[0].Ofs}].push_back(&C);
+ }
+
+ // Candidates already combined (a whole group) or dropped (a failed base).
+ SmallPtrSet<const VectorInfo *, 16> Consumed;
+
+ // Return the last still-available candidate registered under Key. Iterating
+ // in reverse makes a later duplicate offset win over an earlier one.
+ auto FindNeighbor = [&](const OffsetKey &Key) -> VectorInfo * {
+ auto It = OffsetMap.find(Key);
+ if (It == OffsetMap.end())
+ return nullptr;
+ for (VectorInfo *Cand : reverse(It->second))
+ if (!Consumed.contains(Cand))
+ return Cand;
+ return nullptr;
+ };
+
+ for (VectorInfo &C0 : Candidates) {
+ if (Consumed.contains(&C0) || !C0.EI[0].Ofs.isProvenExact())
+ continue;
+
+ unsigned Size = DL.getTypeAllocSize(C0.VTy->getElementType());
+
+ // Collect C0 and its Factor - 1 consecutive neighbours.
+ SmallVector<VectorInfo *, 4> Group;
+ Group.push_back(&C0);
+ for (unsigned i = 1; i < Factor; i++) {
+ VectorInfo *Nb =
+ FindNeighbor({C0.BB, C0.PV, C0.VTy, C0.EI[0].Ofs + i * Size});
+ if (!Nb)
+ break;
+ Group.push_back(Nb);
+ }
+ if (Group.size() != Factor)
+ continue;
+
+ if (combine(Group, ORE)) {
+ // The whole group is combined and left dead.
+ Consumed.insert(Group.begin(), Group.end());
changed = true;
} else {
- // Remove the first element of the Interleaved Load but put the others
- // back on the list and continue searching
- Candidates.splice(Candidates.begin(), InterleavedLoad,
- std::next(InterleavedLoad.begin()),
- InterleavedLoad.end());
+ // Drop only the base and keep its neighbours available as future bases.
+ Consumed.insert(&C0);
}
- InterleavedLoad.clear();
}
}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll b/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll
new file mode 100644
index 0000000000000..9eb0ea2ec24d3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck --check-prefix AS %s
+; RUN: opt -S -passes=interleaved-load-combine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "arm64--linux-gnu"
+
+define void @many_ld2(ptr %ptr) {
+; CHECK-LABEL: define void @many_ld2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 3
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 4
+; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 5
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD:%.*]] = load <8 x float>, ptr [[GEP0]], align 16
+; CHECK-NEXT: [[A0:%.*]] = load <4 x float>, ptr [[GEP0]], align 16
+; CHECK-NEXT: [[A1:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD2:%.*]] = load <8 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[B0:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[B1:%.*]] = load <4 x float>, ptr [[GEP3]], align 16
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD5:%.*]] = load <8 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[C0:%.*]] = load <4 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[C1:%.*]] = load <4 x float>, ptr [[GEP5]], align 16
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[AE:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[AO:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE3:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[BE:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE4:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[BO:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE6:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD5]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[CE:%.*]] = shufflevector <4 x float> [[C0]], <4 x float> [[C1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE7:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD5]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[CO:%.*]] = shufflevector <4 x float> [[C0]], <4 x float> [[C1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP0]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE3]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE4]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE6]], ptr [[GEP4]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE7]], ptr [[GEP5]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+
+; AS-LABEL: many_ld2
+; AS: ld2
+; AS: ld2
+; AS: ld2
+; AS: ret
+
+ %gep0 = getelementptr inbounds <4 x float>, ptr %ptr, i64 0
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 2
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 3
+ %gep4 = getelementptr inbounds <4 x float>, ptr %ptr, i64 4
+ %gep5 = getelementptr inbounds <4 x float>, ptr %ptr, i64 5
+
+ %a0 = load <4 x float>, ptr %gep0, align 16
+ %a1 = load <4 x float>, ptr %gep1, align 16
+ %b0 = load <4 x float>, ptr %gep2, align 16
+ %b1 = load <4 x float>, ptr %gep3, align 16
+ %c0 = load <4 x float>, ptr %gep4, align 16
+ %c1 = load <4 x float>, ptr %gep5, align 16
+
+ %ae = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %ao = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %be = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %bo = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %ce = shufflevector <4 x float> %c0, <4 x float> %c1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %co = shufflevector <4 x float> %c0, <4 x float> %c1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+
+ store <4 x float> %ae, ptr %gep0
+ store <4 x float> %ao, ptr %gep1
+ store <4 x float> %be, ptr %gep2
+ store <4 x float> %bo, ptr %gep3
+ store <4 x float> %ce, ptr %gep4
+ store <4 x float> %co, ptr %gep5
+ ret void
+}
More information about the llvm-commits
mailing list