[llvm] [InterleavedLoadCombine] Index candidates to avoid quadratic matching (PR #213053)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 08:54:50 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Matching scanned every candidate against every other candidate, which got very slow on functions with many interleaved loads. Index by address offset and look up neighbors instead.
llc's Compile-time drops from 212s to ~2s on a Grace machine.
Fixes #<!-- -->162299.
---
Full diff: https://github.com/llvm/llvm-project/pull/213053.diff
2 Files Affected:
- (modified) llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp (+108-78)
- (added) llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll (+83)
``````````diff
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index aaba0d14994e5..4d75255df1510 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -18,6 +18,10 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
@@ -43,6 +47,7 @@
#include <algorithm>
#include <cassert>
#include <list>
+#include <unordered_map>
using namespace llvm;
@@ -95,14 +100,8 @@ struct InterleavedLoadCombineImpl {
/// Replace interleaved load candidates. It does additional
/// analyses if this makes sense. Returns true on success and false
/// of nothing has been changed.
- bool combine(std::list<VectorInfo> &InterleavedLoad,
+ bool combine(ArrayRef<VectorInfo *> InterleavedLoad,
OptimizationRemarkEmitter &ORE);
-
- /// Given a set of VectorInfo containing candidates for a given interleave
- /// factor, find a set that represents a 'factor' interleaved load.
- bool findPattern(std::list<VectorInfo> &Candidates,
- std::list<VectorInfo> &InterleavedLoad, unsigned Factor,
- const DataLayout &DL);
}; // InterleavedLoadCombine
/// First Order Polynomial on an n-Bit Integer Value
@@ -566,12 +565,26 @@ class Polynomial {
}
/// Returns true if it can be proven that two Polynomials are equal.
- bool isProvenEqualTo(const Polynomial &o) {
+ bool isProvenEqualTo(const Polynomial &o) const {
// Subtract both polynomials and test if it is fully defined and zero.
Polynomial r = *this - o;
return (r.ErrorMSBs == 0) && (!r.isFirstOrder()) && (r.A.isZero());
}
+ /// Returns true if every bit of the polynomial is provably exact. An inexact
+ /// polynomial can never be proven equal to another, so it is never a valid
+ /// match candidate.
+ bool isProvenExact() const { return ErrorMSBs == 0; }
+
+ /// Hash the identity checked by isProvenEqualTo. Only meaningful for exact
+ /// polynomials; two exact, proven-equal polynomials hash identically.
+ friend hash_code hash_value(const Polynomial &P) {
+ hash_code H = hash_combine(P.A.getBitWidth(), P.V);
+ for (const auto &BO : P.B)
+ H = hash_combine(H, BO.first, hash_value(BO.second));
+ return hash_combine(H, hash_value(P.A));
+ }
+
/// Print the polynomial into a stream.
void print(raw_ostream &OS) const {
OS << "[{#ErrBits:" << ErrorMSBs << "} ";
@@ -625,6 +638,29 @@ static raw_ostream &operator<<(raw_ostream &OS, const Polynomial &S) {
}
#endif
+/// Address key of a candidate's first vector element: the block, the common
+/// base pointer, the vector type and the offset polynomial. Two candidates
+/// belong to the same interleaved group iff their keys agree on everything but
+/// the constant offset, so consecutive elements are located by building the
+/// neighbouring keys and looking them up.
+struct OffsetKey {
+ BasicBlock *BB;
+ Value *PV;
+ FixedVectorType *VTy;
+ Polynomial Ofs;
+
+ bool operator==(const OffsetKey &O) const {
+ return BB == O.BB && PV == O.PV && VTy == O.VTy &&
+ Ofs.isProvenEqualTo(O.Ofs);
+ }
+};
+
+struct OffsetKeyHash {
+ size_t operator()(const OffsetKey &K) const {
+ return hash_combine(K.BB, K.PV, K.VTy, hash_value(K.Ofs));
+ }
+};
+
/// VectorInfo stores abstract the following information for each vector
/// element:
///
@@ -1053,54 +1089,6 @@ struct VectorInfo {
} // anonymous namespace
-bool InterleavedLoadCombineImpl::findPattern(
- std::list<VectorInfo> &Candidates, std::list<VectorInfo> &InterleavedLoad,
- unsigned Factor, const DataLayout &DL) {
- for (auto C0 = Candidates.begin(), E0 = Candidates.end(); C0 != E0; ++C0) {
- unsigned i;
- // Try to find an interleaved load using the front of Worklist as first line
- unsigned Size = DL.getTypeAllocSize(C0->VTy->getElementType());
-
- // List containing iterators pointing to the VectorInfos of the candidates
- std::vector<std::list<VectorInfo>::iterator> Res(Factor, Candidates.end());
-
- for (auto C = Candidates.begin(), E = Candidates.end(); C != E; C++) {
- if (C->VTy != C0->VTy)
- continue;
- if (C->BB != C0->BB)
- continue;
- if (C->PV != C0->PV)
- continue;
-
- // Check the current value matches any of factor - 1 remaining lines
- for (i = 1; i < Factor; i++) {
- if (C->EI[0].Ofs.isProvenEqualTo(C0->EI[0].Ofs + i * Size)) {
- Res[i] = C;
- }
- }
-
- for (i = 1; i < Factor; i++) {
- if (Res[i] == Candidates.end())
- break;
- }
- if (i == Factor) {
- Res[0] = C0;
- break;
- }
- }
-
- if (Res[0] != Candidates.end()) {
- // Move the result into the output
- for (unsigned i = 0; i < Factor; i++) {
- InterleavedLoad.splice(InterleavedLoad.end(), Candidates, Res[i]);
- }
-
- return true;
- }
- }
- return false;
-}
-
LoadInst *
InterleavedLoadCombineImpl::findFirstLoad(const std::set<LoadInst *> &LIs) {
assert(!LIs.empty() && "No load instructions given.");
@@ -1114,14 +1102,14 @@ InterleavedLoadCombineImpl::findFirstLoad(const std::set<LoadInst *> &LIs) {
return cast<LoadInst>(FLI);
}
-bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
+bool InterleavedLoadCombineImpl::combine(ArrayRef<VectorInfo *> InterleavedLoad,
OptimizationRemarkEmitter &ORE) {
LLVM_DEBUG(dbgs() << "Checking interleaved load\n");
// The insertion point is the LoadInst which loads the first values. The
// following tests are used to proof that the combined load can be inserted
// just before InsertionPoint.
- LoadInst *InsertionPoint = InterleavedLoad.front().EI[0].LI;
+ LoadInst *InsertionPoint = InterleavedLoad.front()->EI[0].LI;
// Test if the offset is computed
if (!InsertionPoint)
@@ -1139,17 +1127,17 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
unsigned Factor = InterleavedLoad.size();
// Merge all input sets used in analysis
- for (auto &VI : InterleavedLoad) {
+ for (const VectorInfo *VI : InterleavedLoad) {
// Generate a set of all load instructions to be combined
- LIs.insert(VI.LIs.begin(), VI.LIs.end());
+ LIs.insert(VI->LIs.begin(), VI->LIs.end());
// Generate a set of all instructions taking part in load
// interleaved. This list excludes the instructions necessary for the
// polynomial construction.
- Is.insert(VI.Is.begin(), VI.Is.end());
+ Is.insert(VI->Is.begin(), VI->Is.end());
// Generate the set of the final ShuffleVectorInst.
- SVIs.insert(VI.SVI);
+ SVIs.insert(VI->SVI);
}
// There is nothing to combine.
@@ -1195,17 +1183,17 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
assert(!LIs.empty() && "There are no LoadInst to combine");
// It is necessary that insertion point dominates all final ShuffleVectorInst.
- for (auto &VI : InterleavedLoad) {
- if (!DT.dominates(InsertionPoint, VI.SVI))
+ for (const VectorInfo *VI : InterleavedLoad) {
+ if (!DT.dominates(InsertionPoint, VI->SVI))
return false;
}
// All checks are done. Add instructions detectable by InterleavedAccessPass
// The old instruction will are left dead.
IRBuilder<> Builder(InsertionPoint);
- Type *ETy = InterleavedLoad.front().SVI->getType()->getElementType();
+ Type *ETy = InterleavedLoad.front()->SVI->getType()->getElementType();
unsigned ElementsPerSVI =
- cast<FixedVectorType>(InterleavedLoad.front().SVI->getType())
+ cast<FixedVectorType>(InterleavedLoad.front()->SVI->getType())
->getNumElements();
FixedVectorType *ILTy = FixedVectorType::get(ETy, Factor * ElementsPerSVI);
@@ -1229,14 +1217,14 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
// Create the final SVIs and replace all uses.
int i = 0;
- for (auto &VI : InterleavedLoad) {
+ for (const VectorInfo *VI : InterleavedLoad) {
SmallVector<int, 4> Mask;
for (unsigned j = 0; j < ElementsPerSVI; j++)
Mask.push_back(i + j * Factor);
- Builder.SetInsertPoint(VI.SVI);
+ Builder.SetInsertPoint(VI->SVI);
auto SVI = Builder.CreateShuffleVector(LI, Mask, "interleaved.shuffle");
- VI.SVI->replaceAllUsesWith(SVI);
+ VI->SVI->replaceAllUsesWith(SVI);
i++;
}
@@ -1282,18 +1270,60 @@ bool InterleavedLoadCombineImpl::run() {
}
}
- std::list<VectorInfo> InterleavedLoad;
- while (findPattern(Candidates, InterleavedLoad, Factor, DL)) {
- if (combine(InterleavedLoad, ORE)) {
+ // Index every candidate whose first element has a provably exact offset by
+ // its address key. Finding an interleaved group then only needs lookups of
+ // the neighbouring keys. The key embeds a Polynomial, which has no natural
+ // empty/tombstone value, so use std::unordered_map rather than DenseMap.
+ std::unordered_map<OffsetKey, SmallVector<VectorInfo *, 1>, OffsetKeyHash>
+ OffsetMap;
+ for (VectorInfo &C : Candidates) {
+ if (!C.EI[0].Ofs.isProvenExact())
+ continue;
+ OffsetMap[{C.BB, C.PV, C.VTy, C.EI[0].Ofs}].push_back(&C);
+ }
+
+ // Candidates already combined (a whole group) or dropped (a failed base).
+ SmallPtrSet<const VectorInfo *, 16> Consumed;
+
+ // Return the last still-available candidate registered under Key. Iterating
+ // in reverse makes a later duplicate offset win over an earlier one.
+ auto FindNeighbor = [&](const OffsetKey &Key) -> VectorInfo * {
+ auto It = OffsetMap.find(Key);
+ if (It == OffsetMap.end())
+ return nullptr;
+ for (VectorInfo *Cand : reverse(It->second))
+ if (!Consumed.contains(Cand))
+ return Cand;
+ return nullptr;
+ };
+
+ for (VectorInfo &C0 : Candidates) {
+ if (Consumed.contains(&C0) || !C0.EI[0].Ofs.isProvenExact())
+ continue;
+
+ unsigned Size = DL.getTypeAllocSize(C0.VTy->getElementType());
+
+ // Collect C0 and its Factor - 1 consecutive neighbours.
+ SmallVector<VectorInfo *, 4> Group;
+ Group.push_back(&C0);
+ for (unsigned i = 1; i < Factor; i++) {
+ VectorInfo *Nb =
+ FindNeighbor({C0.BB, C0.PV, C0.VTy, C0.EI[0].Ofs + i * Size});
+ if (!Nb)
+ break;
+ Group.push_back(Nb);
+ }
+ if (Group.size() != Factor)
+ continue;
+
+ if (combine(Group, ORE)) {
+ // The whole group is combined and left dead.
+ Consumed.insert(Group.begin(), Group.end());
changed = true;
} else {
- // Remove the first element of the Interleaved Load but put the others
- // back on the list and continue searching
- Candidates.splice(Candidates.begin(), InterleavedLoad,
- std::next(InterleavedLoad.begin()),
- InterleavedLoad.end());
+ // Drop only the base and keep its neighbours available as future bases.
+ Consumed.insert(&C0);
}
- InterleavedLoad.clear();
}
}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll b/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll
new file mode 100644
index 0000000000000..9eb0ea2ec24d3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/interleaved-load-combine-many-candidates.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck --check-prefix AS %s
+; RUN: opt -S -passes=interleaved-load-combine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "arm64--linux-gnu"
+
+define void @many_ld2(ptr %ptr) {
+; CHECK-LABEL: define void @many_ld2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 3
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 4
+; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 5
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD:%.*]] = load <8 x float>, ptr [[GEP0]], align 16
+; CHECK-NEXT: [[A0:%.*]] = load <4 x float>, ptr [[GEP0]], align 16
+; CHECK-NEXT: [[A1:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD2:%.*]] = load <8 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[B0:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[B1:%.*]] = load <4 x float>, ptr [[GEP3]], align 16
+; CHECK-NEXT: [[INTERLEAVED_WIDE_LOAD5:%.*]] = load <8 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[C0:%.*]] = load <4 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[C1:%.*]] = load <4 x float>, ptr [[GEP5]], align 16
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[AE:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[AO:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE3:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[BE:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE4:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[BO:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE6:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD5]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[CE:%.*]] = shufflevector <4 x float> [[C0]], <4 x float> [[C1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE7:%.*]] = shufflevector <8 x float> [[INTERLEAVED_WIDE_LOAD5]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[CO:%.*]] = shufflevector <4 x float> [[C0]], <4 x float> [[C1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP0]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE3]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE4]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE6]], ptr [[GEP4]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE7]], ptr [[GEP5]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+
+; AS-LABEL: many_ld2
+; AS: ld2
+; AS: ld2
+; AS: ld2
+; AS: ret
+
+ %gep0 = getelementptr inbounds <4 x float>, ptr %ptr, i64 0
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 2
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 3
+ %gep4 = getelementptr inbounds <4 x float>, ptr %ptr, i64 4
+ %gep5 = getelementptr inbounds <4 x float>, ptr %ptr, i64 5
+
+ %a0 = load <4 x float>, ptr %gep0, align 16
+ %a1 = load <4 x float>, ptr %gep1, align 16
+ %b0 = load <4 x float>, ptr %gep2, align 16
+ %b1 = load <4 x float>, ptr %gep3, align 16
+ %c0 = load <4 x float>, ptr %gep4, align 16
+ %c1 = load <4 x float>, ptr %gep5, align 16
+
+ %ae = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %ao = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %be = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %bo = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %ce = shufflevector <4 x float> %c0, <4 x float> %c1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %co = shufflevector <4 x float> %c0, <4 x float> %c1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+
+ store <4 x float> %ae, ptr %gep0
+ store <4 x float> %ao, ptr %gep1
+ store <4 x float> %be, ptr %gep2
+ store <4 x float> %bo, ptr %gep3
+ store <4 x float> %ce, ptr %gep4
+ store <4 x float> %co, ptr %gep5
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/213053
More information about the llvm-commits
mailing list