[llvm] [SelectionDAG] Share repeated terminal matcher tails (PR #202638)
David Zbarsky via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 07:00:44 PDT 2026
https://github.com/dzbarsky created https://github.com/llvm/llvm-project/pull/202638
Add an absolute jump opcode to the SelectionDAG matcher interpreter. DAGISelMatcherEmitter identifies structurally identical terminal result suffixes, emits one canonical suffix after the main matcher table, and replaces the other occurrences with jumps.
Only non-failing result operations participate. EmitMergeInputChains and every operation that can reject a match remain before the jump. Disable sharing when matcher coverage instrumentation is enabled, and retain histogram accounting for logical matchers replaced by jumps.
On current main, the generated AArch64, AMDGPU, and ARM matcher tables shrink by 83,533 bytes at the 48-byte threshold. On the LLVM 22 all-target stack, including R600, raw matcher tables shrink by 87,470 bytes.
In the LLVM 22 Bazel build, the stripped multicall binary shrinks from 142,889,168 to 142,806,608 bytes, saving 82,560 bytes. Standalone llc shrinks from 78,497,384 to 78,398,312 bytes, saving 99,072 bytes.
Forced AArch64 SelectionDAG code generation produced byte-identical object files for a minimal integer add and an SDOT case. An eight-pair alternating tramp3d benchmark measured 1.5325 seconds mean user CPU before the change and 1.4800 seconds after it.
Add focused TableGen coverage for shared identical tails, distinct tails, chain merges before jumps, and disabled sharing under coverage instrumentation. Compile the shared interpreter and the affected target selectors.
Work towards #202616
>From 7da09b49d3d59097073d2b7fef3a1588157374f2 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 23:36:50 -0400
Subject: [PATCH] [SelectionDAG] Share repeated terminal matcher tails
Add an absolute jump opcode to the SelectionDAG matcher interpreter. DAGISelMatcherEmitter identifies structurally identical terminal result suffixes, emits one canonical suffix after the main matcher table, and replaces the other occurrences with jumps.
Only non-failing result operations participate. EmitMergeInputChains and every operation that can reject a match remain before the jump. Disable sharing when matcher coverage instrumentation is enabled, and retain histogram accounting for logical matchers replaced by jumps.
On current main, the generated AArch64, AMDGPU, and ARM matcher tables shrink by 83,533 bytes at the 48-byte threshold. On the LLVM 22 all-target stack, including R600, raw matcher tables shrink by 87,470 bytes.
In the LLVM 22 Bazel build, the stripped multicall binary shrinks from 142,889,168 to 142,806,608 bytes, saving 82,560 bytes. Standalone llc shrinks from 78,497,384 to 78,398,312 bytes, saving 99,072 bytes.
Forced AArch64 SelectionDAG code generation produced byte-identical object files for a minimal integer add and an SDOT case. An eight-pair alternating tramp3d benchmark measured 1.5325 seconds mean user CPU before the change and 1.4800 seconds after it.
Add focused TableGen coverage for shared identical tails, distinct tails, chain merges before jumps, and disabled sharing under coverage instrumentation. Compile the shared interpreter and the affected target selectors.
---
llvm/include/llvm/CodeGen/SelectionDAGISel.h | 6 +
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 10 +
.../TableGen/dag-isel-result-tail-sharing.td | 109 +++++
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 379 +++++++++++++++++-
4 files changed, 501 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/TableGen/dag-isel-result-tail-sharing.td
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 7b406ef4b4cb0..1cd0f332fd66e 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -282,6 +282,10 @@ class LLVM_ABI SelectionDAGISel {
OPC_CheckImmAllZerosV,
OPC_CheckFoldableChainNode,
+ // Transfer control to a shared result tail. The target is encoded as a
+ // 32-bit absolute MatcherTable offset.
+ OPC_Jump,
+
OPC_EmitInteger,
// Space-optimized forms that implicitly encode integer VT.
OPC_EmitIntegerI8,
@@ -352,6 +356,8 @@ class LLVM_ABI SelectionDAGISel {
// Contains 32-bit offset in table for pattern being selected
OPC_Coverage
};
+ static_assert(OPC_Coverage < 256,
+ "SelectionDAG matcher opcodes must fit in one byte");
enum {
OPFL_None = 0, // Node has no chain or glue input and isn't variadic.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 5ae52cae771fb..24387390bc5f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -4104,6 +4104,16 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
continue;
}
+ case OPC_Jump: {
+ uint32_t Target = MatcherTable[MatcherIndex++];
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 8;
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 16;
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 24;
+ assert(Target < TableSize && "Invalid SelectionDAG matcher jump");
+ MatcherIndex = Target;
+ continue;
+ }
+
case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
diff --git a/llvm/test/TableGen/dag-isel-result-tail-sharing.td b/llvm/test/TableGen/dag-isel-result-tail-sharing.td
new file mode 100644
index 0000000000000..9705e4f00e8f2
--- /dev/null
+++ b/llvm/test/TableGen/dag-isel-result-tail-sharing.td
@@ -0,0 +1,109 @@
+// RUN: llvm-tblgen -gen-dag-isel -I %p/../../include %s -o - \
+// RUN: | FileCheck --check-prefix=SHARED %s
+// RUN: llvm-tblgen -gen-dag-isel -DSTRUCTURAL_DIFFERENCE \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=DIFFERENT %s
+// RUN: llvm-tblgen -gen-dag-isel -DFAILURE_PREFIX \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=FAILURE %s
+// RUN: llvm-tblgen -gen-dag-isel -instrument-coverage \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=COVERAGE %s
+
+include "llvm/Target/Target.td"
+
+def TestInstrInfo : InstrInfo;
+def TestTarget : Target {
+ let InstructionSet = TestInstrInfo;
+}
+
+def R0 : Register<"r0">;
+def GPR : RegisterClass<"TestTarget", [i32], 32, (add R0)>;
+
+def WRAP : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$src);
+}
+
+def WRAP_ALT : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$src);
+}
+
+def PAIR : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$lhs, GPR:$rhs);
+}
+
+#ifndef STRUCTURAL_DIFFERENCE
+#ifndef FAILURE_PREFIX
+// The ADD and SUB patterns have different predicates but identical generated
+// results. Their result suffix is long enough to share.
+def : Pat<
+ (add GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (sub GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// SHARED: OPC_Jump, JUMP_TARGET([[TAIL:[0-9]+]]),
+// SHARED: OPC_Jump, JUMP_TARGET([[TAIL]]),
+// SHARED: // Shared result tail at [[TAIL]]
+// SHARED: OPC_EmitNode
+// SHARED: OPC_MorphNodeTo
+#endif
+#endif
+
+#ifdef STRUCTURAL_DIFFERENCE
+// These result sequences have the same matcher kinds and encoded sizes, but
+// different instruction records. The structural key must keep them separate.
+def : Pat<
+ (add GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (sub GPR:$x, GPR:$y),
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// DIFFERENT-NOT: OPC_Jump
+#endif
+
+#ifdef FAILURE_PREFIX
+// EmitMergeInputChains may fail after inspecting the matched DAG. Each pattern
+// must execute its own merge before entering the shared non-failing suffix.
+def int_chain_a : Intrinsic<
+ [llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
+def int_chain_b : Intrinsic<
+ [llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
+
+def : Pat<
+ (int_chain_a GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (int_chain_b GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// FAILURE: OPC_EmitMergeInputChains
+// FAILURE-NEXT: {{.*}}OPC_Jump, JUMP_TARGET([[TAIL:[0-9]+]]),
+// FAILURE: OPC_EmitMergeInputChains
+// FAILURE-NEXT: {{.*}}OPC_Jump, JUMP_TARGET([[TAIL]]),
+// FAILURE: // Shared result tail at [[TAIL]]
+// FAILURE-NOT: OPC_EmitMergeInputChains
+// FAILURE: OPC_EmitNode
+#endif
+
+// Coverage output must preserve one coverage record per source pattern.
+// COVERAGE-NOT: OPC_Jump
+// COVERAGE-COUNT-2: OPC_Coverage
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 3d69c96ebd900..83fd039f420c6 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -18,6 +18,8 @@
#include "Common/CodeGenTarget.h"
#include "DAGISelMatcher.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/TinyPtrVector.h"
@@ -34,6 +36,8 @@ enum {
IndexWidth = 7,
FullIndexWidth = IndexWidth + 4,
HistOpcWidth = 40,
+ MinSharedResultTailSize = 48,
+ JumpSize = 5,
};
static cl::OptionCategory DAGISelCat("Options for -gen-dag-isel");
@@ -50,6 +54,18 @@ static cl::opt<bool> InstrumentCoverage(
namespace {
class MatcherTableEmitter {
+ struct TailOccurrence {
+ const MatcherList *Matchers;
+ const Matcher *Start;
+ };
+
+ struct SharedTail {
+ const MatcherList *Matchers;
+ const Matcher *Start;
+ unsigned Size;
+ unsigned Offset = 0;
+ };
+
const CodeGenDAGPatterns &CGP;
SmallVector<unsigned, Matcher::HighestKind + 1> OpcodeCounts;
@@ -78,6 +94,10 @@ class MatcherTableEmitter {
SequenceToOffsetTable<std::vector<uint8_t>> OperandTable;
+ SmallVector<SharedTail, 16> SharedTails;
+ DenseMap<const Matcher *, unsigned> SharedTailByStart;
+ bool EmittingSharedTail = false;
+
unsigned getPatternIdxFromTable(std::string &&P, std::string &&include_loc) {
const auto [It, Inserted] =
VecPatterns.try_emplace(std::move(P), VecPatterns.size());
@@ -205,6 +225,9 @@ class MatcherTableEmitter {
else
NodePredicates.push_back(TP);
}
+
+ if (!InstrumentCoverage)
+ collectSharedTails(TheMatcherList);
}
unsigned EmitMatcherList(const MatcherList &ML, const unsigned Indent,
@@ -222,7 +245,132 @@ class MatcherTableEmitter {
void EmitPatternMatchTable(raw_ostream &OS);
+ void layoutSharedTails(unsigned MainTableSize);
+
+ unsigned EmitSharedTails(unsigned CurrentIdx, raw_ostream &OS);
+
+ unsigned getSharedTailsSize() const {
+ unsigned Size = 0;
+ for (const SharedTail &Tail : SharedTails)
+ Size += Tail.Size;
+ return Size;
+ }
+
private:
+ static bool isNonFailingResultMatcher(const Matcher *N) {
+ switch (N->getKind()) {
+ case Matcher::EmitInteger:
+ case Matcher::EmitRegister:
+ case Matcher::EmitConvertToTarget:
+ case Matcher::EmitCopyToReg:
+ case Matcher::EmitNode:
+ case Matcher::EmitNodeXForm:
+ case Matcher::CompleteMatch:
+ case Matcher::MorphNodeTo:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static bool isTerminalResultMatcher(const Matcher *N) {
+ return isa<CompleteMatchMatcher>(N) || isa<MorphNodeToMatcher>(N);
+ }
+
+ static bool areResultMatchersEquivalent(const Matcher *LHS,
+ const Matcher *RHS) {
+ if (LHS->getKind() != RHS->getKind())
+ return false;
+
+ // CompleteMatchMatcher::isEqual() compares Pattern for matcher
+ // optimization. Pattern does not affect the emitted matcher instruction
+ // when coverage instrumentation is disabled.
+ if (const auto *LCM = dyn_cast<CompleteMatchMatcher>(LHS)) {
+ const auto *RCM = cast<CompleteMatchMatcher>(RHS);
+ if (LCM->getNumResults() != RCM->getNumResults())
+ return false;
+ for (unsigned I = 0; I != LCM->getNumResults(); ++I)
+ if (LCM->getResult(I) != RCM->getResult(I))
+ return false;
+ return true;
+ }
+
+ return LHS->isEqual(RHS);
+ }
+
+ static bool areResultTailsEquivalent(ArrayRef<const Matcher *> LHS,
+ ArrayRef<const Matcher *> RHS) {
+ return LHS.size() == RHS.size() &&
+ llvm::equal(LHS, RHS, areResultMatchersEquivalent);
+ }
+
+ static hash_code getValueTypeHash(const ValueTypeByHwMode &VT) {
+ hash_code Hash = hash_value(VT.size());
+ for (const auto &[Mode, Type] : VT)
+ Hash = hash_combine(Hash, Mode, static_cast<unsigned>(Type.SimpleTy));
+ return Hash;
+ }
+
+ static hash_code getResultMatcherHash(const Matcher *N) {
+ hash_code Hash = hash_value(static_cast<unsigned>(N->getKind()));
+ switch (N->getKind()) {
+ case Matcher::EmitInteger: {
+ const auto *M = cast<EmitIntegerMatcher>(N);
+ return hash_combine(Hash, M->getValue(), M->getString(),
+ getValueTypeHash(M->getVT()));
+ }
+ case Matcher::EmitRegister: {
+ const auto *M = cast<EmitRegisterMatcher>(N);
+ return hash_combine(Hash, M->getReg(), getValueTypeHash(M->getVT()));
+ }
+ case Matcher::EmitConvertToTarget:
+ return hash_combine(Hash, cast<EmitConvertToTargetMatcher>(N)->getSlot());
+ case Matcher::EmitCopyToReg: {
+ const auto *M = cast<EmitCopyToRegMatcher>(N);
+ return hash_combine(Hash, M->getSrcSlot(), M->getDestPhysReg());
+ }
+ case Matcher::EmitNodeXForm: {
+ const auto *M = cast<EmitNodeXFormMatcher>(N);
+ return hash_combine(Hash, M->getSlot(), M->getNodeXForm());
+ }
+ case Matcher::EmitNode:
+ case Matcher::MorphNodeTo: {
+ const auto *M = cast<EmitNodeMatcherCommon>(N);
+ Hash = hash_combine(Hash, &M->getInstruction(), M->hasChain(),
+ M->hasInGlue(), M->hasOutGlue(), M->hasMemRefs(),
+ M->getNumFixedArityOperands());
+ for (const ValueTypeByHwMode &VT : M->getVTList())
+ Hash = hash_combine(Hash, getValueTypeHash(VT));
+ return hash_combine(Hash, hash_combine_range(M->getOperandList().begin(),
+ M->getOperandList().end()));
+ }
+ case Matcher::CompleteMatch: {
+ const auto *M = cast<CompleteMatchMatcher>(N);
+ for (unsigned I = 0; I != M->getNumResults(); ++I)
+ Hash = hash_combine(Hash, M->getResult(I));
+ return Hash;
+ }
+ default:
+ llvm_unreachable("not a non-failing result matcher");
+ }
+ }
+
+ static size_t getResultTailHash(ArrayRef<const Matcher *> Tail) {
+ hash_code Hash = hash_value(Tail.size());
+ for (const Matcher *N : Tail)
+ Hash = hash_combine(Hash, getResultMatcherHash(N));
+ return static_cast<size_t>(Hash);
+ }
+
+ void collectSharedTails(const MatcherList &ML);
+
+ unsigned EmitMatcherSuffix(const MatcherList &ML, const Matcher *Start,
+ const unsigned Indent, unsigned CurrentIdx,
+ raw_ostream &OS);
+
+ unsigned EmitJump(const SharedTail &Tail, const unsigned Indent,
+ raw_ostream &OS);
+
// Reorder ValueType indices by usage frequency (most common -> index 0).
// Updates the indices directly in ValueTypeMap.
void sortValueTypeByHwModeByFrequency() {
@@ -377,8 +525,23 @@ static std::string getIncludePath(const Record *R) {
unsigned MatcherTableEmitter::SizeMatcherList(MatcherList &ML,
raw_ostream &OS) {
unsigned Size = 0;
- for (Matcher *N : ML)
+ bool SkippingSharedTail = false;
+ for (Matcher *N : ML) {
+ if (SkippingSharedTail) {
+ ++OpcodeCounts[N->getKind()];
+ continue;
+ }
+ if (!EmittingSharedTail) {
+ auto It = SharedTailByStart.find(N);
+ if (It != SharedTailByStart.end()) {
+ ++OpcodeCounts[N->getKind()];
+ Size += JumpSize;
+ SkippingSharedTail = true;
+ continue;
+ }
+ }
Size += SizeMatcher(N, OS);
+ }
return Size;
}
@@ -1294,6 +1457,177 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
llvm_unreachable("Unreachable");
}
+void MatcherTableEmitter::collectSharedTails(const MatcherList &Root) {
+ struct Candidate {
+ SmallVector<const Matcher *, 16> Key;
+ unsigned Size = 0;
+ SmallVector<TailOccurrence, 2> Occurrences;
+ };
+ SmallVector<Candidate, 64> Candidates;
+ DenseMap<size_t, SmallVector<unsigned, 2>> CandidateBuckets;
+
+ // EmitMatcher() computes the exact encoded size, but some matcher kinds use
+ // emitter maps while doing so. Restore those maps after discovery so tail
+ // selection cannot change the IDs assigned during normal table sizing.
+ auto SavedNodeXFormMap = NodeXFormMap;
+ auto SavedNodeXForms = NodeXForms;
+ auto SavedOpcodeCounts = OpcodeCounts;
+ auto SavedValueTypeMap = ValueTypeMap;
+
+ std::function<void(const MatcherList &)> Visit = [&](const MatcherList &ML) {
+ SmallVector<const Matcher *, 32> Matchers;
+
+ for (const Matcher *N : ML) {
+ Matchers.push_back(N);
+ if (const auto *SM = dyn_cast<ScopeMatcher>(N)) {
+ for (unsigned I = 0; I != SM->getNumChildren(); ++I)
+ Visit(SM->getChild(I));
+ } else if (const auto *SOM = dyn_cast<SwitchOpcodeMatcher>(N)) {
+ for (unsigned I = 0; I != SOM->getNumCases(); ++I)
+ Visit(SOM->getCaseMatcher(I));
+ } else if (const auto *STM = dyn_cast<SwitchTypeMatcher>(N)) {
+ for (unsigned I = 0; I != STM->getNumCases(); ++I)
+ Visit(STM->getCaseMatcher(I));
+ }
+ }
+
+ if (Matchers.empty() || !isTerminalResultMatcher(Matchers.back()))
+ return;
+
+ // Walk backward over the terminal result sequence. This stops at
+ // EmitMergeInputChains, predicates, and every other matcher operation
+ // that can transfer control to a scope's next alternative.
+ unsigned SafeBegin = Matchers.size();
+ while (SafeBegin != 0 && isNonFailingResultMatcher(Matchers[SafeBegin - 1]))
+ --SafeBegin;
+
+ if (SafeBegin == Matchers.size())
+ return;
+
+ raw_null_ostream NullOS;
+ unsigned SuffixSize = 0;
+ for (unsigned I = Matchers.size(); I != SafeBegin;) {
+ --I;
+ SuffixSize += EmitMatcher(Matchers[I], 0, 0, NullOS);
+ if (SuffixSize < MinSharedResultTailSize)
+ continue;
+
+ ArrayRef<const Matcher *> Key(Matchers.data() + I, Matchers.size() - I);
+ size_t Hash = getResultTailHash(Key);
+ Candidate *Entry = nullptr;
+ for (unsigned CandidateIndex : CandidateBuckets[Hash]) {
+ Candidate &Existing = Candidates[CandidateIndex];
+ if (areResultTailsEquivalent(Existing.Key, Key)) {
+ Entry = &Existing;
+ break;
+ }
+ }
+
+ if (!Entry) {
+ unsigned CandidateIndex = Candidates.size();
+ Candidates.emplace_back();
+ CandidateBuckets[Hash].push_back(CandidateIndex);
+ Entry = &Candidates.back();
+ Entry->Key.append(Key.begin(), Key.end());
+ Entry->Size = SuffixSize;
+ } else {
+ assert(Entry->Size == SuffixSize &&
+ "equivalent result tails have different sizes");
+ }
+ Entry->Occurrences.push_back({&ML, Matchers[I]});
+ }
+ };
+
+ Visit(Root);
+ NodeXFormMap = std::move(SavedNodeXFormMap);
+ NodeXForms = std::move(SavedNodeXForms);
+ OpcodeCounts = std::move(SavedOpcodeCounts);
+ ValueTypeMap = std::move(SavedValueTypeMap);
+
+ SmallVector<Candidate *, 64> OrderedCandidates;
+ for (Candidate &Entry : Candidates)
+ if (Entry.Occurrences.size() >= 2)
+ OrderedCandidates.push_back(&Entry);
+
+ llvm::stable_sort(
+ OrderedCandidates, [](const Candidate *LHS, const Candidate *RHS) {
+ uint64_t LHSSaving =
+ uint64_t(LHS->Occurrences.size()) * LHS->Size -
+ (LHS->Size + uint64_t(LHS->Occurrences.size()) * JumpSize);
+ uint64_t RHSSaving =
+ uint64_t(RHS->Occurrences.size()) * RHS->Size -
+ (RHS->Size + uint64_t(RHS->Occurrences.size()) * JumpSize);
+ return LHSSaving > RHSSaving;
+ });
+
+ DenseSet<const MatcherList *> ClaimedLists;
+ for (Candidate *Entry : OrderedCandidates) {
+ SmallVector<TailOccurrence, 4> Available;
+ for (const TailOccurrence &Occurrence : Entry->Occurrences)
+ if (!ClaimedLists.contains(Occurrence.Matchers))
+ Available.push_back(Occurrence);
+ if (Available.size() < 2)
+ continue;
+
+ uint64_t UnsharedSize = uint64_t(Available.size()) * Entry->Size;
+ uint64_t SharedSize = Entry->Size + uint64_t(Available.size()) * JumpSize;
+ if (SharedSize >= UnsharedSize)
+ continue;
+
+ const TailOccurrence &Canonical = Available.front();
+ unsigned TailIndex = SharedTails.size();
+ SharedTails.push_back({Canonical.Matchers, Canonical.Start, Entry->Size,
+ /*Offset=*/0});
+ for (const TailOccurrence &Occurrence : Available) {
+ SharedTailByStart.try_emplace(Occurrence.Start, TailIndex);
+ ClaimedLists.insert(Occurrence.Matchers);
+ }
+ }
+}
+
+void MatcherTableEmitter::layoutSharedTails(unsigned MainTableSize) {
+ uint64_t Offset = MainTableSize;
+ for (SharedTail &Tail : SharedTails) {
+ if (!isUInt<32>(Offset))
+ report_fatal_error("SelectionDAG matcher table exceeds 32-bit offsets");
+ Tail.Offset = Offset;
+ Offset += Tail.Size;
+ }
+ if (!isUInt<32>(Offset))
+ report_fatal_error("SelectionDAG matcher table exceeds 32-bit offsets");
+}
+
+unsigned MatcherTableEmitter::EmitMatcherSuffix(const MatcherList &ML,
+ const Matcher *Start,
+ const unsigned Indent,
+ unsigned CurrentIdx,
+ raw_ostream &OS) {
+ unsigned Size = 0;
+ bool FoundStart = false;
+ for (const Matcher *N : ML) {
+ FoundStart |= N == Start;
+ if (!FoundStart)
+ continue;
+
+ if (!OmitComments)
+ OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/";
+ unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS);
+ Size += MatcherSize;
+ CurrentIdx += MatcherSize;
+ }
+ assert(FoundStart && "shared result tail is not in its matcher list");
+ return Size;
+}
+
+unsigned MatcherTableEmitter::EmitJump(const SharedTail &Tail,
+ const unsigned Indent, raw_ostream &OS) {
+ OS.indent(Indent) << "OPC_Jump, JUMP_TARGET(" << Tail.Offset << "),";
+ if (!OmitComments)
+ OS << " // Shared result tail";
+ OS << '\n';
+ return JumpSize;
+}
+
/// This function traverses the matcher tree and emits all the nodes.
/// The nodes have already been sized.
unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
@@ -1304,6 +1638,15 @@ unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
for (const Matcher *N : ML) {
if (!OmitComments)
OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/";
+
+ if (!EmittingSharedTail) {
+ auto It = SharedTailByStart.find(N);
+ if (It != SharedTailByStart.end()) {
+ Size += EmitJump(SharedTails[It->second], Indent, OS);
+ return Size;
+ }
+ }
+
unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS);
Size += MatcherSize;
CurrentIdx += MatcherSize;
@@ -1311,6 +1654,24 @@ unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
return Size;
}
+unsigned MatcherTableEmitter::EmitSharedTails(unsigned CurrentIdx,
+ raw_ostream &OS) {
+ unsigned Size = 0;
+ EmittingSharedTail = true;
+ for (const SharedTail &Tail : SharedTails) {
+ assert(CurrentIdx == Tail.Offset && "shared result tail offset mismatch");
+ if (!OmitComments)
+ OS << "\n // Shared result tail at " << Tail.Offset << "\n";
+ unsigned TailSize =
+ EmitMatcherSuffix(*Tail.Matchers, Tail.Start, 1, CurrentIdx, OS);
+ assert(TailSize == Tail.Size && "shared result tail size mismatch");
+ Size += TailSize;
+ CurrentIdx += TailSize;
+ }
+ EmittingSharedTail = false;
+ return Size;
+}
+
void MatcherTableEmitter::EmitOperandLists(raw_ostream &OS) {
OperandTable.emit(OS, [](raw_ostream &OS, uint8_t O) { OS << (unsigned)O; });
}
@@ -1633,7 +1994,10 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
bool SaveOmitComments = OmitComments;
OmitComments = true;
raw_null_ostream NullOS;
- unsigned TotalSize = MatcherEmitter.SizeMatcherList(TheMatcherList, NullOS);
+ unsigned MainTableSize =
+ MatcherEmitter.SizeMatcherList(TheMatcherList, NullOS);
+ MatcherEmitter.layoutSharedTails(MainTableSize);
+ unsigned TotalSize = MainTableSize + MatcherEmitter.getSharedTailsSize();
OmitComments = SaveOmitComments;
// Now that the matchers are sized, we can emit the code for them to the
@@ -1643,10 +2007,18 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
OS << " // this. Coverage indexes are emitted as 4 bytes,\n";
OS << " // COVERAGE_IDX_VAL handles this.\n";
OS << " #define TARGET_VAL(X) X & 255, unsigned(X) >> 8\n";
+ OS << " #define JUMP_TARGET(X) X & 255, (unsigned(X) >> 8) & 255, ";
+ OS << "(unsigned(X) >> 16) & 255, (unsigned(X) >> 24) & 255\n";
OS << " #define COVERAGE_IDX_VAL(X) X & 255, (unsigned(X) >> 8) & 255, ";
OS << "(unsigned(X) >> 16) & 255, (unsigned(X) >> 24) & 255\n";
OS << " static const uint8_t MatcherTable[] = {\n";
- TotalSize = MatcherEmitter.EmitMatcherList(TheMatcherList, 1, 0, OS);
+ unsigned EmittedSize =
+ MatcherEmitter.EmitMatcherList(TheMatcherList, 1, 0, OS);
+ assert(EmittedSize == MainTableSize &&
+ "emitted main matcher table size mismatch");
+ EmittedSize += MatcherEmitter.EmitSharedTails(EmittedSize, OS);
+ assert(EmittedSize == TotalSize && "emitted matcher table size mismatch");
+ TotalSize = EmittedSize;
OS << " }; // Total Array size is " << TotalSize << " bytes\n\n";
MatcherEmitter.EmitHistogram(OS);
@@ -1656,6 +2028,7 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
OS << " };\n\n";
OS << " #undef COVERAGE_IDX_VAL\n";
+ OS << " #undef JUMP_TARGET\n";
OS << " #undef TARGET_VAL\n";
OS << " SelectCodeCommon(N, MatcherTable, sizeof(MatcherTable),\n";
OS << " OperandLists);\n";
More information about the llvm-commits
mailing list