[llvm] [SelectionDAG] Share repeated terminal matcher tails (PR #202638)
David Zbarsky via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 20:50:29 PDT 2026
https://github.com/dzbarsky updated https://github.com/llvm/llvm-project/pull/202638
>From e64924e61e26bd4056508e013f3b49408b02c34a Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 23:36:50 -0400
Subject: [PATCH] [SelectionDAG] Share repeated terminal matcher tails
Add an absolute jump opcode to the SelectionDAG matcher interpreter. DAGISelMatcherEmitter identifies identical terminal result suffixes, emits one suffix after the main matcher table, and replaces the occurrences with jumps.
Only non-failing result operations participate. EmitMergeInputChains and every operation that can reject a match remain before the jump. Disable sharing when matcher coverage instrumentation is enabled, and retain histogram accounting for logical matchers replaced by jumps.
With comments disabled, EmitMatcher() output identifies the encoded suffix. Use that output directly instead of maintaining separate hash and equality functions for every result matcher. Identical output guarantees identical generated matcher instructions; different output can only miss a sharing opportunity.
On current main, the generated AArch64, AMDGPU, and ARM matcher tables shrink by 83,533 bytes at the 48-byte threshold. On the LLVM 22 all-target stack, including R600, raw matcher tables shrink by 87,470 bytes. In the LLVM 22 Bazel build, the stripped multicall binary shrinks from 142,889,168 to 142,806,608 bytes (-82,560), and standalone llc shrinks from 78,497,384 to 78,398,312 bytes (-99,072).
The simpler suffix key produces byte-identical AArch64, AMDGPU, ARM, Mips, WebAssembly, and X86 matcher output compared with the previous implementation. For AArch64, median llvm-tblgen user time changes from 1.88 to 1.91 seconds, peak memory is unchanged, and llvm-tblgen shrinks by 624 bytes.
Forced AArch64 SelectionDAG code generation produces byte-identical object files for a minimal integer add and an SDOT case. Validate shared identical tails, distinct tails, chain merges before jumps, and disabled sharing under coverage instrumentation with dag-isel-result-tail-sharing.td.
---
llvm/include/llvm/CodeGen/SelectionDAGISel.h | 6 +
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 10 +
.../TableGen/dag-isel-result-tail-sharing.td | 109 +++++++
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 283 +++++++++++++++++-
4 files changed, 405 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/TableGen/dag-isel-result-tail-sharing.td
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 7b406ef4b4cb0..1cd0f332fd66e 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -282,6 +282,10 @@ class LLVM_ABI SelectionDAGISel {
OPC_CheckImmAllZerosV,
OPC_CheckFoldableChainNode,
+ // Transfer control to a shared result tail. The target is encoded as a
+ // 32-bit absolute MatcherTable offset.
+ OPC_Jump,
+
OPC_EmitInteger,
// Space-optimized forms that implicitly encode integer VT.
OPC_EmitIntegerI8,
@@ -352,6 +356,8 @@ class LLVM_ABI SelectionDAGISel {
// Contains 32-bit offset in table for pattern being selected
OPC_Coverage
};
+ static_assert(OPC_Coverage < 256,
+ "SelectionDAG matcher opcodes must fit in one byte");
enum {
OPFL_None = 0, // Node has no chain or glue input and isn't variadic.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 5ae52cae771fb..24387390bc5f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -4104,6 +4104,16 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
continue;
}
+ case OPC_Jump: {
+ uint32_t Target = MatcherTable[MatcherIndex++];
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 8;
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 16;
+ Target |= static_cast<uint32_t>(MatcherTable[MatcherIndex++]) << 24;
+ assert(Target < TableSize && "Invalid SelectionDAG matcher jump");
+ MatcherIndex = Target;
+ continue;
+ }
+
case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
diff --git a/llvm/test/TableGen/dag-isel-result-tail-sharing.td b/llvm/test/TableGen/dag-isel-result-tail-sharing.td
new file mode 100644
index 0000000000000..f3c9688e5ae6f
--- /dev/null
+++ b/llvm/test/TableGen/dag-isel-result-tail-sharing.td
@@ -0,0 +1,109 @@
+// RUN: llvm-tblgen -gen-dag-isel -I %p/../../include %s -o - \
+// RUN: | FileCheck --check-prefix=SHARED %s
+// RUN: llvm-tblgen -gen-dag-isel -DSTRUCTURAL_DIFFERENCE \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=DIFFERENT %s
+// RUN: llvm-tblgen -gen-dag-isel -DFAILURE_PREFIX \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=FAILURE %s
+// RUN: llvm-tblgen -gen-dag-isel -instrument-coverage \
+// RUN: -I %p/../../include %s -o - | FileCheck --check-prefix=COVERAGE %s
+
+include "llvm/Target/Target.td"
+
+def TestInstrInfo : InstrInfo;
+def TestTarget : Target {
+ let InstructionSet = TestInstrInfo;
+}
+
+def R0 : Register<"r0">;
+def GPR : RegisterClass<"TestTarget", [i32], 32, (add R0)>;
+
+def WRAP : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$src);
+}
+
+def WRAP_ALT : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$src);
+}
+
+def PAIR : Instruction {
+ let OutOperandList = (outs GPR:$dst);
+ let InOperandList = (ins GPR:$lhs, GPR:$rhs);
+}
+
+#ifndef STRUCTURAL_DIFFERENCE
+#ifndef FAILURE_PREFIX
+// The ADD and SUB patterns have different predicates but identical generated
+// results. Their result suffix is long enough to share.
+def : Pat<
+ (add GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (sub GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// SHARED: OPC_Jump, JUMP_TARGET([[TAIL:[0-9]+]]),
+// SHARED: OPC_Jump, JUMP_TARGET([[TAIL]]),
+// SHARED: // Shared result tail at [[TAIL]]
+// SHARED: OPC_EmitNode
+// SHARED: OPC_MorphNodeTo
+#endif
+#endif
+
+#ifdef STRUCTURAL_DIFFERENCE
+// These result sequences have the same matcher kinds and encoded sizes, but
+// different instruction records. The emitted matcher instructions must keep them separate.
+def : Pat<
+ (add GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (sub GPR:$x, GPR:$y),
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (WRAP_ALT (WRAP_ALT (WRAP_ALT (WRAP_ALT
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// DIFFERENT-NOT: OPC_Jump
+#endif
+
+#ifdef FAILURE_PREFIX
+// EmitMergeInputChains may fail after inspecting the matched DAG. Each pattern
+// must execute its own merge before entering the shared non-failing suffix.
+def int_chain_a : Intrinsic<
+ [llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
+def int_chain_b : Intrinsic<
+ [llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
+
+def : Pat<
+ (int_chain_a GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+def : Pat<
+ (int_chain_b GPR:$x, GPR:$y),
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (WRAP (WRAP (WRAP (WRAP (WRAP (WRAP
+ (PAIR GPR:$x, GPR:$y)))))))))))))))))>;
+
+// FAILURE: OPC_EmitMergeInputChains
+// FAILURE-NEXT: {{.*}}OPC_Jump, JUMP_TARGET([[TAIL:[0-9]+]]),
+// FAILURE: OPC_EmitMergeInputChains
+// FAILURE-NEXT: {{.*}}OPC_Jump, JUMP_TARGET([[TAIL]]),
+// FAILURE: // Shared result tail at [[TAIL]]
+// FAILURE-NOT: OPC_EmitMergeInputChains
+// FAILURE: OPC_EmitNode
+#endif
+
+// Coverage output must preserve one coverage record per source pattern.
+// COVERAGE-NOT: OPC_Jump
+// COVERAGE-COUNT-2: OPC_Coverage
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 3d69c96ebd900..3b8c46905499f 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -18,6 +18,7 @@
#include "Common/CodeGenTarget.h"
#include "DAGISelMatcher.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/TinyPtrVector.h"
@@ -34,6 +35,8 @@ enum {
IndexWidth = 7,
FullIndexWidth = IndexWidth + 4,
HistOpcWidth = 40,
+ MinSharedResultTailSize = 48,
+ JumpSize = 5,
};
static cl::OptionCategory DAGISelCat("Options for -gen-dag-isel");
@@ -50,6 +53,18 @@ static cl::opt<bool> InstrumentCoverage(
namespace {
class MatcherTableEmitter {
+ struct TailOccurrence {
+ const MatcherList *Matchers;
+ const Matcher *Start;
+ };
+
+ struct SharedTail {
+ const MatcherList *Matchers;
+ const Matcher *Start;
+ unsigned Size;
+ unsigned Offset = 0;
+ };
+
const CodeGenDAGPatterns &CGP;
SmallVector<unsigned, Matcher::HighestKind + 1> OpcodeCounts;
@@ -78,6 +93,9 @@ class MatcherTableEmitter {
SequenceToOffsetTable<std::vector<uint8_t>> OperandTable;
+ SmallVector<SharedTail, 16> SharedTails;
+ DenseMap<const Matcher *, unsigned> SharedTailByStart;
+
unsigned getPatternIdxFromTable(std::string &&P, std::string &&include_loc) {
const auto [It, Inserted] =
VecPatterns.try_emplace(std::move(P), VecPatterns.size());
@@ -205,6 +223,9 @@ class MatcherTableEmitter {
else
NodePredicates.push_back(TP);
}
+
+ if (!InstrumentCoverage)
+ collectSharedTails(TheMatcherList);
}
unsigned EmitMatcherList(const MatcherList &ML, const unsigned Indent,
@@ -222,7 +243,47 @@ class MatcherTableEmitter {
void EmitPatternMatchTable(raw_ostream &OS);
+ void layoutSharedTails(unsigned MainTableSize);
+
+ unsigned EmitSharedTails(unsigned CurrentIdx, raw_ostream &OS);
+
+ unsigned getSharedTailsSize() const {
+ unsigned Size = 0;
+ for (const SharedTail &Tail : SharedTails)
+ Size += Tail.Size;
+ return Size;
+ }
+
private:
+ static bool isNonFailingResultMatcher(const Matcher *N) {
+ switch (N->getKind()) {
+ case Matcher::EmitInteger:
+ case Matcher::EmitRegister:
+ case Matcher::EmitConvertToTarget:
+ case Matcher::EmitCopyToReg:
+ case Matcher::EmitNode:
+ case Matcher::EmitNodeXForm:
+ case Matcher::CompleteMatch:
+ case Matcher::MorphNodeTo:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static bool isTerminalResultMatcher(const Matcher *N) {
+ return isa<CompleteMatchMatcher>(N) || isa<MorphNodeToMatcher>(N);
+ }
+
+ void collectSharedTails(const MatcherList &ML);
+
+ unsigned EmitMatcherSuffix(const MatcherList &ML, const Matcher *Start,
+ const unsigned Indent, unsigned CurrentIdx,
+ raw_ostream &OS);
+
+ unsigned EmitJump(const SharedTail &Tail, const unsigned Indent,
+ raw_ostream &OS);
+
// Reorder ValueType indices by usage frequency (most common -> index 0).
// Updates the indices directly in ValueTypeMap.
void sortValueTypeByHwModeByFrequency() {
@@ -377,8 +438,21 @@ static std::string getIncludePath(const Record *R) {
unsigned MatcherTableEmitter::SizeMatcherList(MatcherList &ML,
raw_ostream &OS) {
unsigned Size = 0;
- for (Matcher *N : ML)
+ bool SkippingSharedTail = false;
+ for (Matcher *N : ML) {
+ if (SkippingSharedTail) {
+ ++OpcodeCounts[N->getKind()];
+ continue;
+ }
+ auto It = SharedTailByStart.find(N);
+ if (It != SharedTailByStart.end()) {
+ ++OpcodeCounts[N->getKind()];
+ Size += JumpSize;
+ SkippingSharedTail = true;
+ continue;
+ }
Size += SizeMatcher(N, OS);
+ }
return Size;
}
@@ -1294,6 +1368,174 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
llvm_unreachable("Unreachable");
}
+void MatcherTableEmitter::collectSharedTails(const MatcherList &Root) {
+ struct Candidate {
+ unsigned Size = 0;
+ SmallVector<TailOccurrence, 2> Occurrences;
+ };
+ SmallVector<Candidate, 64> Candidates;
+ StringMap<unsigned> CandidateByKey;
+
+ // EmitMatcher() computes the exact encoded size, but some matcher kinds use
+ // emitter maps while doing so. Restore those maps after discovery so tail
+ // selection cannot change the IDs assigned during normal table sizing.
+ auto SavedNodeXFormMap = NodeXFormMap;
+ auto SavedNodeXForms = NodeXForms;
+ auto SavedOpcodeCounts = OpcodeCounts;
+ auto SavedValueTypeMap = ValueTypeMap;
+ bool SavedOmitComments = OmitComments;
+ OmitComments = true;
+
+ std::function<void(const MatcherList &)> Visit = [&](const MatcherList &ML) {
+ SmallVector<const Matcher *, 32> Matchers;
+
+ for (const Matcher *N : ML) {
+ Matchers.push_back(N);
+ if (const auto *SM = dyn_cast<ScopeMatcher>(N)) {
+ for (unsigned I = 0; I != SM->getNumChildren(); ++I)
+ Visit(SM->getChild(I));
+ } else if (const auto *SOM = dyn_cast<SwitchOpcodeMatcher>(N)) {
+ for (unsigned I = 0; I != SOM->getNumCases(); ++I)
+ Visit(SOM->getCaseMatcher(I));
+ } else if (const auto *STM = dyn_cast<SwitchTypeMatcher>(N)) {
+ for (unsigned I = 0; I != STM->getNumCases(); ++I)
+ Visit(STM->getCaseMatcher(I));
+ }
+ }
+
+ if (Matchers.empty() || !isTerminalResultMatcher(Matchers.back()))
+ return;
+
+ // Walk backward over the terminal result sequence. This stops at
+ // EmitMergeInputChains, predicates, and every other matcher operation
+ // that can transfer control to a scope's next alternative.
+ unsigned SafeBegin = Matchers.size();
+ while (SafeBegin != 0 && isNonFailingResultMatcher(Matchers[SafeBegin - 1]))
+ --SafeBegin;
+
+ if (SafeBegin == Matchers.size())
+ return;
+
+ unsigned SuffixSize = 0;
+ std::string SuffixKey;
+ for (unsigned I = Matchers.size(); I != SafeBegin;) {
+ --I;
+
+ // With comments disabled, EmitMatcher()'s output identifies the encoded
+ // suffix. Emit each matcher once to preserve the emitter ID sequence.
+ std::string Fragment;
+ {
+ raw_string_ostream FragmentOS(Fragment);
+ SuffixSize += EmitMatcher(Matchers[I], 0, 0, FragmentOS);
+ }
+ SuffixKey.insert(0, Fragment);
+ if (SuffixSize < MinSharedResultTailSize)
+ continue;
+
+ auto [It, Inserted] =
+ CandidateByKey.try_emplace(SuffixKey, Candidates.size());
+ if (Inserted) {
+ Candidates.emplace_back();
+ Candidates.back().Size = SuffixSize;
+ } else {
+ assert(Candidates[It->second].Size == SuffixSize &&
+ "identical result tails have different sizes");
+ }
+ Candidates[It->second].Occurrences.push_back({&ML, Matchers[I]});
+ }
+ };
+
+ Visit(Root);
+ NodeXFormMap = std::move(SavedNodeXFormMap);
+ NodeXForms = std::move(SavedNodeXForms);
+ OpcodeCounts = std::move(SavedOpcodeCounts);
+ ValueTypeMap = std::move(SavedValueTypeMap);
+ OmitComments = SavedOmitComments;
+
+ SmallVector<Candidate *, 64> OrderedCandidates;
+ for (Candidate &Entry : Candidates)
+ if (Entry.Occurrences.size() >= 2)
+ OrderedCandidates.push_back(&Entry);
+
+ llvm::stable_sort(
+ OrderedCandidates, [](const Candidate *LHS, const Candidate *RHS) {
+ uint64_t LHSSaving =
+ uint64_t(LHS->Occurrences.size()) * LHS->Size -
+ (LHS->Size + uint64_t(LHS->Occurrences.size()) * JumpSize);
+ uint64_t RHSSaving =
+ uint64_t(RHS->Occurrences.size()) * RHS->Size -
+ (RHS->Size + uint64_t(RHS->Occurrences.size()) * JumpSize);
+ return LHSSaving > RHSSaving;
+ });
+
+ DenseSet<const MatcherList *> ClaimedLists;
+ for (Candidate *Entry : OrderedCandidates) {
+ SmallVector<TailOccurrence, 4> Available;
+ for (const TailOccurrence &Occurrence : Entry->Occurrences)
+ if (!ClaimedLists.contains(Occurrence.Matchers))
+ Available.push_back(Occurrence);
+ if (Available.size() < 2)
+ continue;
+
+ uint64_t UnsharedSize = uint64_t(Available.size()) * Entry->Size;
+ uint64_t SharedSize = Entry->Size + uint64_t(Available.size()) * JumpSize;
+ if (SharedSize >= UnsharedSize)
+ continue;
+
+ const TailOccurrence &Canonical = Available.front();
+ unsigned TailIndex = SharedTails.size();
+ SharedTails.push_back({Canonical.Matchers, Canonical.Start, Entry->Size,
+ /*Offset=*/0});
+ for (const TailOccurrence &Occurrence : Available) {
+ SharedTailByStart.try_emplace(Occurrence.Start, TailIndex);
+ ClaimedLists.insert(Occurrence.Matchers);
+ }
+ }
+}
+
+void MatcherTableEmitter::layoutSharedTails(unsigned MainTableSize) {
+ uint64_t Offset = MainTableSize;
+ for (SharedTail &Tail : SharedTails) {
+ if (!isUInt<32>(Offset))
+ report_fatal_error("SelectionDAG matcher table exceeds 32-bit offsets");
+ Tail.Offset = Offset;
+ Offset += Tail.Size;
+ }
+ if (!isUInt<32>(Offset))
+ report_fatal_error("SelectionDAG matcher table exceeds 32-bit offsets");
+}
+
+unsigned MatcherTableEmitter::EmitMatcherSuffix(const MatcherList &ML,
+ const Matcher *Start,
+ const unsigned Indent,
+ unsigned CurrentIdx,
+ raw_ostream &OS) {
+ unsigned Size = 0;
+ bool FoundStart = false;
+ for (const Matcher *N : ML) {
+ FoundStart |= N == Start;
+ if (!FoundStart)
+ continue;
+
+ if (!OmitComments)
+ OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/";
+ unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS);
+ Size += MatcherSize;
+ CurrentIdx += MatcherSize;
+ }
+ assert(FoundStart && "shared result tail is not in its matcher list");
+ return Size;
+}
+
+unsigned MatcherTableEmitter::EmitJump(const SharedTail &Tail,
+ const unsigned Indent, raw_ostream &OS) {
+ OS.indent(Indent) << "OPC_Jump, JUMP_TARGET(" << Tail.Offset << "),";
+ if (!OmitComments)
+ OS << " // Shared result tail";
+ OS << '\n';
+ return JumpSize;
+}
+
/// This function traverses the matcher tree and emits all the nodes.
/// The nodes have already been sized.
unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
@@ -1304,6 +1546,13 @@ unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
for (const Matcher *N : ML) {
if (!OmitComments)
OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/";
+
+ auto It = SharedTailByStart.find(N);
+ if (It != SharedTailByStart.end()) {
+ Size += EmitJump(SharedTails[It->second], Indent, OS);
+ return Size;
+ }
+
unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS);
Size += MatcherSize;
CurrentIdx += MatcherSize;
@@ -1311,6 +1560,22 @@ unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
return Size;
}
+unsigned MatcherTableEmitter::EmitSharedTails(unsigned CurrentIdx,
+ raw_ostream &OS) {
+ unsigned Size = 0;
+ for (const SharedTail &Tail : SharedTails) {
+ assert(CurrentIdx == Tail.Offset && "shared result tail offset mismatch");
+ if (!OmitComments)
+ OS << "\n // Shared result tail at " << Tail.Offset << "\n";
+ unsigned TailSize =
+ EmitMatcherSuffix(*Tail.Matchers, Tail.Start, 1, CurrentIdx, OS);
+ assert(TailSize == Tail.Size && "shared result tail size mismatch");
+ Size += TailSize;
+ CurrentIdx += TailSize;
+ }
+ return Size;
+}
+
void MatcherTableEmitter::EmitOperandLists(raw_ostream &OS) {
OperandTable.emit(OS, [](raw_ostream &OS, uint8_t O) { OS << (unsigned)O; });
}
@@ -1633,7 +1898,10 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
bool SaveOmitComments = OmitComments;
OmitComments = true;
raw_null_ostream NullOS;
- unsigned TotalSize = MatcherEmitter.SizeMatcherList(TheMatcherList, NullOS);
+ unsigned MainTableSize =
+ MatcherEmitter.SizeMatcherList(TheMatcherList, NullOS);
+ MatcherEmitter.layoutSharedTails(MainTableSize);
+ unsigned TotalSize = MainTableSize + MatcherEmitter.getSharedTailsSize();
OmitComments = SaveOmitComments;
// Now that the matchers are sized, we can emit the code for them to the
@@ -1643,10 +1911,18 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
OS << " // this. Coverage indexes are emitted as 4 bytes,\n";
OS << " // COVERAGE_IDX_VAL handles this.\n";
OS << " #define TARGET_VAL(X) X & 255, unsigned(X) >> 8\n";
+ OS << " #define JUMP_TARGET(X) X & 255, (unsigned(X) >> 8) & 255, ";
+ OS << "(unsigned(X) >> 16) & 255, (unsigned(X) >> 24) & 255\n";
OS << " #define COVERAGE_IDX_VAL(X) X & 255, (unsigned(X) >> 8) & 255, ";
OS << "(unsigned(X) >> 16) & 255, (unsigned(X) >> 24) & 255\n";
OS << " static const uint8_t MatcherTable[] = {\n";
- TotalSize = MatcherEmitter.EmitMatcherList(TheMatcherList, 1, 0, OS);
+ unsigned EmittedSize =
+ MatcherEmitter.EmitMatcherList(TheMatcherList, 1, 0, OS);
+ assert(EmittedSize == MainTableSize &&
+ "emitted main matcher table size mismatch");
+ EmittedSize += MatcherEmitter.EmitSharedTails(EmittedSize, OS);
+ assert(EmittedSize == TotalSize && "emitted matcher table size mismatch");
+ TotalSize = EmittedSize;
OS << " }; // Total Array size is " << TotalSize << " bytes\n\n";
MatcherEmitter.EmitHistogram(OS);
@@ -1656,6 +1932,7 @@ void llvm::EmitMatcherTable(MatcherList &TheMatcherList,
OS << " };\n\n";
OS << " #undef COVERAGE_IDX_VAL\n";
+ OS << " #undef JUMP_TARGET\n";
OS << " #undef TARGET_VAL\n";
OS << " SelectCodeCommon(N, MatcherTable, sizeof(MatcherTable),\n";
OS << " OperandLists);\n";
More information about the llvm-commits
mailing list