[llvm] [GlobalISel] Compact failure targets in large match tables (PR #202642)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 04:25:20 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-tablegen
Author: David Zbarsky (dzbarsky)
<details>
<summary>Changes</summary>
GlobalISel instruction selectors encode every failed-match destination as a 32-bit absolute offset. Compact match tables of at least 64 KiB after generation by encoding forward failure destinations as 8-bit or 16-bit relative offsets when possible and fusing the compact forms with feature checks, while retaining the existing absolute encodings as fallbacks.
An arm64 Release build with AArch64, AMDGPU, ARM, RISCV, WebAssembly, and X86 saves 313,728 B in both fully stripped `llc` (0.315%) and the upstream `llvm-driver` multicall binary (0.341%), reduces 20 generated GlobalISel tables by 309,147 B (RISCV 146,123 B; AMDGPU 104,640 B), reduces `__TEXT,__const` by 309,008 B/308,960 B, and leaves `__DATA_CONST,__const` and linked fixups unchanged.
All 71 affected GlobalISel TableGen tests, all 17 TableGen unit tests, and all 1,574 supported RISCV/AMDGPU GlobalISel codegen tests pass (nine unsupported), including focused coverage for both relative-offset widths and `GIM_Try_CheckFeatures` fusion.
Work towards #<!-- -->202616
AI tool disclosure: Co-authored with OpenAI Codex.
---
Full diff: https://github.com/llvm/llvm-project/pull/202642.diff
7 Files Affected:
- (modified) llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h (+10)
- (modified) llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h (+27-4)
- (modified) llvm/unittests/TableGen/CMakeLists.txt (+1)
- (added) llvm/unittests/TableGen/GlobalISelMatchTableTest.cpp (+47)
- (modified) llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.cpp (+78-6)
- (modified) llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.h (+13-1)
- (modified) llvm/utils/TableGen/Common/GlobalISel/MatchTable/Matchers.cpp (+3-1)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 6e3ccf1923c40..1829c87b7eb93 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -91,11 +91,19 @@ enum {
/// requires an extra opcode and iteration in the interpreter on each
/// failed match.
GIM_Try,
+ /// GIM_Try with an 8-bit forward-relative OnFail.
+ GIM_Try8,
+ /// GIM_Try with a 16-bit forward-relative OnFail.
+ GIM_Try16,
/// GIM_Try only if the feature bits match.
/// - OnFail(4) - The MatchTable entry at which to resume if the match fails.
/// - Feature(2) - Expected features
GIM_Try_CheckFeatures,
+ /// GIM_Try_CheckFeatures with an 8-bit forward-relative OnFail.
+ GIM_Try_CheckFeatures8,
+ /// GIM_Try_CheckFeatures with a 16-bit forward-relative OnFail.
+ GIM_Try_CheckFeatures16,
/// Switch over the opcode on the specified instruction
/// - InsnID(ULEB128) - Instruction ID
@@ -597,6 +605,8 @@ enum {
/// Keeping track of the number of the GI opcodes. Must be the last entry.
GIU_NumOpcodes,
};
+static_assert(GIU_NumOpcodes <= 256,
+ "GlobalISel opcodes must fit in a match table byte");
/// Provides the logic to execute GlobalISel match tables, which are used by the
/// instruction selector and instruction combiners as their engine to match and
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index e0720928c6526..87a194d141fc4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -149,16 +149,39 @@ bool GIMatchTableExecutor::executeMatchTable(
assert(CurrentIdx != ~0u && "Invalid MatchTable index");
uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
switch (MatcherOpcode) {
- case GIM_Try: {
+ case GIM_Try:
+ case GIM_Try8:
+ case GIM_Try16: {
DEBUG_WITH_TYPE(TgtExecutor::getName(),
dbgs() << CurrentIdx << ": Begin try-block\n");
- OnFailResumeAt.push_back(readU32());
+ unsigned OnFail;
+ if (MatcherOpcode == GIM_Try) {
+ OnFail = readU32();
+ } else if (MatcherOpcode == GIM_Try8) {
+ OnFail = MatchTable[CurrentIdx++];
+ OnFail += CurrentIdx;
+ } else {
+ OnFail = readU16();
+ OnFail += CurrentIdx;
+ }
+ OnFailResumeAt.push_back(OnFail);
break;
}
- case GIM_Try_CheckFeatures: {
+ case GIM_Try_CheckFeatures:
+ case GIM_Try_CheckFeatures8:
+ case GIM_Try_CheckFeatures16: {
// This is optimized so that if the feature is not present, we don't even
// modify OnFailResumeAt. Instead we directly jump to OnFail.
- unsigned OnFail = readU32();
+ unsigned OnFail;
+ if (MatcherOpcode == GIM_Try_CheckFeatures) {
+ OnFail = readU32();
+ } else if (MatcherOpcode == GIM_Try_CheckFeatures8) {
+ OnFail = MatchTable[CurrentIdx++];
+ OnFail += CurrentIdx;
+ } else {
+ OnFail = readU16();
+ OnFail += CurrentIdx;
+ }
uint16_t ExpectedBitsetID = readU16();
DEBUG_WITH_TYPE(TgtExecutor::getName(),
dbgs() << CurrentIdx
diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt
index 854f6c0f9b162..343f5696caa24 100644
--- a/llvm/unittests/TableGen/CMakeLists.txt
+++ b/llvm/unittests/TableGen/CMakeLists.txt
@@ -12,6 +12,7 @@ add_public_tablegen_target(AutomataTestTableGen)
add_llvm_unittest(TableGenTests
AutomataTest.cpp
CodeExpanderTest.cpp
+ GlobalISelMatchTableTest.cpp
ParserEntryPointTest.cpp
DISABLE_LLVM_LINK_LLVM_DYLIB
diff --git a/llvm/unittests/TableGen/GlobalISelMatchTableTest.cpp b/llvm/unittests/TableGen/GlobalISelMatchTableTest.cpp
new file mode 100644
index 0000000000000..c8c1e5c3c4712
--- /dev/null
+++ b/llvm/unittests/TableGen/GlobalISelMatchTableTest.cpp
@@ -0,0 +1,47 @@
+//===- GlobalISelMatchTableTest.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Common/GlobalISel/MatchTable/MatchTable.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::gi;
+
+TEST(GlobalISelMatchTableTest, CompactFailureTargets) {
+ MatchTable Table(/*WithCoverage=*/false, /*IsCombinerTable=*/false);
+
+ // Cross the production compaction threshold before adding representative
+ // match records. Keeping the padding before the jumps does not affect their
+ // relative distances.
+ for (unsigned I = 0; I != 8192; ++I)
+ Table << MatchTable::IntValue(8, 0);
+
+ unsigned FarLabel = Table.allocateLabelID();
+ Table << MatchTable::Opcode("GIM_Try") << MatchTable::JumpTarget(FarLabel);
+ for (unsigned I = 0; I != 300; ++I)
+ Table << MatchTable::IntValue(1, 0);
+ Table << MatchTable::Label(FarLabel);
+
+ unsigned NearLabel = Table.allocateLabelID();
+ Table << MatchTable::Opcode("GIM_Try_CheckFeatures")
+ << MatchTable::JumpTarget(NearLabel)
+ << MatchTable::NamedValue(2, "FeatureBitset")
+ << MatchTable::IntValue(1, 0) << MatchTable::Label(NearLabel);
+
+ Table.compactFailureTargets();
+
+ std::string Output;
+ raw_string_ostream OS(Output);
+ Table.emitDeclaration(OS);
+
+ EXPECT_NE(Output.find("GIM_Try16, /*Label 0*/ GIMT_Encode2(300)"),
+ std::string::npos);
+ EXPECT_NE(Output.find("GIM_Try_CheckFeatures8, /*Label 1*/ 3"),
+ std::string::npos);
+}
diff --git a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.cpp
index 7bebeee0db7df..f8b8c64c8cfa3 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.cpp
@@ -20,6 +20,10 @@ namespace gi {
// GIMT_Encode2/4/8
constexpr StringLiteral EncodeMacroName = "GIMT_Encode";
+// Avoid adding specialized bytecode to small tables where the linked size
+// reduction is negligible.
+constexpr unsigned MinMatchTableSizeForCompaction = 64 * 1024;
+
//===- Helpers ------------------------------------------------------------===//
void emitEncodingMacrosDef(raw_ostream &OS) {
@@ -73,7 +77,8 @@ static std::string getEncodedEmitStr(StringRef NamedValue, unsigned NumBytes) {
//===- MatchTableRecord ---------------------------------------------------===//
void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
- const MatchTable &Table) const {
+ const MatchTable &Table,
+ unsigned CurrentIndex) const {
bool UseLineComment =
LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows);
if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows))
@@ -96,9 +101,17 @@ void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
if (Flags & MTRF_JumpTarget) {
if (Flags & MTRF_Comment)
OS << " ";
- // TODO: Could encode this AOT to speed up build of generated file
- OS << getEncodedEmitStr(llvm::to_string(Table.getLabelIndex(LabelID)),
- NumElements);
+ unsigned Target = Table.getLabelIndex(LabelID);
+ if (Flags & MTRF_RelativeJumpTarget) {
+ assert(Target >= CurrentIndex + NumElements &&
+ "Relative jumps must be forward");
+ Target -= CurrentIndex + NumElements;
+ }
+ // TODO: Could encode this AOT to speed up build of generated file.
+ if (NumElements == 1)
+ OS << Target;
+ else
+ OS << getEncodedEmitStr(llvm::to_string(Target), NumElements);
}
if (Flags & MTRF_CommaFollows) {
@@ -204,7 +217,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const {
static constexpr unsigned BaseIndent = 4;
unsigned Indentation = 0;
OS << " constexpr static uint8_t MatchTable" << ID << "[] = {";
- LineBreak.emit(OS, true, *this);
+ LineBreak.emit(OS, true, *this, 0);
// We want to display the table index of each line in a consistent
// manner. It has to appear as a column on the left side of the table.
@@ -236,7 +249,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const {
if (I->Flags & MatchTableRecord::MTRF_Indent)
Indentation += 2;
- I->emit(OS, LineBreakIsNext, *this);
+ I->emit(OS, LineBreakIsNext, *this, CurIndex);
if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
BeginLine();
@@ -249,5 +262,64 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const {
OS << "}; // Size: " << CurrentSize << " bytes\n";
}
+void MatchTable::rebuildLabelMap() {
+ LabelMap.clear();
+ CurrentSize = 0;
+ for (const MatchTableRecord &Record : Contents) {
+ if (Record.Flags & MatchTableRecord::MTRF_Label)
+ defineLabel(Record.LabelID);
+ CurrentSize += Record.size();
+ }
+}
+
+void MatchTable::compactFailureTargets() {
+ if (CurrentSize < MinMatchTableSizeForCompaction)
+ return;
+
+ std::vector<unsigned> RecordOffsets;
+ RecordOffsets.reserve(Contents.size());
+ unsigned Offset = 0;
+ for (const MatchTableRecord &Record : Contents) {
+ RecordOffsets.push_back(Offset);
+ Offset += Record.size();
+ }
+
+ for (unsigned I = 0, E = Contents.size(); I != E; ++I) {
+ MatchTableRecord &Opcode = Contents[I];
+ StringRef OpcodeName = Opcode.EmitStr;
+ if (OpcodeName != "GIM_Try" && OpcodeName != "GIM_Try_CheckFeatures")
+ continue;
+
+ unsigned JumpRecord = I + 1;
+ while (JumpRecord != E && Contents[JumpRecord].size() == 0)
+ ++JumpRecord;
+ assert(JumpRecord != E &&
+ (Contents[JumpRecord].Flags & MatchTableRecord::MTRF_JumpTarget));
+
+ MatchTableRecord &Jump = Contents[JumpRecord];
+ unsigned Target = getLabelIndex(Jump.LabelID);
+ unsigned JumpOffset = RecordOffsets[JumpRecord];
+ assert(Target > JumpOffset && "GIM_Try targets must be forward");
+
+ unsigned NumBytes;
+ StringRef Suffix;
+ unsigned Distance = Target - JumpOffset;
+ if (Distance <= 256) {
+ NumBytes = 1;
+ Suffix = "8";
+ } else if (Distance <= 65537) {
+ NumBytes = 2;
+ Suffix = "16";
+ } else {
+ continue;
+ }
+
+ Opcode.EmitStr += Suffix;
+ Jump.makeRelativeJumpTarget(NumBytes);
+ }
+
+ rebuildLabelMap();
+}
+
} // namespace gi
} // namespace llvm
diff --git a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.h
index a7ac109fb99c0..23a8ea1cc5ff5 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/MatchTable.h
@@ -63,6 +63,8 @@ struct MatchTableRecord {
/// Causes the formatter to not use encoding macros to emit this multi-byte
/// value.
MTRF_PreEncoded = 0x80,
+ /// Causes a jump target to be emitted relative to the end of this record.
+ MTRF_RelativeJumpTarget = 0x100,
};
/// When MTRF_Label or MTRF_JumpTarget is used, indicates a label id to
@@ -99,8 +101,15 @@ struct MatchTableRecord {
NumElements = 0;
}
+ void makeRelativeJumpTarget(unsigned NumBytes) {
+ assert(Flags & MTRF_JumpTarget);
+ assert((NumBytes == 1 || NumBytes == 2) && "Unsupported jump size");
+ Flags |= MTRF_RelativeJumpTarget;
+ NumElements = NumBytes;
+ }
+
void emit(raw_ostream &OS, bool LineBreakNextAfterThis,
- const MatchTable &Table) const;
+ const MatchTable &Table, unsigned CurrentIndex) const;
unsigned size() const { return NumElements; }
};
@@ -124,6 +133,8 @@ class MatchTable {
/// Whether this table is for the GISel combiner.
bool IsCombinerTable;
+ void rebuildLabelMap();
+
public:
static MatchTableRecord LineBreak;
static MatchTableRecord Comment(StringRef Comment);
@@ -142,6 +153,7 @@ class MatchTable {
bool isWithCoverage() const { return IsWithCoverage; }
bool isCombiner() const { return IsCombinerTable; }
+ void compactFailureTargets();
void push_back(const MatchTableRecord &Value) {
if (Value.Flags & MatchTableRecord::MTRF_Label)
diff --git a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/Matchers.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/Matchers.cpp
index cc1089fc316c7..66b2ccb305097 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/MatchTable/Matchers.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/MatchTable/Matchers.cpp
@@ -141,7 +141,9 @@ MatchTable llvm::gi::buildMatchTable(ArrayRef<Matcher *> Rules,
for (Matcher *Rule : Rules)
Rule->emit(Table);
- return Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
+ Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
+ Table.compactFailureTargets();
+ return Table;
}
template <class Range> static bool matchersRecordOperand(Range &&R) {
``````````
</details>
https://github.com/llvm/llvm-project/pull/202642
More information about the llvm-commits
mailing list