[llvm] [SelectionDAG] Fuse frequent adjacent matcher operations (PR #202635)
David Zbarsky via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 20:49:07 PDT 2026
https://github.com/dzbarsky updated https://github.com/llvm/llvm-project/pull/202635
>From 4545bb1b96fb307aa1f6e1f6dfba8d88be2eb3a2 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 20:31:52 -0400
Subject: [PATCH 1/3] [SelectionDAG] Encode common integer constants in matcher
opcodes
DAGISelMatcherEmitter currently writes an EmitInteger opcode followed by a signed-VBR value for common i32 and i64 constants. Add one-byte opcodes for i32 values -1 through 8 and i64 values -1 through 7. SelectionDAGISel decodes each contiguous opcode range with arithmetic and performs the same getSignedConstant call.
The generated AArch64, AMDGPU, ARM, Mips, WebAssembly, and X86 matcher tables shrink from 2,246,495 to 2,200,010 bytes, saving 46,485 bytes (2.07%). The equivalent LLVM 22 encoding changed mean user CPU for a ten-run AMDGPU instruction-selection workload from 1.829 to 1.828 seconds (-0.055%), and generated AArch64, AMDGPU, RISCV, and X86 assembly remained byte-identical.
Add a compile-time assertion that BuiltinOpcodes still fits uint8_t. Validate DAGDefaultOps.td and dag-isel-subregs.td with llvm-tblgen and FileCheck, and compile DAGISelMatcherEmitter.cpp and SelectionDAGISel.cpp with Clang.
---
llvm/include/llvm/CodeGen/SelectionDAGISel.h | 22 ++++++++++++++
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 30 +++++++++++++++++++
llvm/test/TableGen/DAGDefaultOps.td | 8 ++---
llvm/test/TableGen/dag-isel-subregs.td | 2 +-
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 22 ++++++++++++++
5 files changed, 79 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 7b406ef4b4cb0..98f5c9e63e26a 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -288,6 +288,27 @@ class LLVM_ABI SelectionDAGISel {
OPC_EmitIntegerI16,
OPC_EmitIntegerI32,
OPC_EmitIntegerI64,
+ // Space-optimized forms that encode common integer types and values in
+ // the opcode. Keep each type's values contiguous for inexpensive decoding.
+ OPC_EmitIntegerI32Neg1,
+ OPC_EmitIntegerI32_0,
+ OPC_EmitIntegerI32_1,
+ OPC_EmitIntegerI32_2,
+ OPC_EmitIntegerI32_3,
+ OPC_EmitIntegerI32_4,
+ OPC_EmitIntegerI32_5,
+ OPC_EmitIntegerI32_6,
+ OPC_EmitIntegerI32_7,
+ OPC_EmitIntegerI32_8,
+ OPC_EmitIntegerI64Neg1,
+ OPC_EmitIntegerI64_0,
+ OPC_EmitIntegerI64_1,
+ OPC_EmitIntegerI64_2,
+ OPC_EmitIntegerI64_3,
+ OPC_EmitIntegerI64_4,
+ OPC_EmitIntegerI64_5,
+ OPC_EmitIntegerI64_6,
+ OPC_EmitIntegerI64_7,
OPC_EmitIntegerByHwMode,
OPC_EmitIntegerByHwMode0,
OPC_EmitRegister,
@@ -352,6 +373,7 @@ class LLVM_ABI SelectionDAGISel {
// Contains 32-bit offset in table for pattern being selected
OPC_Coverage
};
+ static_assert(OPC_Coverage < 256, "DAGISel opcodes must fit in one byte");
enum {
OPFL_None = 0, // Node has no chain or glue input and isn't variadic.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 5ae52cae771fb..9ee60bbad2106 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3997,6 +3997,36 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
continue;
}
+ case OPC_EmitIntegerI32Neg1:
+ case OPC_EmitIntegerI32_0:
+ case OPC_EmitIntegerI32_1:
+ case OPC_EmitIntegerI32_2:
+ case OPC_EmitIntegerI32_3:
+ case OPC_EmitIntegerI32_4:
+ case OPC_EmitIntegerI32_5:
+ case OPC_EmitIntegerI32_6:
+ case OPC_EmitIntegerI32_7:
+ case OPC_EmitIntegerI32_8:
+ case OPC_EmitIntegerI64Neg1:
+ case OPC_EmitIntegerI64_0:
+ case OPC_EmitIntegerI64_1:
+ case OPC_EmitIntegerI64_2:
+ case OPC_EmitIntegerI64_3:
+ case OPC_EmitIntegerI64_4:
+ case OPC_EmitIntegerI64_5:
+ case OPC_EmitIntegerI64_6:
+ case OPC_EmitIntegerI64_7: {
+ bool IsI32 = Opcode <= OPC_EmitIntegerI32_8;
+ unsigned FirstOpcode =
+ IsI32 ? OPC_EmitIntegerI32Neg1 : OPC_EmitIntegerI64Neg1;
+ MVT::SimpleValueType VT = IsI32 ? MVT::i32 : MVT::i64;
+ int64_t Val = static_cast<int64_t>(Opcode - FirstOpcode) - 1;
+ RecordedNodes.emplace_back(
+ CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
+ /*isTarget=*/true),
+ nullptr);
+ continue;
+ }
case OPC_EmitInteger:
case OPC_EmitIntegerI8:
case OPC_EmitIntegerI16:
diff --git a/llvm/test/TableGen/DAGDefaultOps.td b/llvm/test/TableGen/DAGDefaultOps.td
index 8c7e3de58b734..4688ce73cd9a7 100644
--- a/llvm/test/TableGen/DAGDefaultOps.td
+++ b/llvm/test/TableGen/DAGDefaultOps.td
@@ -77,20 +77,20 @@ def MulIRRPat : Pat<(mul i32:$x, i32:$y), (MulIRR Reg:$x, Reg:$y)>;
// ADD: SwitchOpcode{{.*}}TARGET_VAL(ISD::ADD)
// ADD-NEXT: OPC_RecordChild0
// ADD-NEXT: OPC_RecordChild1
-// ADD-NEXT: OPC_EmitIntegerI32, 0
+// ADD-NEXT: OPC_EmitIntegerI32_0
// ADD-NEXT: OPC_MorphNodeTo1None, TARGET_VAL(::AddRRI)
// ADDINT: SwitchOpcode{{.*}}TARGET_VAL(ISD::INTRINSIC_WO_CHAIN)
// ADDINT-NEXT: OPC_CheckChild0Integer
// ADDINT-NEXT: OPC_RecordChild1
// ADDINT-NEXT: OPC_RecordChild2
-// ADDINT-NEXT: OPC_EmitIntegerI32, 1
+// ADDINT-NEXT: OPC_EmitIntegerI32_1
// ADDINT-NEXT: OPC_MorphNodeTo1None, TARGET_VAL(::AddRRI)
// SUB: SwitchOpcode{{.*}}TARGET_VAL(ISD::SUB)
// SUB-NEXT: OPC_RecordChild0
// SUB-NEXT: OPC_RecordChild1
-// SUB-NEXT: OPC_EmitIntegerI32, 0
+// SUB-NEXT: OPC_EmitIntegerI32_0
// SUB-NEXT: OPC_MorphNodeTo1None, TARGET_VAL(::SubRRI)
// MULINT: SwitchOpcode{{.*}}TARGET_VAL(ISD::INTRINSIC_W_CHAIN)
@@ -105,5 +105,5 @@ def MulIRRPat : Pat<(mul i32:$x, i32:$y), (MulIRR Reg:$x, Reg:$y)>;
// MUL: SwitchOpcode{{.*}}TARGET_VAL(ISD::MUL)
// MUL-NEXT: OPC_RecordChild0
// MUL-NEXT: OPC_RecordChild1
-// MUL-NEXT: OPC_EmitIntegerI32, 0
+// MUL-NEXT: OPC_EmitIntegerI32_0
// MUL-NEXT: OPC_MorphNodeTo1None, TARGET_VAL(::MulIRR)
diff --git a/llvm/test/TableGen/dag-isel-subregs.td b/llvm/test/TableGen/dag-isel-subregs.td
index e0e94e033a9b2..da90ad0ced635 100644
--- a/llvm/test/TableGen/dag-isel-subregs.td
+++ b/llvm/test/TableGen/dag-isel-subregs.td
@@ -4,7 +4,7 @@ include "reg-with-subregs-common.td"
// CHECK-LABEL: OPC_CheckOpcode, TARGET_VAL(ISD::EXTRACT_SUBVECTOR),
// CHECK: OPC_CheckChild1Integer, 0,
-// CHECK: OPC_EmitIntegerI32, sub0_sub1,
+// CHECK: OPC_EmitIntegerI32_2,
def : Pat<(v2i32 (extract_subvector v32i32:$src, (i32 0))),
(EXTRACT_SUBREG GPR_1024:$src, sub0_sub1)>;
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 3d69c96ebd900..46c0430d6b023 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -924,6 +924,28 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
int64_t Val = IM->getValue();
const std::string &Str = IM->getString();
const ValueTypeByHwMode &VTBH = IM->getVT();
+ if (VTBH.isSimple()) {
+ MVT VT = VTBH.getSimple();
+ if ((VT == MVT::i32 && Val >= -1 && Val <= 8) ||
+ (VT == MVT::i64 && Val >= -1 && Val <= 7)) {
+ OS << "OPC_EmitIntegerI" << VT.getSizeInBits();
+ if (Val < 0)
+ OS << "Neg1";
+ else
+ OS << '_' << Val;
+ OS << ',';
+ if (!OmitComments) {
+ OS << " // #" << IM->getResultNo() << " = ";
+ if (!Str.empty())
+ OS << Str;
+ else
+ OS << Val;
+ }
+ OS << '\n';
+ return 1;
+ }
+ }
+
unsigned TypeBytes = 0;
if (VTBH.isSimple()) {
MVT VT = VTBH.getSimple();
>From 7513c92ce403e711c3a85e99ef7652521a70a972 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 21:32:42 -0400
Subject: [PATCH 2/3] [SelectionDAG] Encode frequent ISD opcode checks directly
DAGISelMatcherEmitter emits OPC_CheckOpcode followed by a two-byte
opcode for every CheckOpcodeMatcher. Add one-byte matcher opcodes for
the 32 most frequent built-in ISD opcodes and decode them through one
constexpr table in SelectionDAGISel.
The generated AArch64, AMDGPU, ARM, Mips, WebAssembly, and X86 matcher
tables shrink by 96,012 bytes on top of the compact integer-constant
encoding. In the LLVM 22 Bazel build, stripped llc shrinks from
79,058,960 to 78,926,848 bytes, saving 132,112 bytes (0.167%).
The compact form removes two matcher-table loads and the uint16_t
reconstruction for each encoded check. A 12-pair alternating clang -O2
compile of tramp3d measured 6.392 seconds mean user CPU for the prior
binary and 6.013 seconds for the complete compact-opcode and fused-pair
stack, with no measured runtime regression.
Validate CPtrWildcard.td, DAGDefaultOps.td,
dag-isel-regclass-emit-enum.td, and dag-isel-subregs.td with llvm-tblgen
and FileCheck. Compile DAGISelMatcherEmitter.cpp and
SelectionDAGISel.cpp with Clang.
---
llvm/include/llvm/CodeGen/SelectionDAGISel.h | 33 +++++++
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 88 +++++++++++++++++++
llvm/test/TableGen/CPtrWildcard.td | 22 ++---
.../TableGen/dag-isel-regclass-emit-enum.td | 4 +-
llvm/test/TableGen/dag-isel-subregs.td | 2 +-
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 54 +++++++++++-
6 files changed, 186 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 98f5c9e63e26a..6837b8c2f7385 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -202,6 +202,39 @@ class LLVM_ABI SelectionDAGISel {
OPC_CheckPredicate7,
OPC_CheckPredicateWithOperands,
OPC_CheckOpcode,
+ // Space-optimized forms for the most frequent built-in ISD opcodes.
+ OPC_CheckOpcodeISD_SRL,
+ OPC_CheckOpcodeISD_SIGN_EXTEND_INREG,
+ OPC_CheckOpcodeISD_TargetConstant,
+ OPC_CheckOpcodeISD_LOAD,
+ OPC_CheckOpcodeISD_Constant,
+ OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN,
+ OPC_CheckOpcodeISD_SRA,
+ OPC_CheckOpcodeISD_ADD,
+ OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR,
+ OPC_CheckOpcodeISD_SPLAT_VECTOR,
+ OPC_CheckOpcodeISD_EXTRACT_VECTOR_ELT,
+ OPC_CheckOpcodeISD_XOR,
+ OPC_CheckOpcodeISD_MUL,
+ OPC_CheckOpcodeISD_AND,
+ OPC_CheckOpcodeISD_ZERO_EXTEND,
+ OPC_CheckOpcodeISD_BITCAST,
+ OPC_CheckOpcodeISD_SIGN_EXTEND,
+ OPC_CheckOpcodeISD_UNDEF,
+ OPC_CheckOpcodeISD_CONDCODE,
+ OPC_CheckOpcodeISD_FMINNUM,
+ OPC_CheckOpcodeISD_FMAXNUM,
+ OPC_CheckOpcodeISD_FMINNUM_IEEE,
+ OPC_CheckOpcodeISD_FMAXNUM_IEEE,
+ OPC_CheckOpcodeISD_ConstantFP,
+ OPC_CheckOpcodeISD_VSELECT,
+ OPC_CheckOpcodeISD_TRUNCATE,
+ OPC_CheckOpcodeISD_FNEG,
+ OPC_CheckOpcodeISD_SHL,
+ OPC_CheckOpcodeISD_BasicBlock,
+ OPC_CheckOpcodeISD_SMAX,
+ OPC_CheckOpcodeISD_SMIN,
+ OPC_CheckOpcodeISD_BUILD_VECTOR,
OPC_SwitchOpcode,
OPC_CheckType,
// Space-optimized forms that implicitly encode VT.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 9ee60bbad2106..a80518250e5ab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3016,6 +3016,53 @@ CheckOpcode(const uint8_t *MatcherTable, size_t &MatcherIndex, SDNode *N) {
return N->getOpcode() == Opc;
}
+static constexpr uint16_t CompactISDOpcodes[] = {
+ ISD::SRL,
+ ISD::SIGN_EXTEND_INREG,
+ ISD::TargetConstant,
+ ISD::LOAD,
+ ISD::Constant,
+ ISD::INTRINSIC_WO_CHAIN,
+ ISD::SRA,
+ ISD::ADD,
+ ISD::EXTRACT_SUBVECTOR,
+ ISD::SPLAT_VECTOR,
+ ISD::EXTRACT_VECTOR_ELT,
+ ISD::XOR,
+ ISD::MUL,
+ ISD::AND,
+ ISD::ZERO_EXTEND,
+ ISD::BITCAST,
+ ISD::SIGN_EXTEND,
+ ISD::UNDEF,
+ ISD::CONDCODE,
+ ISD::FMINNUM,
+ ISD::FMAXNUM,
+ ISD::FMINNUM_IEEE,
+ ISD::FMAXNUM_IEEE,
+ ISD::ConstantFP,
+ ISD::VSELECT,
+ ISD::TRUNCATE,
+ ISD::FNEG,
+ ISD::SHL,
+ ISD::BasicBlock,
+ ISD::SMAX,
+ ISD::SMIN,
+ ISD::BUILD_VECTOR,
+};
+
+static_assert(sizeof(CompactISDOpcodes) / sizeof(*CompactISDOpcodes) ==
+ SelectionDAGISel::OPC_CheckOpcodeISD_BUILD_VECTOR -
+ SelectionDAGISel::OPC_CheckOpcodeISD_SRL +
+ 1);
+
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
+CheckCompactISDOpcode(unsigned MatcherOpcode, SDNode *N) {
+ return N->getOpcode() ==
+ CompactISDOpcodes[MatcherOpcode -
+ SelectionDAGISel::OPC_CheckOpcodeISD_SRL];
+}
+
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckType(MVT::SimpleValueType VT,
SDValue N,
const TargetLowering *TLI,
@@ -3113,6 +3160,12 @@ static size_t IsPredicateKnownToFail(
const SelectionDAGISel &SDISel,
SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
unsigned Opcode = Table[Index++];
+ if (Opcode >= SelectionDAGISel::OPC_CheckOpcodeISD_SRL &&
+ Opcode <= SelectionDAGISel::OPC_CheckOpcodeISD_BUILD_VECTOR) {
+ Result = !::CheckCompactISDOpcode(Opcode, N.getNode());
+ return Index;
+ }
+
switch (Opcode) {
default:
Result = false;
@@ -3756,6 +3809,41 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
case OPC_CheckOpcode:
if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
continue;
+ case OPC_CheckOpcodeISD_SRL:
+ case OPC_CheckOpcodeISD_SIGN_EXTEND_INREG:
+ case OPC_CheckOpcodeISD_TargetConstant:
+ case OPC_CheckOpcodeISD_LOAD:
+ case OPC_CheckOpcodeISD_Constant:
+ case OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN:
+ case OPC_CheckOpcodeISD_SRA:
+ case OPC_CheckOpcodeISD_ADD:
+ case OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR:
+ case OPC_CheckOpcodeISD_SPLAT_VECTOR:
+ case OPC_CheckOpcodeISD_EXTRACT_VECTOR_ELT:
+ case OPC_CheckOpcodeISD_XOR:
+ case OPC_CheckOpcodeISD_MUL:
+ case OPC_CheckOpcodeISD_AND:
+ case OPC_CheckOpcodeISD_ZERO_EXTEND:
+ case OPC_CheckOpcodeISD_BITCAST:
+ case OPC_CheckOpcodeISD_SIGN_EXTEND:
+ case OPC_CheckOpcodeISD_UNDEF:
+ case OPC_CheckOpcodeISD_CONDCODE:
+ case OPC_CheckOpcodeISD_FMINNUM:
+ case OPC_CheckOpcodeISD_FMAXNUM:
+ case OPC_CheckOpcodeISD_FMINNUM_IEEE:
+ case OPC_CheckOpcodeISD_FMAXNUM_IEEE:
+ case OPC_CheckOpcodeISD_ConstantFP:
+ case OPC_CheckOpcodeISD_VSELECT:
+ case OPC_CheckOpcodeISD_TRUNCATE:
+ case OPC_CheckOpcodeISD_FNEG:
+ case OPC_CheckOpcodeISD_SHL:
+ case OPC_CheckOpcodeISD_BasicBlock:
+ case OPC_CheckOpcodeISD_SMAX:
+ case OPC_CheckOpcodeISD_SMIN:
+ case OPC_CheckOpcodeISD_BUILD_VECTOR:
+ if (!::CheckCompactISDOpcode(Opcode, N.getNode()))
+ break;
+ continue;
case OPC_CheckType:
case OPC_CheckTypeI32:
diff --git a/llvm/test/TableGen/CPtrWildcard.td b/llvm/test/TableGen/CPtrWildcard.td
index 6b1312b6a1fe2..844e208f4b3d1 100644
--- a/llvm/test/TableGen/CPtrWildcard.td
+++ b/llvm/test/TableGen/CPtrWildcard.td
@@ -4,23 +4,23 @@
// and verify that we can match it correct in SelectionDAG.
// CHECK: static const uint8_t MatcherTable[] = {
-// CHECK-NEXT: /* 0*/ OPC_CheckOpcode, TARGET_VAL(ISD::INTRINSIC_WO_CHAIN),
-// CHECK-NEXT:/* 3*/ OPC_CheckChild0Integer, [[#]],
-// CHECK-NEXT:/* 5*/ OPC_RecordChild1, // #0 = $src
-// CHECK-NEXT:/* 6*/ OPC_Scope /*2 children */, 9, // ->17
-// CHECK-NEXT:/* 8*/ OPC_CheckChild1Type, /*MVT::c64*/5|128,2/*261*/,
-// CHECK-NEXT:/* 11*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C64_TO_I64),
+// CHECK-NEXT: /* 0*/ OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN,
+// CHECK-NEXT:/* 1*/ OPC_CheckChild0Integer, [[#]],
+// CHECK-NEXT:/* 3*/ OPC_RecordChild1, // #0 = $src
+// CHECK-NEXT:/* 4*/ OPC_Scope /*2 children */, 9, // ->15
+// CHECK-NEXT:/* 6*/ OPC_CheckChild1Type, /*MVT::c64*/5|128,2/*261*/,
+// CHECK-NEXT:/* 9*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C64_TO_I64),
// CHECK-NEXT: MVT::i64, 1/*#Ops*/, /*OperandList*/0, // Ops = #0
// CHECK-NEXT: // Src: (intrinsic_wo_chain:{ *:[i64] } [[#]]:{ *:[iPTR] }, c64:{ *:[c64] }:$src) - Complexity = 8
// CHECK-NEXT: // Dst: (C64_TO_I64:{ *:[i64] } ?:{ *:[c64] }:$src)
-// CHECK-NEXT:/* 17*/ /*Scope*/ 9, // ->27
-// CHECK-NEXT:/* 18*/ OPC_CheckChild1Type, /*MVT::c128*/6|128,2/*262*/,
-// CHECK-NEXT:/* 21*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C128_TO_I64),
+// CHECK-NEXT:/* 15*/ /*Scope*/ 9, // ->25
+// CHECK-NEXT:/* 16*/ OPC_CheckChild1Type, /*MVT::c128*/6|128,2/*262*/,
+// CHECK-NEXT:/* 19*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C128_TO_I64),
// CHECK-NEXT: MVT::i64, 1/*#Ops*/, /*OperandList*/0, // Ops = #0
// CHECK-NEXT: // Src: (intrinsic_wo_chain:{ *:[i64] } [[#]]:{ *:[iPTR] }, c128:{ *:[c128] }:$src) - Complexity = 8
// CHECK-NEXT: // Dst: (C128_TO_I64:{ *:[i64] } ?:{ *:[c128] }:$src)
-// CHECK-NEXT:/* 27*/ 0, // End of Scope
-// CHECK-NEXT: }; // Total Array size is 28 bytes
+// CHECK-NEXT:/* 25*/ 0, // End of Scope
+// CHECK-NEXT: }; // Total Array size is 26 bytes
// CHECK: static const uint8_t OperandLists[] = {
// CHECK-NEXT: /* 0 */ 0,
diff --git a/llvm/test/TableGen/dag-isel-regclass-emit-enum.td b/llvm/test/TableGen/dag-isel-regclass-emit-enum.td
index 5a3b4a47953ca..631e6552b0768 100644
--- a/llvm/test/TableGen/dag-isel-regclass-emit-enum.td
+++ b/llvm/test/TableGen/dag-isel-regclass-emit-enum.td
@@ -21,9 +21,9 @@ def GPRAbove127 : RegisterClass<"TestTarget", [i32], 32,
(add R0)>;
} // end Namespace TestNamespace
-// CHECK: OPC_CheckOpcode, TARGET_VAL(ISD::ADD),
+// CHECK: OPC_CheckOpcodeISD_ADD,
// CHECK-NEXT: OPC_RecordChild0, // #0 = $src
-// CHECK-NEXT: OPC_Scope /*2 children */, 11, // ->17
+// CHECK-NEXT: OPC_Scope /*2 children */, 11, // ->15
// CHECK-NEXT: OPC_CheckChild1Integer, 0,
// CHECK-NEXT: OPC_EmitIntegerI32, 0|128,1/*128*/, // #1 = TestNamespace::GPRAbove127RegClassID
// CHECK-NEXT: OPC_MorphNodeTo1None, TARGET_VAL(TargetOpcode::COPY_TO_REGCLASS),
diff --git a/llvm/test/TableGen/dag-isel-subregs.td b/llvm/test/TableGen/dag-isel-subregs.td
index da90ad0ced635..2e86ac4f92caf 100644
--- a/llvm/test/TableGen/dag-isel-subregs.td
+++ b/llvm/test/TableGen/dag-isel-subregs.td
@@ -2,7 +2,7 @@
include "reg-with-subregs-common.td"
-// CHECK-LABEL: OPC_CheckOpcode, TARGET_VAL(ISD::EXTRACT_SUBVECTOR),
+// CHECK-LABEL: OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR,
// CHECK: OPC_CheckChild1Integer, 0,
// CHECK: OPC_EmitIntegerI32_2,
def : Pat<(v2i32 (extract_subvector v32i32:$src, (i32 0))),
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 46c0430d6b023..564fff8e2b97e 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -20,6 +20,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Format.h"
@@ -513,6 +514,47 @@ static unsigned emitMVT(MVT VT, raw_ostream &OS) {
return EmitVBRValue(VT.SimpleTy, OS);
}
+static StringRef getCompactISDOpcode(StringRef Opcode) {
+ return StringSwitch<StringRef>(Opcode)
+ .Case("ISD::SRL", "OPC_CheckOpcodeISD_SRL")
+ .Case("ISD::SIGN_EXTEND_INREG",
+ "OPC_CheckOpcodeISD_SIGN_EXTEND_INREG")
+ .Case("ISD::TargetConstant", "OPC_CheckOpcodeISD_TargetConstant")
+ .Case("ISD::LOAD", "OPC_CheckOpcodeISD_LOAD")
+ .Case("ISD::Constant", "OPC_CheckOpcodeISD_Constant")
+ .Case("ISD::INTRINSIC_WO_CHAIN",
+ "OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN")
+ .Case("ISD::SRA", "OPC_CheckOpcodeISD_SRA")
+ .Case("ISD::ADD", "OPC_CheckOpcodeISD_ADD")
+ .Case("ISD::EXTRACT_SUBVECTOR",
+ "OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR")
+ .Case("ISD::SPLAT_VECTOR", "OPC_CheckOpcodeISD_SPLAT_VECTOR")
+ .Case("ISD::EXTRACT_VECTOR_ELT",
+ "OPC_CheckOpcodeISD_EXTRACT_VECTOR_ELT")
+ .Case("ISD::XOR", "OPC_CheckOpcodeISD_XOR")
+ .Case("ISD::MUL", "OPC_CheckOpcodeISD_MUL")
+ .Case("ISD::AND", "OPC_CheckOpcodeISD_AND")
+ .Case("ISD::ZERO_EXTEND", "OPC_CheckOpcodeISD_ZERO_EXTEND")
+ .Case("ISD::BITCAST", "OPC_CheckOpcodeISD_BITCAST")
+ .Case("ISD::SIGN_EXTEND", "OPC_CheckOpcodeISD_SIGN_EXTEND")
+ .Case("ISD::UNDEF", "OPC_CheckOpcodeISD_UNDEF")
+ .Case("ISD::CONDCODE", "OPC_CheckOpcodeISD_CONDCODE")
+ .Case("ISD::FMINNUM", "OPC_CheckOpcodeISD_FMINNUM")
+ .Case("ISD::FMAXNUM", "OPC_CheckOpcodeISD_FMAXNUM")
+ .Case("ISD::FMINNUM_IEEE", "OPC_CheckOpcodeISD_FMINNUM_IEEE")
+ .Case("ISD::FMAXNUM_IEEE", "OPC_CheckOpcodeISD_FMAXNUM_IEEE")
+ .Case("ISD::ConstantFP", "OPC_CheckOpcodeISD_ConstantFP")
+ .Case("ISD::VSELECT", "OPC_CheckOpcodeISD_VSELECT")
+ .Case("ISD::TRUNCATE", "OPC_CheckOpcodeISD_TRUNCATE")
+ .Case("ISD::FNEG", "OPC_CheckOpcodeISD_FNEG")
+ .Case("ISD::SHL", "OPC_CheckOpcodeISD_SHL")
+ .Case("ISD::BasicBlock", "OPC_CheckOpcodeISD_BasicBlock")
+ .Case("ISD::SMAX", "OPC_CheckOpcodeISD_SMAX")
+ .Case("ISD::SMIN", "OPC_CheckOpcodeISD_SMIN")
+ .Case("ISD::BUILD_VECTOR", "OPC_CheckOpcodeISD_BUILD_VECTOR")
+ .Default("");
+}
+
unsigned
MatcherTableEmitter::emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
unsigned Index, raw_ostream &OS) {
@@ -676,10 +718,16 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
return 2 + OperandBytes;
}
- case Matcher::CheckOpcode:
- OS << "OPC_CheckOpcode, TARGET_VAL("
- << cast<CheckOpcodeMatcher>(N)->getOpcode().getEnumName() << "),\n";
+ case Matcher::CheckOpcode: {
+ StringRef Opcode = cast<CheckOpcodeMatcher>(N)->getOpcode().getEnumName();
+ if (StringRef CompactOpcode = getCompactISDOpcode(Opcode);
+ !CompactOpcode.empty()) {
+ OS << CompactOpcode << ",\n";
+ return 1;
+ }
+ OS << "OPC_CheckOpcode, TARGET_VAL(" << Opcode << "),\n";
return 3;
+ }
case Matcher::SwitchOpcode:
case Matcher::SwitchType: {
>From c44de86fefb043b8308889b04959d5380a9a95d8 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 21:32:42 -0400
Subject: [PATCH 3/3] [SelectionDAG] Fuse frequent adjacent matcher operations
DAGISel matcher tables contain repeated adjacent one-byte operations. Add eight one-byte superinstructions for the most frequent fixed pairs, including consecutive MoveParent operations and common predicate, child, record, type, and opcode checks.
Each fused opcode executes the same checks and state updates while removing one interpreter dispatch and one matcher-table opcode. Keep only the eight pairs whose table savings justify their interpreter handlers.
In a Release Native AArch64 build, the complete three-commit matcher-encoding stack reduces stripped llc from 48,208,016 to 48,174,992 bytes (-33,024), the __TEXT segment from 42,778,624 to 42,745,856 bytes (-32,768), and __TEXT,__const by 35,856 bytes while adding 348 bytes to __TEXT,__text. __DATA_CONST,__const and 62,637 linked fixups are unchanged. SelectionDAGISel.cpp.o grows from 169,616 to 170,456 bytes and its relocations grow from 2,226 to 2,239.
Removing four low-frequency fusions from the previous version reduces SelectionDAGISel.cpp.o by 552 bytes and three relocations. It reduces linked section payload by another 488 bytes without changing the linked file size.
Validate CPtrWildcard.td, dag-isel-regclass-emit-enum.td, and dag-isel-subregs.td with llvm-lit, and build Release Native AArch64 llc.
---
llvm/include/llvm/CodeGen/SelectionDAGISel.h | 9 ++
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 61 ++++++++++++-
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 87 ++++++++++++++++---
3 files changed, 145 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 6837b8c2f7385..1adff743bf91d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -403,6 +403,15 @@ class LLVM_ABI SelectionDAGISel {
OPC_MorphNodeTo1GlueOutput,
OPC_MorphNodeTo2GlueOutput,
OPC_CompleteMatch,
+ // Space-optimized forms for frequent adjacent matcher operations.
+ OPC_MoveParent_CheckPredicate0,
+ OPC_MoveParent2,
+ OPC_MoveChild0_CheckOpcodeISD_SRL,
+ OPC_MoveParent_RecordChild1,
+ OPC_MoveParent_CheckTypeI32,
+ OPC_MoveParent_RecordChild2,
+ OPC_MoveParent_CheckPredicate1,
+ OPC_MoveChild1_CheckOpcodeISD_ADD,
// Contains 32-bit offset in table for pattern being selected
OPC_Coverage
};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index a80518250e5ab..1eabaa26989d9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3053,8 +3053,7 @@ static constexpr uint16_t CompactISDOpcodes[] = {
static_assert(sizeof(CompactISDOpcodes) / sizeof(*CompactISDOpcodes) ==
SelectionDAGISel::OPC_CheckOpcodeISD_BUILD_VECTOR -
- SelectionDAGISel::OPC_CheckOpcodeISD_SRL +
- 1);
+ SelectionDAGISel::OPC_CheckOpcodeISD_SRL + 1);
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckCompactISDOpcode(unsigned MatcherOpcode, SDNode *N) {
@@ -3730,6 +3729,64 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
N = NodeStack.back();
continue;
+ case OPC_MoveParent_CheckPredicate0:
+ case OPC_MoveParent_CheckPredicate1: {
+ NodeStack.pop_back();
+ assert(!NodeStack.empty() && "Node stack imbalance!");
+ N = NodeStack.back();
+ unsigned PredicateOpcode = Opcode == OPC_MoveParent_CheckPredicate0
+ ? OPC_CheckPredicate0
+ : OPC_CheckPredicate1;
+ if (!::CheckNodePredicate(PredicateOpcode, MatcherTable, MatcherIndex,
+ *this, N))
+ break;
+ continue;
+ }
+
+ case OPC_MoveParent2:
+ NodeStack.pop_back();
+ assert(!NodeStack.empty() && "Node stack imbalance!");
+ N = NodeStack.back();
+ NodeStack.pop_back();
+ assert(!NodeStack.empty() && "Node stack imbalance!");
+ N = NodeStack.back();
+ continue;
+
+ case OPC_MoveChild0_CheckOpcodeISD_SRL:
+ case OPC_MoveChild1_CheckOpcodeISD_ADD: {
+ unsigned ChildNo = Opcode == OPC_MoveChild0_CheckOpcodeISD_SRL ? 0 : 1;
+ if (ChildNo >= N.getNumOperands())
+ break;
+ N = N.getOperand(ChildNo);
+ NodeStack.push_back(N);
+ unsigned CheckOpcode = Opcode == OPC_MoveChild0_CheckOpcodeISD_SRL
+ ? OPC_CheckOpcodeISD_SRL
+ : OPC_CheckOpcodeISD_ADD;
+ if (!::CheckCompactISDOpcode(CheckOpcode, N.getNode()))
+ break;
+ continue;
+ }
+
+ case OPC_MoveParent_RecordChild1:
+ case OPC_MoveParent_RecordChild2: {
+ NodeStack.pop_back();
+ assert(!NodeStack.empty() && "Node stack imbalance!");
+ N = NodeStack.back();
+ unsigned ChildNo = Opcode == OPC_MoveParent_RecordChild1 ? 1 : 2;
+ if (ChildNo >= N.getNumOperands())
+ break;
+ RecordedNodes.emplace_back(N->getOperand(ChildNo), N.getNode());
+ continue;
+ }
+
+ case OPC_MoveParent_CheckTypeI32:
+ NodeStack.pop_back();
+ assert(!NodeStack.empty() && "Node stack imbalance!");
+ N = NodeStack.back();
+ if (!::CheckType(MVT::i32, N, TLI, CurDAG->getDataLayout()))
+ break;
+ continue;
+
case OPC_CheckSame:
if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
continue;
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 564fff8e2b97e..cf89d105e6e34 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -252,6 +252,51 @@ class MatcherTableEmitter {
unsigned EmitMatcher(const Matcher *N, const unsigned Indent,
unsigned CurrentIdx, raw_ostream &OS);
+ StringRef getFusedOpcode(const Matcher *First, const Matcher *Second) {
+ if (isa<MoveParentMatcher>(First)) {
+ if (isa<MoveParentMatcher>(Second))
+ return "OPC_MoveParent2";
+
+ if (const auto *PM = dyn_cast<CheckPredicateMatcher>(Second)) {
+ if (!PM->getPredicate().usesOperands()) {
+ switch (getNodePredicate(PM->getPredicate())) {
+ case 0:
+ return "OPC_MoveParent_CheckPredicate0";
+ case 1:
+ return "OPC_MoveParent_CheckPredicate1";
+ default:
+ break;
+ }
+ }
+ }
+
+ if (const auto *RCM = dyn_cast<RecordChildMatcher>(Second)) {
+ if (RCM->getChildNo() == 1)
+ return "OPC_MoveParent_RecordChild1";
+ if (RCM->getChildNo() == 2)
+ return "OPC_MoveParent_RecordChild2";
+ }
+
+ if (const auto *CTM = dyn_cast<CheckTypeMatcher>(Second)) {
+ if (CTM->getResNo() == 0 && CTM->getType().isSimple() &&
+ CTM->getType().getSimple() == MVT::i32)
+ return "OPC_MoveParent_CheckTypeI32";
+ }
+ }
+
+ if (const auto *MCM = dyn_cast<MoveChildMatcher>(First)) {
+ if (const auto *COM = dyn_cast<CheckOpcodeMatcher>(Second)) {
+ StringRef Opcode = COM->getOpcode().getEnumName();
+ if (MCM->getChildNo() == 0 && Opcode == "ISD::SRL")
+ return "OPC_MoveChild0_CheckOpcodeISD_SRL";
+ if (MCM->getChildNo() == 1 && Opcode == "ISD::ADD")
+ return "OPC_MoveChild1_CheckOpcodeISD_ADD";
+ }
+ }
+
+ return {};
+ }
+
unsigned getNodePredicate(TreePredicateFn Pred) {
// We use the first predicate.
TreePattern *PredPat =
@@ -378,8 +423,20 @@ static std::string getIncludePath(const Record *R) {
unsigned MatcherTableEmitter::SizeMatcherList(MatcherList &ML,
raw_ostream &OS) {
unsigned Size = 0;
- for (Matcher *N : ML)
+ for (auto I = ML.begin(), E = ML.end(); I != E;) {
+ Matcher *N = *I;
+ ++I;
+ if (I != E && !getFusedOpcode(N, *I).empty()) {
+ unsigned PairSize = SizeMatcher(N, OS) + SizeMatcher(*I, OS);
+ ++I;
+ assert(PairSize == 2 &&
+ "fused matchers must both have one-byte encodings");
+ (void)PairSize;
+ ++Size;
+ continue;
+ }
Size += SizeMatcher(N, OS);
+ }
return Size;
}
@@ -517,20 +574,16 @@ static unsigned emitMVT(MVT VT, raw_ostream &OS) {
static StringRef getCompactISDOpcode(StringRef Opcode) {
return StringSwitch<StringRef>(Opcode)
.Case("ISD::SRL", "OPC_CheckOpcodeISD_SRL")
- .Case("ISD::SIGN_EXTEND_INREG",
- "OPC_CheckOpcodeISD_SIGN_EXTEND_INREG")
+ .Case("ISD::SIGN_EXTEND_INREG", "OPC_CheckOpcodeISD_SIGN_EXTEND_INREG")
.Case("ISD::TargetConstant", "OPC_CheckOpcodeISD_TargetConstant")
.Case("ISD::LOAD", "OPC_CheckOpcodeISD_LOAD")
.Case("ISD::Constant", "OPC_CheckOpcodeISD_Constant")
- .Case("ISD::INTRINSIC_WO_CHAIN",
- "OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN")
+ .Case("ISD::INTRINSIC_WO_CHAIN", "OPC_CheckOpcodeISD_INTRINSIC_WO_CHAIN")
.Case("ISD::SRA", "OPC_CheckOpcodeISD_SRA")
.Case("ISD::ADD", "OPC_CheckOpcodeISD_ADD")
- .Case("ISD::EXTRACT_SUBVECTOR",
- "OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR")
+ .Case("ISD::EXTRACT_SUBVECTOR", "OPC_CheckOpcodeISD_EXTRACT_SUBVECTOR")
.Case("ISD::SPLAT_VECTOR", "OPC_CheckOpcodeISD_SPLAT_VECTOR")
- .Case("ISD::EXTRACT_VECTOR_ELT",
- "OPC_CheckOpcodeISD_EXTRACT_VECTOR_ELT")
+ .Case("ISD::EXTRACT_VECTOR_ELT", "OPC_CheckOpcodeISD_EXTRACT_VECTOR_ELT")
.Case("ISD::XOR", "OPC_CheckOpcodeISD_XOR")
.Case("ISD::MUL", "OPC_CheckOpcodeISD_MUL")
.Case("ISD::AND", "OPC_CheckOpcodeISD_AND")
@@ -1371,9 +1424,23 @@ unsigned MatcherTableEmitter::EmitMatcherList(const MatcherList &ML,
unsigned CurrentIdx,
raw_ostream &OS) {
unsigned Size = 0;
- for (const Matcher *N : ML) {
+ for (auto I = ML.begin(), E = ML.end(); I != E;) {
+ const Matcher *N = *I;
+ ++I;
if (!OmitComments)
OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/";
+
+ if (I != E) {
+ StringRef FusedOpcode = getFusedOpcode(N, *I);
+ if (!FusedOpcode.empty()) {
+ OS.indent(Indent) << FusedOpcode << ",\n";
+ ++I;
+ ++Size;
+ ++CurrentIdx;
+ continue;
+ }
+ }
+
unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS);
Size += MatcherSize;
CurrentIdx += MatcherSize;
More information about the llvm-commits
mailing list