[llvm] [SelectionDAG] Extend the direct MVT encoding range (PR #202636)
David Zbarsky via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 07:00:33 PDT 2026
https://github.com/dzbarsky created https://github.com/llvm/llvm-project/pull/202636
DAGISel matcher tables use the high bit of the first byte to mark VBR-encoded MVT values, so values 128 through 254 previously required two bytes. Reserve byte 255 as an escape and encode every value below it directly. Values at or above 255 use the escape followed by a VBR-encoded offset.
Use the same encoding-size calculation when sizing SwitchType case offsets. Retaining GetVBRSize there overestimated cases for MVT values 128 through 254 and made the interpreter jump into unrelated matcher bytes. Add an AArch64 code-generation regression test that exercises the affected switch.
The generated AArch64, AMDGPU, ARM, Mips, WebAssembly, and X86 matcher tables shrink by 40,171 bytes. In the LLVM 22 Bazel build, stripped llc shrinks from 78,860,808 to 78,629,632 bytes, saving 231,176 bytes (0.293%) across all targets.
The common direct decode still performs one byte load and one branch. A 12-pair alternating llc -O0 compile of tramp3d measured 1.0892 seconds mean user CPU before the change and 1.0825 seconds after it.
Validate boundary values 127, 128, 254, 255, and 262 with dag-isel-mvt-encoding.td, CPtrWildcard.td, and the dag-isel TableGen tests. Compile DAGISelMatcherEmitter.cpp and SelectionDAGISel.cpp. In the LLVM 22 all-target remote build, verify the AArch64 regression, a dot-product case, and byte-identical output against the pre-encoding binary.
Work towards #202616
>From ce483cb85eafcdbe2cd9de95f79ae70b838bb2e9 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Mon, 8 Jun 2026 21:43:19 -0400
Subject: [PATCH] [SelectionDAG] Extend the direct MVT encoding range
DAGISel matcher tables use the high bit of the first byte to mark VBR-encoded MVT values, so values 128 through 254 previously required two bytes. Reserve byte 255 as an escape and encode every value below it directly. Values at or above 255 use the escape followed by a VBR-encoded offset.
Use the same encoding-size calculation when sizing SwitchType case offsets. Retaining GetVBRSize there overestimated cases for MVT values 128 through 254 and made the interpreter jump into unrelated matcher bytes. Add an AArch64 code-generation regression test that exercises the affected switch.
The generated AArch64, AMDGPU, ARM, Mips, WebAssembly, and X86 matcher tables shrink by 40,171 bytes. In the LLVM 22 Bazel build, stripped llc shrinks from 78,860,808 to 78,629,632 bytes, saving 231,176 bytes (0.293%) across all targets.
The common direct decode still performs one byte load and one branch. A 12-pair alternating llc -O0 compile of tramp3d measured 1.0892 seconds mean user CPU before the change and 1.0825 seconds after it.
Validate boundary values 127, 128, 254, 255, and 262 with dag-isel-mvt-encoding.td, CPtrWildcard.td, and the dag-isel TableGen tests. Compile DAGISelMatcherEmitter.cpp and SelectionDAGISel.cpp. In the LLVM 22 all-target remote build, verify the AArch64 regression, a dot-product case, and byte-identical output against the pre-encoding binary.
---
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 22 ++++++--
.../AArch64/dag-isel-mvt-switch-offset.ll | 11 ++++
llvm/test/TableGen/CPtrWildcard.td | 4 +-
llvm/test/TableGen/dag-isel-mvt-encoding.td | 56 +++++++++++++++++++
llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 36 ++++++++----
5 files changed, 112 insertions(+), 17 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/dag-isel-mvt-switch-offset.ll
create mode 100644 llvm/test/TableGen/dag-isel-mvt-encoding.td
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 5ae52cae771fb..387cee9094b6a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -109,6 +109,7 @@
#include <memory>
#include <optional>
#include <string>
+#include <type_traits>
#include <utility>
#include <vector>
@@ -2749,13 +2750,24 @@ GetSignedVBR(const unsigned char *MatcherTable, size_t &Idx) {
return Val;
}
-/// getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value,
-/// use GetVBR to decode it.
+/// Decode an MVT from MatcherTable. Values below the escape byte are encoded
+/// directly; larger values use the escape followed by a VBR-encoded offset.
LLVM_ATTRIBUTE_ALWAYS_INLINE static MVT::SimpleValueType
getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex) {
- unsigned SimpleVT = MatcherTable[MatcherIndex++];
- if (SimpleVT & 128)
- SimpleVT = GetVBR(SimpleVT, MatcherTable, MatcherIndex);
+ constexpr uint64_t Escape = std::numeric_limits<uint8_t>::max();
+ using SimpleVTStorage = std::underlying_type_t<MVT::SimpleValueType>;
+ static_assert(std::is_unsigned_v<SimpleVTStorage>);
+ static_assert(sizeof(SimpleVTStorage) <= sizeof(uint64_t));
+
+ uint64_t SimpleVT = MatcherTable[MatcherIndex++];
+ if (SimpleVT == Escape) {
+ uint64_t Offset = MatcherTable[MatcherIndex++];
+ if (Offset & 128)
+ Offset = GetVBR(Offset, MatcherTable, MatcherIndex);
+ assert(Offset <= std::numeric_limits<SimpleVTStorage>::max() - Escape &&
+ "invalid MVT encoding");
+ SimpleVT += Offset;
+ }
return static_cast<MVT::SimpleValueType>(SimpleVT);
}
diff --git a/llvm/test/CodeGen/AArch64/dag-isel-mvt-switch-offset.ll b/llvm/test/CodeGen/AArch64/dag-isel-mvt-switch-offset.ll
new file mode 100644
index 0000000000000..8f0a3bd427412
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/dag-isel-mvt-switch-offset.ll
@@ -0,0 +1,11 @@
+; RUN: llc -mtriple=aarch64 -o - %s | FileCheck %s
+
+define i32 @add(i32 %lhs, i32 %rhs) {
+entry:
+ %sum = add i32 %lhs, %rhs
+ ret i32 %sum
+}
+
+; CHECK-LABEL: add:
+; CHECK: add w0, w0, w1
+; CHECK: ret
diff --git a/llvm/test/TableGen/CPtrWildcard.td b/llvm/test/TableGen/CPtrWildcard.td
index 6b1312b6a1fe2..e452d1f80ced7 100644
--- a/llvm/test/TableGen/CPtrWildcard.td
+++ b/llvm/test/TableGen/CPtrWildcard.td
@@ -8,13 +8,13 @@
// CHECK-NEXT:/* 3*/ OPC_CheckChild0Integer, [[#]],
// CHECK-NEXT:/* 5*/ OPC_RecordChild1, // #0 = $src
// CHECK-NEXT:/* 6*/ OPC_Scope /*2 children */, 9, // ->17
-// CHECK-NEXT:/* 8*/ OPC_CheckChild1Type, /*MVT::c64*/5|128,2/*261*/,
+// CHECK-NEXT:/* 8*/ OPC_CheckChild1Type, 255,/*MVT::c64*/6,
// CHECK-NEXT:/* 11*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C64_TO_I64),
// CHECK-NEXT: MVT::i64, 1/*#Ops*/, /*OperandList*/0, // Ops = #0
// CHECK-NEXT: // Src: (intrinsic_wo_chain:{ *:[i64] } [[#]]:{ *:[iPTR] }, c64:{ *:[c64] }:$src) - Complexity = 8
// CHECK-NEXT: // Dst: (C64_TO_I64:{ *:[i64] } ?:{ *:[c64] }:$src)
// CHECK-NEXT:/* 17*/ /*Scope*/ 9, // ->27
-// CHECK-NEXT:/* 18*/ OPC_CheckChild1Type, /*MVT::c128*/6|128,2/*262*/,
+// CHECK-NEXT:/* 18*/ OPC_CheckChild1Type, 255,/*MVT::c128*/7,
// CHECK-NEXT:/* 21*/ OPC_MorphNodeTo1None, TARGET_VAL(MyTarget::C128_TO_I64),
// CHECK-NEXT: MVT::i64, 1/*#Ops*/, /*OperandList*/0, // Ops = #0
// CHECK-NEXT: // Src: (intrinsic_wo_chain:{ *:[i64] } [[#]]:{ *:[iPTR] }, c128:{ *:[c128] }:$src) - Complexity = 8
diff --git a/llvm/test/TableGen/dag-isel-mvt-encoding.td b/llvm/test/TableGen/dag-isel-mvt-encoding.td
new file mode 100644
index 0000000000000..685aaa451607d
--- /dev/null
+++ b/llvm/test/TableGen/dag-isel-mvt-encoding.td
@@ -0,0 +1,56 @@
+// RUN: llvm-tblgen -gen-dag-isel -I %p/../../include %s -o - | FileCheck %s
+
+// Verify the direct/escaped MVT boundary and representative values on both
+// sides of the old VBR boundary at value 127.
+
+include "llvm/Target/Target.td"
+
+def TestInstrInfo : InstrInfo;
+def TestTarget : Target {
+ let InstructionSet = TestInstrInfo;
+}
+
+def R0 : Register<"r0">;
+def GPR : RegisterClass<"TestTarget", [i32], 32, (add R0)>;
+
+class TestRegisterClass<ValueType VT>
+ : RegisterClass<"TestTarget", [VT], VT.Size, (add R0)>;
+
+def RC127 : TestRegisterClass<v64bf16>;
+def RC128 : TestRegisterClass<v128bf16>;
+def RC254 : TestRegisterClass<x86amx>;
+def RC255 : TestRegisterClass<i64x8>;
+def RC262 : TestRegisterClass<c128>;
+
+class TestNode<ValueType VT, string Opcode>
+ : SDNode<Opcode, SDTypeProfile<1, 0, [SDTCisVT<0, VT>]>>;
+
+def Node127 : TestNode<v64bf16, "TestISD::NODE127">;
+def Node128 : TestNode<v128bf16, "TestISD::NODE128">;
+def Node254 : TestNode<x86amx, "TestISD::NODE254">;
+def Node255 : TestNode<i64x8, "TestISD::NODE255">;
+def Node262 : TestNode<c128, "TestISD::NODE262">;
+
+class TestInstruction<RegisterClass RC> : Instruction {
+ let OutOperandList = (outs RC:$dst);
+ let InOperandList = (ins);
+}
+
+def Inst127 : TestInstruction<RC127>;
+def Inst128 : TestInstruction<RC128>;
+def Inst254 : TestInstruction<RC254>;
+def Inst255 : TestInstruction<RC255>;
+def Inst262 : TestInstruction<RC262>;
+
+def : Pat<(Node127), (Inst127)>;
+def : Pat<(Node128), (Inst128)>;
+def : Pat<(Node254), (Inst254)>;
+def : Pat<(Node255), (Inst255)>;
+def : Pat<(Node262), (Inst262)>;
+
+// CHECK-DAG: MVT::v64bf16, 0/*#Ops*/
+// CHECK-DAG: MVT::v128bf16, 0/*#Ops*/
+// CHECK-DAG: MVT::x86amx, 0/*#Ops*/
+// CHECK-DAG: 255,/*MVT::i64x8*/0, 0/*#Ops*/
+// CHECK-DAG: 255,/*MVT::c128*/7, 0/*#Ops*/
+// CHECK: Total Array size is 44 bytes
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 3d69c96ebd900..da9dc2db8e7c3 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -27,6 +27,9 @@
#include "llvm/Support/SourceMgr.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"
+#include <cstdint>
+#include <limits>
+#include <type_traits>
using namespace llvm;
@@ -313,6 +316,16 @@ static unsigned GetVBRSize(unsigned Val) {
return NumBytes + 1;
}
+static unsigned getMVTEncodingSize(MVT VT) {
+ constexpr uint64_t Escape = std::numeric_limits<uint8_t>::max();
+ using SimpleVTStorage = std::underlying_type_t<MVT::SimpleValueType>;
+ static_assert(std::is_unsigned_v<SimpleVTStorage>);
+ static_assert(sizeof(SimpleVTStorage) <= sizeof(uint64_t));
+
+ uint64_t SimpleVT = static_cast<SimpleVTStorage>(VT.SimpleTy);
+ return SimpleVT < Escape ? 1 : 1 + GetVBRSize(SimpleVT - Escape);
+}
+
/// EmitVBRValue - Emit the specified value as a VBR, returning the number of
/// bytes emitted.
static unsigned EmitVBRValue(uint64_t Val, raw_ostream &OS) {
@@ -422,9 +435,8 @@ unsigned MatcherTableEmitter::SizeMatcher(Matcher *N, raw_ostream &OS) {
Size += 2; // Count the child's opcode.
} else {
Child = &cast<SwitchTypeMatcher>(N)->getCaseMatcher(i);
- Size += GetVBRSize(cast<SwitchTypeMatcher>(N)
- ->getCaseType(i)
- .SimpleTy); // Count the child's type.
+ Size += getMVTEncodingSize(
+ cast<SwitchTypeMatcher>(N)->getCaseType(i));
}
const unsigned ChildSize = SizeMatcherList(*Child, OS);
assert(ChildSize != 0 && "Matcher cannot have child of size 0");
@@ -502,15 +514,21 @@ void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) {
}
static unsigned emitMVT(MVT VT, raw_ostream &OS) {
- // Print the MVT directly if it doesn't require a VBR.
- if (VT.SimpleTy <= 127) {
+ constexpr uint64_t Escape = std::numeric_limits<uint8_t>::max();
+ using SimpleVTStorage = std::underlying_type_t<MVT::SimpleValueType>;
+ static_assert(std::is_unsigned_v<SimpleVTStorage>);
+ static_assert(sizeof(SimpleVTStorage) <= sizeof(uint64_t));
+
+ uint64_t SimpleVT = static_cast<SimpleVTStorage>(VT.SimpleTy);
+ if (SimpleVT < Escape) {
OS << getEnumName(VT) << ',';
return 1;
}
+ OS << Escape << ',';
if (!OmitComments)
OS << "/*" << getEnumName(VT) << "*/";
- return EmitVBRValue(VT.SimpleTy, OS);
+ return 1 + EmitVBRValue(SimpleVT - Escape, OS);
}
unsigned
@@ -708,10 +726,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
IdxSize = 2; // size of opcode in table is 2 bytes.
} else {
Child = &cast<SwitchTypeMatcher>(N)->getCaseMatcher(i);
- IdxSize = GetVBRSize(
- cast<SwitchTypeMatcher>(N)
- ->getCaseType(i)
- .SimpleTy); // size of type in table is sizeof(VBR(MVT)) byte.
+ IdxSize = getMVTEncodingSize(
+ cast<SwitchTypeMatcher>(N)->getCaseType(i));
}
if (i != 0) {
More information about the llvm-commits
mailing list