[llvm] [GlobalISel][TableGen] Support Intrinsics in MIR Patterns (PR #79278)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 24 04:40:07 PST 2024
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/79278
>From 60265e8bcb0331f83da1cbcc961213f98a5b2443 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 24 Jan 2024 12:41:52 +0100
Subject: [PATCH 1/2] [GlobalISel][TableGen] Support Intrinsics Matching &
Rewriting in MIR Patterns
---
llvm/docs/GlobalISel/MIRPatterns.rst | 5 +-
.../CodeGen/GlobalISel/GIMatchTableExecutor.h | 5 +
.../GlobalISel/GIMatchTableExecutorImpl.h | 10 ++
.../Inputs/test-intrinsics.td | 10 ++
.../builtins/builtin-pattern-errors.td | 2 +-
.../match-table-intrinsics.td | 86 ++++++++++++++++++
.../pattern-errors.td | 12 ++-
.../pattern-parsing.td | 50 +++++++++-
llvm/test/TableGen/lit.local.cfg | 2 +-
llvm/utils/TableGen/GlobalISel/Patterns.cpp | 12 ++-
llvm/utils/TableGen/GlobalISel/Patterns.h | 20 +++-
.../TableGen/GlobalISelCombinerEmitter.cpp | 91 ++++++++++++++++---
llvm/utils/TableGen/GlobalISelMatchTable.cpp | 10 ++
llvm/utils/TableGen/GlobalISelMatchTable.h | 18 ++++
14 files changed, 311 insertions(+), 22 deletions(-)
create mode 100644 llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td
create mode 100644 llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst
index cbbe962dcb8180e..728e32470144523 100644
--- a/llvm/docs/GlobalISel/MIRPatterns.rst
+++ b/llvm/docs/GlobalISel/MIRPatterns.rst
@@ -36,8 +36,8 @@ MIR patterns use the DAG datatype in TableGen.
(inst operand0, operand1, ...)
-``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``)
-or ``GICombinePatFrag``.
+``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``),
+``Intrinsic`` or ``GICombinePatFrag``.
Operands essentially fall into one of two categories:
@@ -227,7 +227,6 @@ Limitations
This a non-exhaustive list of known issues with MIR patterns at this time.
-* Matching intrinsics is not yet possible.
* Using ``GICombinePatFrag`` within another ``GICombinePatFrag`` is not
supported.
* ``GICombinePatFrag`` can only have a single root.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 694d3d8004afff1..bec111420068c5f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -379,6 +379,11 @@ enum {
/// - Flags(2) - Register Flags
GIR_AddRegister,
+ /// Adds an intrinsic ID to the specified instruction.
+ /// - InsnID(ULEB128) - Instruction ID to modify
+ /// - IID(2) - Intrinsic ID
+ GIR_AddIntrinsicID,
+
/// Marks the implicit def of a register as dead.
/// - InsnID(ULEB128) - Instruction ID to modify
/// - OpIdx(ULEB128) - The implicit def operand index
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 0a2709ef216be44..3753ea0ee52ab29 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -1116,6 +1116,16 @@ bool GIMatchTableExecutor::executeMatchTable(
<< "], " << RegNum << ", " << RegFlags << ")\n");
break;
}
+ case GIR_AddIntrinsicID: {
+ uint64_t InsnID = readULEB();
+ uint16_t Value = readU16();
+ assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+ OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
+ DEBUG_WITH_TYPE(TgtExecutor::getName(),
+ dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
+ << InsnID << "], " << Value << ")\n");
+ break;
+ }
case GIR_SetImplicitDefDead: {
uint64_t InsnID = readULEB();
uint64_t OpIdx = readULEB();
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td
new file mode 100644
index 000000000000000..90d04f752384476
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td
@@ -0,0 +1,10 @@
+// Dummy intrinsic definitions for TableGen.
+
+
+def int_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], []>;
+def int_0in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [], []>;
+
+def int_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrHasSideEffects]>;
+
+def int_convergent_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
+def int_convergent_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrHasSideEffects]>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
index f9aa926591e4998..8e3e27daaa5bfaf 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td
@@ -53,7 +53,7 @@ def TestPF: GICombinePatFrag<
(outs root:$def),
(ins),
[(pattern (COPY $def, $src))]>;
-// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction or Intrinsic
def eraseroot_notinstmatch: GICombineRule<
(defs root:$mi),
(match (TestPF $dst):$mi),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
new file mode 100644
index 000000000000000..94cc3e58dfc9a15
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
@@ -0,0 +1,86 @@
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
+// RUN: -combiners=MyCombiner %s | \
+// RUN: FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+include "test-intrinsics.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def IntrinTest0 : GICombineRule<
+ (defs root:$a),
+ (match (int_1in_1out $a, 0)),
+ (apply (int_1in_1out $a, $x),
+ (int_0in_1out i32:$x))>;
+
+def SpecialIntrins : GICombineRule<
+ (defs root:$a),
+ (match (int_sideeffects_1in_1out $a, $b)),
+ (apply (int_convergent_1in_1out i32:$x, $b),
+ (int_convergent_sideeffects_1in_1out $a, $x))>;
+
+def MyCombiner: GICombiner<"GenMyCombiner", [
+ IntrinTest0,
+ SpecialIntrins
+]>;
+
+
+// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(114), GIMT_Encode2(116), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132),
+// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18),
+// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73),
+// CHECK-NEXT: // Label 0: @18
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 //
+// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT: // MIs[0] a
+// CHECK-NEXT: // No operand predicates
+// CHECK-NEXT: // MIs[0] Operand 2
+// CHECK-NEXT: GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 0,
+// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT: // Combiner Rule #0: IntrinTest0
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT: GIR_Done,
+// CHECK-NEXT: // Label 3: @72
+// CHECK-NEXT: GIM_Reject,
+// CHECK-NEXT: // Label 1: @73
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 //
+// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
+// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out),
+// CHECK-NEXT: // MIs[0] a
+// CHECK-NEXT: // No operand predicates
+// CHECK-NEXT: // MIs[0] b
+// CHECK-NEXT: // No operand predicates
+// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT: // Combiner Rule #1: SpecialIntrins
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out),
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0,
+// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT: GIR_Done,
+// CHECK-NEXT: // Label 4: @131
+// CHECK-NEXT: GIM_Reject,
+// CHECK-NEXT: // Label 2: @132
+// CHECK-NEXT: GIM_Reject,
+// CHECK-NEXT: }; // Size: 133 bytes
+// CHECK-NEXT: return MatchTable0;
+// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
index 318438b977dc953..4b214d2ca89edfd 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
@@ -1,10 +1,12 @@
-// RUN: not llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: not llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
// RUN: -combiners=MyCombiner %s 2>&1| \
// RUN: FileCheck %s -implicit-check-not=error:
include "llvm/Target/Target.td"
include "llvm/Target/GlobalISel/Combine.td"
+include "test-intrinsics.td"
+
def MyTargetISA : InstrInfo;
def MyTarget : Target { let InstructionSet = MyTargetISA; }
@@ -248,6 +250,13 @@ def miflags_in_builtin : GICombineRule<
(match (COPY $x, $y)),
(apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))'
+def miflags_in_intrin : GICombineRule<
+ (defs root:$x),
+ (match (int_1in_1out $x, $y)),
+ (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
+
// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions
// CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor
def using_flagref_in_match : GICombineRule<
@@ -300,6 +309,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
not_miflagenum_1,
not_miflagenum_2,
miflags_in_builtin,
+ miflags_in_intrin,
using_flagref_in_match,
badflagref_in_apply
]>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
index 26f3bb88da951c4..c261ec457545342 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
@@ -1,10 +1,12 @@
-// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
// RUN: -gicombiner-stop-after-parse -combiners=MyCombiner %s | \
// RUN: FileCheck %s
include "llvm/Target/Target.td"
include "llvm/Target/GlobalISel/Combine.td"
+include "test-intrinsics.td"
+
def MyTargetISA : InstrInfo;
def MyTarget : Target { let InstructionSet = MyTargetISA; }
@@ -342,6 +344,48 @@ def MIFlagsTest : GICombineRule<
(match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi),
(apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>;
+// CHECK-NEXT: (CombineRule name:IntrinTest0 id:12 root:a
+// CHECK-NEXT: (MatchPats
+// CHECK-NEXT: <match_root>__IntrinTest0_match_0:(CodeGenInstructionPattern G_INTRINSIC operands:[<def>$a, $b] intrinsic(@llvm.1in.1out))
+// CHECK-NEXT: )
+// CHECK-NEXT: (ApplyPats
+// CHECK-NEXT: <apply_root>__IntrinTest0_apply_0:(CodeGenInstructionPattern G_INTRINSIC_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.sideeffects.1in.1out))
+// CHECK-NEXT: )
+// CHECK-NEXT: (OperandTable MatchPats
+// CHECK-NEXT: a -> __IntrinTest0_match_0
+// CHECK-NEXT: b -> <live-in>
+// CHECK-NEXT: )
+// CHECK-NEXT: (OperandTable ApplyPats
+// CHECK-NEXT: a -> __IntrinTest0_apply_0
+// CHECK-NEXT: b -> <live-in>
+// CHECK-NEXT: )
+// CHECK-NEXT: )
+def IntrinTest0 : GICombineRule<
+ (defs root:$a),
+ (match (int_1in_1out $a, $b)),
+ (apply (int_sideeffects_1in_1out $a, $b))>;
+
+// CHECK: (CombineRule name:IntrinTest1 id:13 root:a
+// CHECK-NEXT: (MatchPats
+// CHECK-NEXT: <match_root>__IntrinTest1_match_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT operands:[<def>$a, $b] intrinsic(@llvm.convergent.1in.1out))
+// CHECK-NEXT: )
+// CHECK-NEXT: (ApplyPats
+// CHECK-NEXT: <apply_root>__IntrinTest1_apply_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.convergent.sideeffects.1in.1out))
+// CHECK-NEXT: )
+// CHECK-NEXT: (OperandTable MatchPats
+// CHECK-NEXT: a -> __IntrinTest1_match_0
+// CHECK-NEXT: b -> <live-in>
+// CHECK-NEXT: )
+// CHECK-NEXT: (OperandTable ApplyPats
+// CHECK-NEXT: a -> __IntrinTest1_apply_0
+// CHECK-NEXT: b -> <live-in>
+// CHECK-NEXT: )
+// CHECK-NEXT: )
+def IntrinTest1 : GICombineRule<
+ (defs root:$a),
+ (match (int_convergent_1in_1out $a, $b)),
+ (apply (int_convergent_sideeffects_1in_1out $a, $b))>;
+
def MyCombiner: GICombiner<"GenMyCombiner", [
WipOpcodeTest0,
WipOpcodeTest1,
@@ -354,5 +398,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
VariadicsInTest,
VariadicsOutTest,
TypeOfTest,
- MIFlagsTest
+ MIFlagsTest,
+ IntrinTest0,
+ IntrinTest1
]>;
diff --git a/llvm/test/TableGen/lit.local.cfg b/llvm/test/TableGen/lit.local.cfg
index 6d3ea55784535aa..0e827479cd41235 100644
--- a/llvm/test/TableGen/lit.local.cfg
+++ b/llvm/test/TableGen/lit.local.cfg
@@ -1,2 +1,2 @@
config.suffixes = [".td"]
-config.excludes = ["Common"]
+config.excludes = ["Common", "Inputs"]
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/GlobalISel/Patterns.cpp
index 0a6d05e06dca128..758eac2dfebd302 100644
--- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp
+++ b/llvm/utils/TableGen/GlobalISel/Patterns.cpp
@@ -8,6 +8,7 @@
#include "Patterns.h"
#include "../CodeGenInstruction.h"
+#include "../CodeGenIntrinsics.h"
#include "CXXPredicates.h"
#include "CodeExpander.h"
#include "CodeExpansions.h"
@@ -331,7 +332,7 @@ bool CodeGenInstructionPattern::is(StringRef OpcodeName) const {
}
bool CodeGenInstructionPattern::isVariadic() const {
- return I.Operands.isVariadic;
+ return !isIntrinsic() && I.Operands.isVariadic;
}
bool CodeGenInstructionPattern::hasVariadicDefs() const {
@@ -352,6 +353,9 @@ bool CodeGenInstructionPattern::hasVariadicDefs() const {
}
unsigned CodeGenInstructionPattern::getNumInstDefs() const {
+ if (isIntrinsic())
+ return IntrinInfo->IS.RetTys.size();
+
if (!isVariadic() || !hasVariadicDefs())
return I.Operands.NumDefs;
unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs;
@@ -360,6 +364,9 @@ unsigned CodeGenInstructionPattern::getNumInstDefs() const {
}
unsigned CodeGenInstructionPattern::getNumInstOperands() const {
+ if (isIntrinsic())
+ return IntrinInfo->IS.RetTys.size() + IntrinInfo->IS.ParamTys.size();
+
unsigned NumCGIOps = I.Operands.size();
return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size())
: NumCGIOps;
@@ -376,6 +383,9 @@ StringRef CodeGenInstructionPattern::getInstName() const {
}
void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const {
+ if (isIntrinsic())
+ OS << " intrinsic(@" << IntrinInfo->Name << ")";
+
if (!FI)
return;
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/GlobalISel/Patterns.h
index b3160552a21fef3..dac092556548d95 100644
--- a/llvm/utils/TableGen/GlobalISel/Patterns.h
+++ b/llvm/utils/TableGen/GlobalISel/Patterns.h
@@ -34,6 +34,7 @@ class SMLoc;
class StringInit;
class CodeExpansions;
class CodeGenInstruction;
+struct CodeGenIntrinsic;
namespace gi {
@@ -396,7 +397,7 @@ class OperandTable {
StringMap<InstructionPattern *> Table;
};
-//===- CodeGenInstructionPattern ------------------------------------------===//
+//===- MIFlagsInfo --------------------------------------------------------===//
/// Helper class to contain data associated with a MIFlags operand.
class MIFlagsInfo {
@@ -413,7 +414,17 @@ class MIFlagsInfo {
SetVector<StringRef> SetF, UnsetF, CopyF;
};
-/// Matches an instruction, e.g. `G_ADD $x, $y, $z`.
+//===- CodeGenInstructionPattern ------------------------------------------===//
+
+/// Matches an instruction or intrinsic:
+/// e.g. `G_ADD $x, $y, $z` or `int_amdgcn_cos $a`
+///
+/// Intrinsics are just normal instructions with a special operand for intrinsic
+/// ID. Despite G_INTRINSIC opcodes being variadic, we consider that the
+/// Intrinsic's info takes priority. This means we return:
+/// - false for isVariadic() and other variadic-related queries.
+/// - getNumInstDefs and getNumInstOperands use the intrinsic's in/out
+/// operands.
class CodeGenInstructionPattern : public InstructionPattern {
public:
CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name)
@@ -425,6 +436,10 @@ class CodeGenInstructionPattern : public InstructionPattern {
bool is(StringRef OpcodeName) const;
+ void setIntrinsic(const CodeGenIntrinsic *I) { IntrinInfo = I; }
+ const CodeGenIntrinsic *getIntrinsic() const { return IntrinInfo; }
+ bool isIntrinsic() const { return IntrinInfo; }
+
bool hasVariadicDefs() const;
bool isVariadic() const override;
unsigned getNumInstDefs() const override;
@@ -440,6 +455,7 @@ class CodeGenInstructionPattern : public InstructionPattern {
void printExtras(raw_ostream &OS) const override;
const CodeGenInstruction &I;
+ const CodeGenIntrinsic *IntrinInfo = nullptr;
std::unique_ptr<MIFlagsInfo> FI;
};
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index 124e416eea28e61..46eb4d37e49c7ec 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -27,6 +27,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenInstruction.h"
+#include "CodeGenIntrinsics.h"
#include "CodeGenTarget.h"
#include "GlobalISel/CXXPredicates.h"
#include "GlobalISel/CodeExpander.h"
@@ -401,9 +402,9 @@ void CombineRuleOperandTypeChecker::propagateAndInferTypes() {
PatternType CombineRuleOperandTypeChecker::inferImmediateType(
const InstructionPattern &IP, unsigned ImmOpIdx,
const TypeEquivalenceClasses &TECs) const {
- // We can only infer CGPs.
+ // We can only infer CGPs (except intrinsics).
const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&IP);
- if (!CGP)
+ if (!CGP || CGP->isIntrinsic())
return {};
// For CGPs, we try to infer immediates by trying to infer another named
@@ -521,14 +522,16 @@ void CombineRuleOperandTypeChecker::getInstEqClasses(
// - Iterating over the map, filtering types we don't like, and just adding
// the array of Operand Indexes to \p OutTECs.
- // We can only do this on CodeGenInstructions. Other InstructionPatterns have
- // no type inference information associated with them.
+ // We can only do this on CodeGenInstructions that aren't intrinsics. Other
+ // InstructionPatterns have no type inference information associated with
+ // them. Intrinsics also don't have any constraints to guide inference.
+
// TODO: Could we add some inference information to builtins at least? e.g.
// ReplaceReg should always replace with a reg of the same type, for instance.
// Though, those patterns are often used alone so it might not be worth the
// trouble to infer their types.
auto *CGP = dyn_cast<CodeGenInstructionPattern>(&P);
- if (!CGP)
+ if (!CGP || CGP->isIntrinsic())
return;
const auto MCOITypes = getMCOIOperandTypes(*CGP);
@@ -1251,8 +1254,8 @@ bool CombineRuleBuilder::checkSemantics() {
const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(MatchRoot);
if (!IRoot) {
- PrintError(Name +
- " can only be used if the root is a CodeGenInstruction");
+ PrintError(Name + " can only be used if the root is a "
+ "CodeGenInstruction or Intrinsic");
return false;
}
@@ -1546,6 +1549,32 @@ bool CombineRuleBuilder::parsePatternList(
return true;
}
+static const CodeGenInstruction &
+getInstrForIntrinsic(const CodeGenTarget &CGT, const CodeGenIntrinsic *I) {
+ StringRef Opc;
+ if (I->isConvergent) {
+ Opc = I->hasSideEffects ? "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"
+ : "G_INTRINSIC_CONVERGENT";
+ } else {
+ Opc = I->hasSideEffects ? "G_INTRINSIC_W_SIDE_EFFECTS" : "G_INTRINSIC";
+ }
+
+ RecordKeeper &RK = I->TheDef->getRecords();
+ return CGT.getInstruction(RK.getDef(Opc));
+}
+
+static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
+ // Intrinsics need to have a static lifetime because the match table keeps
+ // references to CodeGenIntrinsic objects.
+ static DenseMap<const Record *, std::unique_ptr<CodeGenIntrinsic>>
+ AllIntrinsics;
+
+ auto &Ptr = AllIntrinsics[R];
+ if (!Ptr)
+ Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
+ return Ptr.get();
+}
+
std::unique_ptr<Pattern>
CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
StringRef Name) const {
@@ -1558,6 +1587,14 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(RuleDef.getLoc()));
Pat =
std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+ } else if (const DagInit *IP =
+ getDagWithOperatorOfSubClass(Arg, "Intrinsic")) {
+ Record *TheDef = IP->getOperatorAsDef(RuleDef.getLoc());
+ const CodeGenIntrinsic *Intrin = getCodeGenIntrinsic(TheDef);
+ const CodeGenInstruction &Instr = getInstrForIntrinsic(CGT, Intrin);
+ Pat =
+ std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+ cast<CodeGenInstructionPattern>(*Pat).setIntrinsic(Intrin);
} else if (const DagInit *PFP =
getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) {
const Record *Def = PFP->getOperatorAsDef(RuleDef.getLoc());
@@ -1569,9 +1606,8 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) {
Pat = std::make_unique<BuiltinPattern>(
*BP->getOperatorAsDef(RuleDef.getLoc()), insertStrRef(Name));
- } else {
+ } else
return nullptr;
- }
for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) {
Init *Arg = DagPat->getArg(K);
@@ -2237,7 +2273,18 @@ bool CombineRuleBuilder::emitInstructionApplyPattern(
auto &DstMI =
M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &CGIP.getInst());
+ bool HasEmittedIntrinsicID = false;
+ const auto EmitIntrinsicID = [&]() {
+ assert(CGIP.isIntrinsic());
+ DstMI.addRenderer<IntrinsicIDRenderer>(CGIP.getIntrinsic());
+ HasEmittedIntrinsicID = true;
+ };
+
for (auto &Op : P.operands()) {
+ // Emit the intrinsic ID after the last def.
+ if (CGIP.isIntrinsic() && !Op.isDef() && !HasEmittedIntrinsicID)
+ EmitIntrinsicID();
+
if (Op.isNamedImmediate()) {
PrintError("invalid output operand '" + Op.getOperandName() +
"': output immediates cannot be named");
@@ -2326,6 +2373,11 @@ bool CombineRuleBuilder::emitInstructionApplyPattern(
DstMI.addRenderer<TempRegRenderer>(TempRegID, /*IsDef=*/true);
}
+ // Some intrinsics have no in operands, ensure the ID is still emitted in such
+ // cases.
+ if (CGIP.isIntrinsic() && !HasEmittedIntrinsicID)
+ EmitIntrinsicID();
+
// Render MIFlags
if (const auto *FI = CGIP.getMIFlagsInfo()) {
for (StringRef InstName : FI->copy_flags())
@@ -2455,6 +2507,14 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst());
declareInstExpansion(CE, IM, P.getName());
+ // If this is an intrinsic, check the intrinsic ID.
+ if (P.isIntrinsic()) {
+ // The IntrinsicID's operand is the first operand after the defs.
+ OperandMatcher &OM = IM.addOperand(P.getNumInstDefs(), "$intrinsic_id",
+ AllocatedTemporariesBaseID++);
+ OM.addPredicate<IntrinsicIDOperandMatcher>(P.getIntrinsic());
+ }
+
// Check flags if needed.
if (const auto *FI = P.getMIFlagsInfo()) {
assert(FI->copy_flags().empty());
@@ -2466,7 +2526,7 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
/*CheckNot=*/true);
}
- for (const auto &[Idx, OriginalO] : enumerate(P.operands())) {
+ for (auto [Idx, OriginalO] : enumerate(P.operands())) {
// Remap the operand. This is used when emitting InstructionPatterns inside
// PatFrags, so it can remap them to the arguments passed to the pattern.
//
@@ -2481,8 +2541,17 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
const auto OpName =
RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : "";
+
+ // For intrinsics, the first use operand is the intrinsic id, so the true
+ // operand index is shifted by 1.
+ //
+ // From now on:
+ // Idx = index in the pattern operand list.
+ // RealIdx = expected index in the MachineInstr.
+ const unsigned RealIdx =
+ (P.isIntrinsic() && !OriginalO.isDef()) ? (Idx + 1) : Idx;
OperandMatcher &OM =
- IM.addOperand(Idx, OpName, AllocatedTemporariesBaseID++);
+ IM.addOperand(RealIdx, OpName, AllocatedTemporariesBaseID++);
if (!OpName.empty())
declareOperandExpansion(CE, OM, OriginalO.getOperandName());
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
index 1ae8e30da2fb5c6..051c536f113f7bf 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
@@ -2030,6 +2030,16 @@ void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
}
+//===- IntrinsicIDRenderer ------------------------------------------------===//
+
+void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
+ RuleMatcher &Rule) const {
+ Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI")
+ << MatchTable::ULEB128Value(InsnID)
+ << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+ << MatchTable::LineBreak;
+}
+
//===- CustomRenderer -----------------------------------------------------===//
void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h
index 7cb5345f51f8cb3..8f754dd9d6922e0 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.h
@@ -1845,6 +1845,7 @@ class OperandRenderer {
OR_Register,
OR_TempRegister,
OR_ComplexPattern,
+ OR_Intrinsic,
OR_Custom,
OR_CustomOperand
};
@@ -2135,6 +2136,23 @@ class RenderComplexPatternOperand : public OperandRenderer {
void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
};
+/// Adds an intrinsic ID operand to the instruction being built.
+class IntrinsicIDRenderer : public OperandRenderer {
+protected:
+ unsigned InsnID;
+ const CodeGenIntrinsic *II;
+
+public:
+ IntrinsicIDRenderer(unsigned InsnID, const CodeGenIntrinsic *II)
+ : OperandRenderer(OR_Intrinsic), InsnID(InsnID), II(II) {}
+
+ static bool classof(const OperandRenderer *R) {
+ return R->getKind() == OR_Intrinsic;
+ }
+
+ void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
class CustomRenderer : public OperandRenderer {
protected:
unsigned InsnID;
>From e17e1dc2e6b51564b6441d9b9942d1330f5dd3c4 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 24 Jan 2024 13:39:57 +0100
Subject: [PATCH 2/2] clang-format
---
.../llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 3753ea0ee52ab29..8e456015cd37365 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -1121,9 +1121,9 @@ bool GIMatchTableExecutor::executeMatchTable(
uint16_t Value = readU16();
assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
- DEBUG_WITH_TYPE(TgtExecutor::getName(),
- dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
- << InsnID << "], " << Value << ")\n");
+ DEBUG_WITH_TYPE(TgtExecutor::getName(),
+ dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
+ << InsnID << "], " << Value << ")\n");
break;
}
case GIR_SetImplicitDefDead: {
More information about the llvm-commits
mailing list