[llvm-branch-commits] [llvm] [3/7][PISA] Add PISA register file, instruction set, and MC layer (PR #214107)
Michal Paszkowski via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Aug 4 17:56:29 PDT 2026
https://github.com/michalpaszkowski updated https://github.com/llvm/llvm-project/pull/214107
>From 5650cd1f8d730e00925eca81dc2396e25aa2e6ce Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Mon, 3 Aug 2026 03:04:59 -0700
Subject: [PATCH 1/3] Add PISA register file, instruction set, and MC layer
Add the PISA register file and register-bank descriptions, the instruction
formats and definitions, and the MC/assembly layer (instruction printer,
register and code encoders, target streamer, MC target description and the
supporting enums), replacing the stub descriptions from
the initial target skeleton.
This provides the instruction-set description and assembly-emission
foundation. The GlobalISel lowering and instruction selection that consume it
are added in the following changes.
---
llvm/include/llvm/IR/CallingConv.h | 4 +
llvm/include/llvm/MC/MCInst.h | 20 +
.../llvm/Target/PISA/MnemonicOperand.td | 344 +++
llvm/lib/MC/MCInst.cpp | 2 +
llvm/lib/Target/PISA/CMakeLists.txt | 14 +-
.../Target/PISA/MCTargetDesc/CMakeLists.txt | 5 +
.../Target/PISA/MCTargetDesc/PISABaseInfo.h | 44 +
.../lib/Target/PISA/MCTargetDesc/PISAEnum.cpp | 19 +
llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.h | 65 +
.../PISA/MCTargetDesc/PISAInstPrinter.cpp | 405 ++-
.../PISA/MCTargetDesc/PISAInstPrinter.h | 86 +-
.../Target/PISA/MCTargetDesc/PISAMCExpr.cpp | 31 +
.../lib/Target/PISA/MCTargetDesc/PISAMCExpr.h | 54 +
.../PISA/MCTargetDesc/PISAMCTargetDesc.cpp | 27 +-
.../PISA/MCTargetDesc/PISAMCTargetDesc.h | 26 +-
.../PISA/MCTargetDesc/PISARegEncoder.cpp | 183 ++
.../Target/PISA/MCTargetDesc/PISARegEncoder.h | 73 +
.../PISA/MCTargetDesc/PISATargetStreamer.cpp | 572 ++++
.../PISA/MCTargetDesc/PISATargetStreamer.h | 286 ++
llvm/lib/Target/PISA/PISA.td | 22 +-
llvm/lib/Target/PISA/PISACacheCtrlMMRA.h | 92 +
llvm/lib/Target/PISA/PISACombine.td | 1075 +++++++
llvm/lib/Target/PISA/PISADefines.h | 52 +
llvm/lib/Target/PISA/PISAGenericOpcodes.td | 99 +
llvm/lib/Target/PISA/PISAHelpers.h | 38 +
llvm/lib/Target/PISA/PISAInstrFormats.td | 834 ++++++
llvm/lib/Target/PISA/PISAInstrInfo.cpp | 449 ++-
llvm/lib/Target/PISA/PISAInstrInfo.h | 32 +-
llvm/lib/Target/PISA/PISAInstrInfo.td | 2522 ++++++++++++++++-
llvm/lib/Target/PISA/PISAMCInstLower.cpp | 134 +
llvm/lib/Target/PISA/PISAMCInstLower.h | 103 +
llvm/lib/Target/PISA/PISARegManager.cpp | 62 +
llvm/lib/Target/PISA/PISARegManager.h | 47 +
llvm/lib/Target/PISA/PISARegisterBanks.td | 15 +
llvm/lib/Target/PISA/PISARegisterInfo.cpp | 272 +-
llvm/lib/Target/PISA/PISARegisterInfo.h | 55 +
llvm/lib/Target/PISA/PISARegisterInfo.td | 260 +-
llvm/lib/Target/PISA/PISAUtils.cpp | 52 +
llvm/lib/Target/PISA/PISAUtils.h | 36 +
39 files changed, 8448 insertions(+), 63 deletions(-)
create mode 100644 llvm/include/llvm/Target/PISA/MnemonicOperand.td
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISABaseInfo.h
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.cpp
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.h
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.cpp
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.h
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.h
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
create mode 100644 llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.h
create mode 100644 llvm/lib/Target/PISA/PISACacheCtrlMMRA.h
create mode 100644 llvm/lib/Target/PISA/PISACombine.td
create mode 100644 llvm/lib/Target/PISA/PISADefines.h
create mode 100644 llvm/lib/Target/PISA/PISAGenericOpcodes.td
create mode 100644 llvm/lib/Target/PISA/PISAHelpers.h
create mode 100644 llvm/lib/Target/PISA/PISAInstrFormats.td
create mode 100644 llvm/lib/Target/PISA/PISAMCInstLower.cpp
create mode 100644 llvm/lib/Target/PISA/PISAMCInstLower.h
create mode 100644 llvm/lib/Target/PISA/PISARegManager.cpp
create mode 100644 llvm/lib/Target/PISA/PISARegManager.h
create mode 100644 llvm/lib/Target/PISA/PISARegisterBanks.td
create mode 100644 llvm/lib/Target/PISA/PISAUtils.cpp
create mode 100644 llvm/lib/Target/PISA/PISAUtils.h
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index 249c512dda532..7992dac627a76 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -297,6 +297,9 @@ namespace CallingConv {
/// stateless compartment.
CHERIoT_LibraryCall = 127,
+ /// Used for PISA kernel functions.
+ PISA_KERNEL = 183,
+
/// The highest possible ID. Must be some 2^k - 1.
MaxID = 1023
};
@@ -323,6 +326,7 @@ constexpr bool isCallableCC(CallingConv::ID CC) {
case CallingConv::AMDGPU_PS:
case CallingConv::AMDGPU_VS:
case CallingConv::SPIR_KERNEL:
+ case CallingConv::PISA_KERNEL:
case CallingConv::PTX_Kernel:
return false;
default:
diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h
index b0db6b8408da5..adba42f4ac1fa 100644
--- a/llvm/include/llvm/MC/MCInst.h
+++ b/llvm/include/llvm/MC/MCInst.h
@@ -42,6 +42,7 @@ class MCOperand {
kInvalid, ///< Uninitialized.
kRegister, ///< Register operand.
kImmediate, ///< Immediate operand.
+ kHFPImmediate, ///< Half-floating-point immediate operand.
kSFPImmediate, ///< Single-floating-point immediate operand.
kDFPImmediate, ///< Double-Floating-point immediate operand.
kExpr, ///< Relocatable immediate operand.
@@ -52,6 +53,7 @@ class MCOperand {
union {
unsigned RegVal;
int64_t ImmVal;
+ uint16_t HFPImmVal;
uint32_t SFPImmVal;
uint64_t FPImmVal;
const MCExpr *ExprVal;
@@ -64,6 +66,7 @@ class MCOperand {
bool isValid() const { return Kind != kInvalid; }
bool isReg() const { return Kind == kRegister; }
bool isImm() const { return Kind == kImmediate; }
+ bool isHFPImm() const { return Kind == kHFPImmediate; }
bool isSFPImm() const { return Kind == kSFPImmediate; }
bool isDFPImm() const { return Kind == kDFPImmediate; }
bool isExpr() const { return Kind == kExpr; }
@@ -91,6 +94,16 @@ class MCOperand {
ImmVal = Val;
}
+ uint16_t getHFPImm() const {
+ assert(isHFPImm() && "This is not an HFP immediate");
+ return HFPImmVal;
+ }
+
+ void setHFPImm(uint16_t Val) {
+ assert(isHFPImm() && "This is not an HFP immediate");
+ HFPImmVal = Val;
+ }
+
uint32_t getSFPImm() const {
assert(isSFPImm() && "This is not an SFP immediate");
return SFPImmVal;
@@ -149,6 +162,13 @@ class MCOperand {
return Op;
}
+ static MCOperand createHFPImm(uint16_t Val) {
+ MCOperand Op;
+ Op.Kind = kHFPImmediate;
+ Op.HFPImmVal = Val;
+ return Op;
+ }
+
static MCOperand createSFPImm(uint32_t Val) {
MCOperand Op;
Op.Kind = kSFPImmediate;
diff --git a/llvm/include/llvm/Target/PISA/MnemonicOperand.td b/llvm/include/llvm/Target/PISA/MnemonicOperand.td
new file mode 100644
index 0000000000000..ecdfe71282df5
--- /dev/null
+++ b/llvm/include/llvm/Target/PISA/MnemonicOperand.td
@@ -0,0 +1,344 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// DAG operator used for dag representation of a condition flag: (cflag FlagName value)
+// These are used in EnumOptionOpnd to represent certain
+def cflag;
+
+// Function to find index of a item from a list of string.
+// Note that tablegen does not allow recursive definition.
+class getIndexFromList<list<string> slist, string item> {
+ int Idx = !cond(!and(!ge(!size(slist),1), !eq(slist[0],item)) : 0,
+ !and(!ge(!size(slist),2), !eq(slist[1],item)) : 1,
+ !and(!ge(!size(slist),3), !eq(slist[2],item)) : 2,
+ !and(!ge(!size(slist),4), !eq(slist[3],item)) : 3,
+ !and(!ge(!size(slist),5), !eq(slist[4],item)) : 4,
+ !and(!ge(!size(slist),6), !eq(slist[5],item)) : 5,
+ !and(!ge(!size(slist),7), !eq(slist[6],item)) : 6,
+ !and(!ge(!size(slist),8), !eq(slist[7],item)) : 7,
+ !and(!ge(!size(slist),9), !eq(slist[8],item)) : 8,
+ !and(!ge(!size(slist),10),!eq(slist[9],item)) : 9,
+ !and(!ge(!size(slist),11),!eq(slist[10],item)) : 10,
+ !and(!ge(!size(slist),12),!eq(slist[11],item)) : 11,
+ !and(!ge(!size(slist),13),!eq(slist[12],item)) : 12,
+ !and(!ge(!size(slist),14),!eq(slist[13],item)) : 13,
+ !and(!ge(!size(slist),15),!eq(slist[14],item)) : 14,
+ !and(!ge(!size(slist),16),!eq(slist[15],item)) : 15,
+ true : -1);
+}
+
+// Flags are encoded from msb - 1 - numflags to lsb
+class getShiftedFlag<list<string> cFlagNames, string flagName> {
+ defvar LsbIdx = !sub(!sub(!size(cFlagNames), 1), getIndexFromList<cFlagNames, flagName>.Idx);
+ bits<16> enc = !shl(1, LsbIdx);
+}
+
+// Get bits enabled for all flags in daglist regardless of the value.
+// e.g. if flag f1 is positioned at 2^2 and f2 in 2^0,
+// [(cflag "f1", true), (cflag "f2", false)] -> {... 0, 1, 0, 1}
+class EncodeCFlagMask<list<string> cFlagNames, list<dag> cFlags> {
+ bits<16> enc = !foldl(0, cFlags, acc, fdag,
+ !or(getShiftedFlag<cFlagNames, !getdagarg<string>(fdag, "name")>.enc, acc));
+}
+
+// Get bits enables for all flags in daglist if the value is true.
+// e.g. if flag f1 is positioned at 2^2 and f2 in 2^0,
+// [(cflag "f1", true), (cflag "f2", false)] -> {... 0, 1, 0, 0}
+class EncodeCFlag<list<string> cFlagNames, list<dag> cFlags> {
+ bits<16> enc = !foldl(0, cFlags, acc, fdag,
+ !or(!and(!getdagarg<bit>(fdag, "val"),
+ getShiftedFlag<cFlagNames, !getdagarg<string>(fdag, "name")>.enc),
+ acc));
+}
+
+
+// ----- Definitions for EnumOptions.
+// Base definitions for operands that have a value selected from a
+// set of limited options.
+
+// Convert list of flag bits into a single accumulated value in bits<16>
+class EncodeCFlagFromList<list<bit> bitList> {
+ bits<16> enc = !foldl(0, bitList, acc, flag, !or(!shl(acc, 1), flag));
+}
+
+// Enum to list all type of EnumOptions
+def EnumOptionClass: GenericEnum {
+ let FilterClass = "EnumOptionClassEntry";
+ let NameField = "EnumName";
+}
+
+// Lookup name string for enum types
+def EnumOptionStrTable : GenericTable {
+ let FilterClass = "EnumOptionClassEntry";
+ let CppTypeName = "EnumOptionStrEntry";
+ let Fields = ["Type", "Str"];
+ string TypeOf_Type = "EnumOptionClass";
+ let PrimaryKey = ["Type"];
+ let PrimaryKeyName = "lookupEnumOptionStr";
+}
+
+class EnumOptionClassEntry<string name> {
+ EnumOptionClassEntry Type = !cast<EnumOptionClassEntry>(NAME);
+ string EnumName = "EO_"#name;
+ string Str = name;
+}
+
+// Base class used to define per option enum entries
+class EnumOptionEntry<EnumOptionClassEntry ClassID, string name, int val, string enumPrefix="", string optStr=!tolower(name), string enumName="", list<bit> cFlags=[]> {
+ EnumOptionClassEntry OptClass = ClassID;
+ EnumOptionEntry EnumValue = !cast<EnumOptionEntry>(NAME);
+ string OptStr = optStr;
+
+ // Capitalize the name and add prefix for enum type
+ string EnumName = !cond(!not(!empty(enumName)) : enumName,
+ !empty(enumPrefix) : !toupper(name),
+ true : enumPrefix#"_"#!toupper(name));
+
+ // Need "bits<bit_size> Value" in all inheriting classes
+ bits<32> Value = val;
+
+ // Flags to classify the enum values
+ bits<16> CFlagEnc = EncodeCFlagFromList<cFlags>.enc;
+}
+
+// Store EnumOption related info
+class EnumOptionInfo<EnumOptionClassEntry OptID, ValueType vt, string enumPrefix,
+ bit isMandatory, int defaultVal,
+ Operand opnd, list<string> cFlagNames> {
+ EnumOptionClassEntry OptClassID = OptID;
+ ValueType VT = vt;
+ string EnumPrefix = enumPrefix;
+ bit IsMandatory = isMandatory;
+ int DefaultVal = defaultVal;
+ Operand Opnd = opnd;
+ list<string> CFlagNames = cFlagNames;
+}
+
+class EnumOptionEntryDef<EnumOptionInfo info, string name, int val, string optStr=!tolower(name), string enumName="">
+ : EnumOptionEntry<info.OptClassID, name, val, info.EnumPrefix, optStr, enumName>;
+
+def EnumOptionTable : GenericTable {
+ let FilterClass = "EnumOptionEntry";
+ let Fields = ["OptClass", "Value", "OptStr", "CFlagEnc"];
+ string TypeOf_OptClass = "EnumOptionClass";
+ // FIXME: Is there a way to get EnumName printed in the table
+ // rather than int literals?
+ let PrimaryKey = ["OptClass", "Value"];
+ let PrimaryKeyName = "lookupEnumOptionByValue";
+}
+
+def lookupEnumOptionByOptStr : SearchIndex {
+ let Table = EnumOptionTable;
+ let Key = ["OptClass", "OptStr"];
+}
+
+def lookupEnumOptionsWithClass : SearchIndex {
+ let Table = EnumOptionTable;
+ let Key = ["OptClass"];
+}
+
+// Base for defining enum and lookup tables
+multiclass EnumTypeDef<string EnumEntryClassName=NAME#"Entry", string enumPrefix=""> {
+ // Define enum definition
+ // User must define a entry class named as EnumEntryClassName by inheriting EnumOptionEntry
+ def NAME : GenericEnum {
+ let FilterClass = EnumEntryClassName;
+ let NameField = "EnumName";
+ let ValueField = "Value";
+ string EnumPrefix = enumPrefix;
+ }
+}
+
+class EnumOptionOpndPMC<string name,
+ string optClassStr,
+ bit isMandatory = false,
+ int defaultVal = 0,
+ bits<16> CFlagMask = 0,
+ bits<16> CFlagCond = 0> : AsmOperandClass {
+ let Name = name;
+ let ParserMethod = "parseEnumOptionOpnd("#optClassStr#","#isMandatory#","#defaultVal#")";
+ let ParserMethod = "parseEnumOptionOpnd("#optClassStr#","#isMandatory#","#CFlagMask#","#CFlagCond#","#defaultVal#")";
+ let PredicateMethod = "isEnumOptionOpnd("#optClassStr#")";
+ let RenderMethod = "addEnumOptionOpnd("#optClassStr#")";
+}
+
+class EnumOptionOpnd<ValueType vt,
+ string opndName,
+ EnumOptionClassEntry optClassID,
+ bit isMandatory = false,
+ int defaultVal = 0> : Operand<vt> {
+ // Mask that need to filter relevant flags
+ bits<16> CFlagMask = 0;
+ // Matched value after masking
+ bits<16> CFlagCond = 0;
+ let OperandType = "OPERAND_IMMEDIATE";
+ let PrintMethod = "printEnumOption("#optClassID.EnumName#","#isMandatory#","#defaultVal#")";
+ let ParserMatchClass = EnumOptionOpndPMC<opndName, optClassID.EnumName, isMandatory, defaultVal, CFlagMask, CFlagCond>;
+}
+
+multiclass EnumOptionOpndDef<
+ ValueType vt,
+ string enumPrefix,
+ bit isMandatory = false,
+ int defaultVal = 0,
+ list<string> cFlagNames = [],
+ string EnumEntryClassName = NAME#"Entry"> {
+ // Define enum type representing the enum class
+ defvar TypeName = NAME#"ClassID";
+ def TypeName : EnumOptionClassEntry<NAME>;
+ defvar optClassID = !cast<EnumOptionClassEntry>(TypeName);
+ defm NAME : EnumTypeDef<EnumEntryClassName, enumPrefix>;
+
+ defvar OpndName = NAME#"Opnd";
+ def OpndName
+ : EnumOptionOpnd<vt, EnumEntryClassName#"Opnd", optClassID, isMandatory, defaultVal> ;
+ def NAME#"EnumOptionInfo"
+ : EnumOptionInfo<optClassID, vt, enumPrefix,
+ isMandatory, defaultVal,
+ !cast<Operand>(OpndName), cFlagNames>;
+}
+
+class EnumOptionIDEntry;
+
+def EnumOptionID : GenericEnum {
+ let FilterClass = "EnumOptionIDEntry";
+}
+
+class EnumOptionTableEntry<string optstr>{
+ EnumOptionIDEntry OptID = !cast<EnumOptionIDEntry>(NAME);
+ string OptStr = optstr;
+}
+
+class EnumOptionOpndParserMatchClass<string OptID> : AsmOperandClass {
+ let Name = NAME;
+ let ParserMethod = "parseEnumOptionOpnd("#OptID#")";
+ let PredicateMethod = "isEnumOptionOpnd("#OptID#")";
+ let RenderMethod = "addEnumOptionOpnd("#OptID#")";
+}
+
+class EnumOptionOpndWithDefaultOps<
+ string OptID, EnumOptionOpndParserMatchClass pmc, bit isMandatory = false,
+ int defaultVal = 0> :
+ OperandWithDefaultOps<i32, (ops(i32 defaultVal))> {
+ let OperandType = "OPERAND_IMMEDIATE";
+ let MIOperandInfo = (ops i32imm);
+ let PrintMethod = "printEnumOption("#OptID#","#isMandatory#","#defaultVal#")";
+ let ParserMatchClass = pmc;
+}
+
+multiclass EnumOptionOpndWithDefaultOpsDef<
+ string enumPrefix, bit isMandatory = false, int defaultVal = 0,
+ string EnumEntryClassName = NAME#"Entry"> {
+ defvar TypeName = NAME#"ClassID";
+ def TypeName : EnumOptionClassEntry<NAME>;
+ defvar optClassID = !cast<EnumOptionClassEntry>(TypeName);
+ defm NAME : EnumTypeDef<EnumEntryClassName, enumPrefix>;
+
+ defvar OptID = "EO_"#NAME;
+ defvar OpndName = NAME#"Opnd";
+
+ def NAME#"OpndPMC" : EnumOptionOpndParserMatchClass<OptID> {
+ let ParserMethod = "parseEnumOptionOpnd("#OptID#","#isMandatory#",0,0,"#defaultVal#")";
+ }
+
+ def OpndName : EnumOptionOpndWithDefaultOps<
+ OptID, !cast<EnumOptionOpndParserMatchClass>(NAME#"OpndPMC"), isMandatory,
+ defaultVal>;
+}
+
+// Selects subset of given enum types to be used as an operand type.
+// cFlags are condition flags used to confine enum values to ones
+// that matches given flag name and values.
+// e.g. [(cflag "MyFlag1", true), (cflag "MyFlag2", false)] can be given
+// to only select enum values with MyFlag1 = true and MyFlag2 = false.
+class EnumOptionSubsetOpnd<EnumOptionInfo EOInfo, list<dag> cFlags>
+ : EnumOptionOpnd<EOInfo.VT, NAME#"Opnd", EOInfo.OptClassID, EOInfo.IsMandatory, EOInfo.DefaultVal> {
+ // This is used to mask only relevant flag bits this operand needs to match
+ let CFlagMask = EncodeCFlagMask<EOInfo.CFlagNames, cFlags>.enc;
+ // Flags values to be matched after masking.
+ let CFlagCond = EncodeCFlag<EOInfo.CFlagNames, cFlags>.enc;
+}
+
+// EnableFlag: Tells which bool value is valid for the option
+// 1: Accept only disabled, 2: Accept only enabled, 3: Accept both
+class BoolOptionOpndParserMatchClass<string OptID, int EnableFlag=3> : AsmOperandClass {
+ let Name = NAME;
+ let ParserMethod = "parseBoolOptionOpnd("#OptID#", "#EnableFlag#")";
+ let PredicateMethod = "isBoolOptionOpnd("#OptID#","#EnableFlag#")";
+ let RenderMethod = "addBoolOptionOpnd("#OptID#")";
+}
+
+class BoolOptionOpnd<string OptID, BoolOptionOpndParserMatchClass pmc> : Operand<i16> {
+ let OperandType = "OPERAND_IMMEDIATE";
+
+ // Pass optString as name
+ let PrintMethod = "printBoolOptionOpnd("#OptID#")";
+ let ParserMatchClass = pmc;
+}
+
+class BoolOptionOpndWithDefaultOps<string OptID, BoolOptionOpndParserMatchClass pmc, bit defaultVal = 0> : OperandWithDefaultOps<i16, (ops(i16 defaultVal))> {
+ let OperandType = "OPERAND_IMMEDIATE";
+
+ // Pass optString as name
+ let PrintMethod = "printBoolOptionOpnd("#OptID#")";
+ let ParserMatchClass = pmc;
+}
+
+// Assign IDs for each bool options
+def BoolOptionID : GenericEnum {
+ let FilterClass = "BoolOptionIDEntry";
+}
+
+// Table with option id to option string mapping
+def BoolOptionTable : GenericTable {
+ let FilterClass = "BoolOptionTableEntry";
+ let Fields = ["OptID", "OptStr"];
+ string TypeOf_OptID = "BoolOptionID";
+ let PrimaryKey = ["OptID"];
+ let PrimaryKeyName = "lookupBoolOptionByID";
+}
+
+def lookupBoolOptionByOptStr : SearchIndex {
+ let Table = BoolOptionTable;
+ let Key = ["OptStr"];
+}
+
+class BoolOptionIDEntry;
+
+class BoolOptionTableEntry<string optstr>{
+ BoolOptionIDEntry OptID = !cast<BoolOptionIDEntry>(NAME);
+ string OptStr = optstr;
+}
+
+// Dummy entry to ensure BoolOptionTable is never empty
+def BO_DUMMY_ : BoolOptionIDEntry, BoolOptionTableEntry<"">;
+
+multiclass DefBoolOptionOpndType<string OptID, int EnableFlag = 3> {
+ defvar PMCName = NAME#"OpndPMC";
+ def PMCName : BoolOptionOpndParserMatchClass<OptID, EnableFlag>;
+ def NAME#"Opnd" : BoolOptionOpnd<OptID, !cast<BoolOptionOpndParserMatchClass>(PMCName)>;
+}
+
+// DisabledOpnd type does not accept option but creates MIR operand with false value
+// EnabledOpnd mandates optional field to be present and create true operand
+multiclass BoolOptionOpndDef<string OptStr=!tolower(NAME), bit DefineDerivedOpnds = false> {
+ def "BO_"#NAME : BoolOptionIDEntry, BoolOptionTableEntry<OptStr>;
+ defvar OptID = "BO_"#NAME;
+ defm NAME : DefBoolOptionOpndType<OptID, 3>;
+ if DefineDerivedOpnds then {
+ defm NAME#"Disabled" : DefBoolOptionOpndType<OptID, 1>;
+ defm NAME#"Enabled" : DefBoolOptionOpndType<OptID, 2>;
+ }
+}
+
+multiclass BoolOptionOpndWithDefaultOpsDef<string OptStr=!tolower(NAME), bit defaultVal = 0> {
+ def "BO_"#NAME : BoolOptionIDEntry, BoolOptionTableEntry<OptStr>;
+ defvar OptID = "BO_"#NAME;
+ def NAME#"OpndPMC" : BoolOptionOpndParserMatchClass<OptID>;
+ def NAME#"Opnd" : BoolOptionOpndWithDefaultOps<OptID, !cast<BoolOptionOpndParserMatchClass>(NAME#"OpndPMC"), defaultVal>;
+}
diff --git a/llvm/lib/MC/MCInst.cpp b/llvm/lib/MC/MCInst.cpp
index f29cb48a16ece..7684f872c90fd 100644
--- a/llvm/lib/MC/MCInst.cpp
+++ b/llvm/lib/MC/MCInst.cpp
@@ -32,6 +32,8 @@ void MCOperand::print(raw_ostream &OS, const MCContext *Ctx) const {
OS << getReg().id();
} else if (isImm())
OS << "Imm:" << getImm();
+ else if (isHFPImm())
+ OS << "HFPImm:" << getHFPImm();
else if (isSFPImm())
OS << "SFPImm:" << bit_cast<float>(getSFPImm());
else if (isDFPImm())
diff --git a/llvm/lib/Target/PISA/CMakeLists.txt b/llvm/lib/Target/PISA/CMakeLists.txt
index fafef2a3c3fe8..5c3bfe6ec02bf 100644
--- a/llvm/lib/Target/PISA/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/CMakeLists.txt
@@ -1,36 +1,46 @@
add_llvm_component_group(PISA)
set(LLVM_TARGET_DEFINITIONS PISA.td)
-
tablegen(LLVM PISAGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM PISAGenInstrInfo.inc -gen-instr-info)
+tablegen(LLVM PISAGenMCCodeEmitter.inc -gen-emitter)
tablegen(LLVM PISAGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM PISAGenSubtargetInfo.inc -gen-subtarget)
+tablegen(LLVM PISAGenSearchableTables.inc -gen-searchable-tables)
add_public_tablegen_target(PISACommonTableGen)
add_llvm_target(PISACodeGen
PISAInstrInfo.cpp
+ PISAMCInstLower.cpp
+ PISARegManager.cpp
PISARegisterInfo.cpp
PISASubtarget.cpp
PISATargetMachine.cpp
+ PISAUtils.cpp
LINK_COMPONENTS
Analysis
+ AsmPrinter
CodeGen
CodeGenTypes
Core
+ Demangle
+ GlobalISel
MC
+ Passes
+ Scalar
SelectionDAG
Support
Target
TargetParser
+ TransformUtils
PISADesc
PISAInfo
ADD_TO_COMPONENT
PISA
- )
+)
add_subdirectory(MCTargetDesc)
add_subdirectory(TargetInfo)
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
index 1e7639e93d484..80a21ad6c6aa2 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
@@ -1,7 +1,11 @@
add_llvm_component_library(LLVMPISADesc
+ PISAEnum.cpp
PISAMCAsmInfo.cpp
PISAMCTargetDesc.cpp
+ PISATargetStreamer.cpp
PISAInstPrinter.cpp
+ PISAMCExpr.cpp
+ PISARegEncoder.cpp
LINK_COMPONENTS
CodeGenTypes
@@ -14,3 +18,4 @@ add_llvm_component_library(LLVMPISADesc
ADD_TO_COMPONENT
PISA
)
+
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISABaseInfo.h b/llvm/lib/Target/PISA/MCTargetDesc/PISABaseInfo.h
new file mode 100644
index 0000000000000..a396a106f28e7
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISABaseInfo.h
@@ -0,0 +1,44 @@
+//===-- PISABaseInfo.h - Top level PISA definitions -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISABASEINFO_H
+#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISABASEINFO_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include <string>
+
+namespace llvm {
+
+// Return a string representation of the operands from startIndex onwards.
+// Templated to allow both MachineInstr and MCInst to use the same logic.
+template <class InstType>
+std::string getPISAStringOperand(const InstType &MI, unsigned StartIndex) {
+ std::string S; // Iteratively append to this string.
+
+ const unsigned NumOps = MI.getNumOperands();
+ bool IsFinished = false;
+ for (unsigned I = StartIndex; I < NumOps && !IsFinished; ++I) {
+ const auto &Op = MI.getOperand(I);
+ if (!Op.isImm()) // Stop if we hit a register operand.
+ break;
+ assert((Op.getImm() >> 32) == 0 && "Imm operand should be i32 word");
+ const uint32_t Imm = Op.getImm(); // Each i32 word is up to 4 characters.
+ for (unsigned ShiftAmount = 0; ShiftAmount < 32; ShiftAmount += 8) {
+ char C = (Imm >> ShiftAmount) & 0xff;
+ if (C == 0) { // Stop if we hit a null-terminator character.
+ IsFinished = true;
+ break;
+ }
+ S += C; // Otherwise, append the character to the result string.
+ }
+ }
+ return S;
+}
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISABASEINFO_H
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.cpp
new file mode 100644
index 0000000000000..4dd7d019bb66f
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.cpp
@@ -0,0 +1,19 @@
+//===-- PISAEnum.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISAEnum.h"
+
+namespace llvm {
+namespace PISA {
+
+#define GET_BoolOptionTable_IMPL
+#define GET_EnumOptionTable_IMPL
+#include "PISAGenSearchableTables.inc"
+
+} // namespace PISA
+} // namespace llvm
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.h
new file mode 100644
index 0000000000000..77c47b724c27c
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAEnum.h
@@ -0,0 +1,65 @@
+//===-- PISAEnum.h --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAENUM_H
+#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAENUM_H
+
+#include "MCTargetDesc/PISAMCExpr.h"
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "MCTargetDesc/PISARegEncoder.h"
+#include "MCTargetDesc/PISATargetStreamer.h"
+#include "PISADefines.h"
+#include "PISAMCInstLower.h"
+#include "TargetInfo/PISATargetInfo.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringTable.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCParser/AsmLexer.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm::PISA;
+
+namespace llvm {
+namespace PISA {
+
+#define GET_BoolOptionID_DECL
+#define GET_EnumOptionClass_DECL
+#define GET_LoadCacheControl_DECL
+#define GET_StoreCacheControl_DECL
+#define GET_AtomicCacheControl_DECL
+#include "PISAGenSearchableTables.inc"
+
+struct BoolOptionTableEntry {
+ BoolOptionID OptID;
+ StringTable::Offset OptStr;
+};
+
+struct EnumOptionEntry {
+ EnumOptionClass OptClass;
+ unsigned Value;
+ StringTable::Offset OptStr;
+ unsigned CFlags;
+};
+
+#define GET_BoolOptionTable_DECL
+#define GET_EnumOptionTable_DECL
+#include "PISAGenSearchableTables.inc"
+
+} // namespace PISA
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAENUM_H
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.cpp
index f16a2e1177fc8..3d39eed23963c 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.cpp
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.cpp
@@ -7,29 +7,412 @@
//===----------------------------------------------------------------------===//
#include "PISAInstPrinter.h"
+#include "MCTargetDesc/PISARegEncoder.h"
+#include "PISA.h"
+#include "PISABaseInfo.h"
+#include "PISAEnum.h"
+#include "PISAInstrInfo.h"
+#include "PISAMCExpr.h"
+#include "PISAMCInstLower.h"
+#include "PISAUtils.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/IR/PISAIntrinsicUtils.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormattedStream.h"
using namespace llvm;
+using namespace llvm::PISA;
-#include "PISAGenAsmWriter.inc"
+#define DEBUG_TYPE "asm-printer"
-void PISAInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
- OS << getRegisterName(Reg);
-}
+static cl::opt<bool>
+ AlwaysPrintDefaultValue("pisa-always-print-default-value",
+ cl::desc("Always print default value"),
+ cl::init(false), cl::ReallyHidden);
+
+// Include the auto-generated portion of the assembly writer.
+#include "PISAGenAsmWriter.inc"
void PISAInstPrinter::printInst(const MCInst *MI, uint64_t Address,
StringRef Annot, const MCSubtargetInfo &STI,
raw_ostream &OS) {
+ this->STI = &STI;
+
printInstruction(MI, Address, OS);
+
printAnnotation(OS, Annot);
}
-void PISAInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
+void PISAInstPrinter::printRegName(raw_ostream &OS, MCRegister Reg) {
+ if (MCRegister::isPhysicalRegister(Reg)) {
+ OS << getRegisterName(Reg);
+ } else {
+ auto [Prefix, Idx] = RegEncoder::decodeVirtualRegister(Reg);
+ OS << Prefix << Idx;
+ }
+}
+
+void PISAInstPrinter::printImm1Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printImm8Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printImm16Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printImm32Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printImm64Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printImm128Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << formatImm(MCOp.getImm());
+}
+
+void PISAInstPrinter::printFpImm16Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << format_hex(MCOp.getHFPImm(), /*Width=*/6, /*Upper=*/true);
+}
+
+void PISAInstPrinter::printFpImm32Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << format_hex(MCOp.getSFPImm(), /*Width=*/10, /*Upper=*/true);
+}
+
+void PISAInstPrinter::printFpImm64Opnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ OS << format_hex(MCOp.getDFPImm(), /*Width=*/18, /*Upper=*/true);
+}
+
+void PISAInstPrinter::printRegOpnd(unsigned RCID, const MCInst *MCI,
+ unsigned OpNo, raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ printRegName(OS, MCOp.getReg());
+ printSwizzle(MCI, OpNo, OS);
+}
+
+void PISAInstPrinter::printBrTargetOpnd(const MCInst *MCI, uint64_t Address,
+ unsigned OpNo, raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ MAI.printExpr(OS, *MCOp.getExpr());
+}
+
+void PISAInstPrinter::printLocalVariableOpnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ assert(PISAMCInstLower::isVariableRef(*MCI, OpNo));
+ printOperand(MCI, OpNo, OS);
+}
+
+void PISAInstPrinter::printGlobalVariableOpnd(const MCInst *MCI,
+ uint64_t Address, unsigned OpNo,
+ raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ MAI.printExpr(OS, *MCOp.getExpr());
+}
+
+void PISAInstPrinter::printMemScopeOpnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS) {
+ static const char *MemScopeStrs[] = {
+ ".system", // pisa::MemoryScope::system
+ ".gpu", // pisa::MemoryScope::gpu
+ ".workgroup", // pisa::MemoryScope::workgroup
+ ".subgroup", // pisa::MemoryScope::subgroup
+ };
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ const unsigned Scope = MCOp.getImm();
+ assert(Scope <= pisa::MemoryScope::subgroup && "Invalid memory scope");
+ OS << MemScopeStrs[Scope];
+}
+
+void PISAInstPrinter::printSwizzle(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
- const MCOperand &Op = MI->getOperand(OpNo);
- if (Op.isReg())
- printRegName(O, Op.getReg());
- else if (Op.isImm())
- O << Op.getImm();
+ auto Swizzle = PISAMCInstLower::getSwizzle(*MI, OpNo);
+ swizzleRepr(O, static_cast<unsigned>(Swizzle));
+}
+
+void PISAInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O, const char *Modifier) {
+ assert((Modifier == 0 || Modifier[0] == 0) && "No modifiers supported");
+ if (PISAMCInstLower::isVariableRef(*MI, OpNo)) {
+ assert(MI->getOperand(OpNo).isImm() && "expecting imm operand");
+ unsigned int FrameIndex = MI->getOperand(OpNo).getImm();
+ O << "@R" << FrameIndex;
+ } else if (OpNo < MI->getNumOperands()) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isReg()) {
+ printRegName(O, Op.getReg());
+ printSwizzle(MI, OpNo, O);
+ } else if (Op.isImm())
+ O << formatImm((int64_t)Op.getImm());
+ else if (Op.isSFPImm())
+ printFpImm32Opnd(MI, OpNo, O);
+ else if (Op.isDFPImm())
+ printFpImm64Opnd(MI, OpNo, O);
+ else if (Op.isExpr())
+ MAI.printExpr(O, *Op.getExpr());
+ else
+ llvm_unreachable("Unexpected operand type");
+ }
+}
+
+void PISAInstPrinter::negateRepr(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O) {
+ int64_t DoNegate = MI->getOperand(OpNo).getImm();
+ if (DoNegate)
+ O << "!";
+}
+
+void PISAInstPrinter::swizzleRepr(raw_ostream &O, unsigned SwizzleVal) {
+ switch (static_cast<Swizzle>(SwizzleVal)) {
+ case Swizzle::X:
+ O << ".x";
+ break;
+ case Swizzle::Y:
+ O << ".y";
+ break;
+ case Swizzle::Z:
+ O << ".z";
+ break;
+ case Swizzle::W:
+ O << ".w";
+ break;
+ case Swizzle::XYZW:
+ O << ".xyzw";
+ break;
+ case Swizzle::XY:
+ O << ".xy";
+ break;
+ case Swizzle::ZW:
+ O << ".zw";
+ break;
+ case Swizzle::NONE:
+ break;
+ }
+}
+
+void PISAInstPrinter::printFunctionCallTargetOpnd(const MCInst *MI,
+ unsigned OpNo,
+ raw_ostream &O) {
+ printOperand(MI, OpNo, O);
+
+ // print function args if any
+ O << " (";
+ for (unsigned ArgIdx = OpNo + 1; ArgIdx < MI->getNumOperands(); ++ArgIdx) {
+ printOperand(MI, ArgIdx, O);
+ if (ArgIdx + 1 < MI->getNumOperands())
+ O << ", ";
+ }
+ O << ");";
+}
+
+void PISAInstPrinter::printAddrOffsetImm(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O) {
+ int64_t Val = MI->getOperand(OpNo).getImm();
+ // print imm offset only when it's not zero
+ if (Val > 0) {
+ O << " + " << formatImm(Val);
+ } else if (Val < 0) {
+ if (Val == std::numeric_limits<int64_t>::min())
+ O << " - " << llvm::format("%" PRIu64, Val);
+ else
+ O << " - " << formatImm(-Val);
+ }
+}
+
+void PISAInstPrinter::printBfnOpcode(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O) {
+ int64_t Val = MI->getOperand(OpNo).getImm();
+ O << format_hex(Val & 0xFF, 4);
+}
+
+void PISAInstPrinter::printStringImm(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O) {
+ const unsigned NumOps = MI->getNumOperands();
+ unsigned StrStartIndex = OpNo;
+ while (StrStartIndex < NumOps) {
+ if (MI->getOperand(StrStartIndex).isReg())
+ break;
+
+ std::string Str = getPISAStringOperand(*MI, OpNo);
+ if (StrStartIndex != OpNo)
+ O << ' '; // Add a space if we're starting a new string/argument.
+ O << '"';
+ for (char C : Str) {
+ if (C == '"')
+ O.write('\\'); // Escape " characters (might break for complex UTF-8).
+ O.write(C);
+ }
+ O << '"';
+
+ unsigned NumOpsInString = (Str.size() / 4) + 1;
+ StrStartIndex += NumOpsInString;
+ }
+}
+
+void PISAInstPrinter::printMemOperand(const MCInst *MI, int OpNo,
+ raw_ostream &OS,
+ const char * /*Modifier*/) {
+ // [ Base OP Offset ]
+ OS << "[";
+ printOperand(MI, OpNo, OS);
+ const MCOperand &OffsetOp = MI->getOperand(OpNo + 1);
+ if (OffsetOp.isImm()) {
+ auto Val = OffsetOp.getImm();
+ if (Val != 0) {
+ if (Val > 0)
+ OS << " + " << formatImm(Val);
+ else if (Val == std::numeric_limits<int64_t>::min())
+ OS << " - " << llvm::format("%" PRIu64, Val);
+ else
+ OS << " - " << formatImm(-Val);
+ }
+ } else {
+ assert(OffsetOp.isReg() && "Register expected");
+ OS << " + ";
+ printRegName(OS, OffsetOp.getReg());
+ printSwizzle(MI, (OpNo + 1), OS);
+ }
+ OS << "]";
+}
+
+void PISAInstPrinter::printMemSeqOperand(StringRef Pattern, const MCInst *MI,
+ int OpNo, raw_ostream &OS,
+ const char * /* Modifier */) {
+ OS << "[";
+ for (char C : Pattern) {
+ switch (C) {
+ case '{':
+ OS << "{";
+ break;
+ case '}':
+ OS << "}";
+ break;
+ case ',':
+ OS << ", ";
+ break;
+ default:
+ printOperand(MI, OpNo++, OS);
+ break;
+ }
+ }
+ OS << "]";
+}
+
+void PISAInstPrinter::printParamMemOperand(const MCInst *MCI, int OpNo,
+ raw_ostream &OS) {
+ const MCOperand &ArgIdxOp = MCI->getOperand(OpNo);
+ const MCOperand &OffsetOp = MCI->getOperand(OpNo + 1);
+ assert(ArgIdxOp.isImm());
+
+ // Check for an extra expression operand carrying the kernel argument name.
+ unsigned NameOpIdx = OpNo + 2;
+ if (NameOpIdx < MCI->getNumOperands() &&
+ MCI->getOperand(NameOpIdx).isExpr()) {
+ const auto *SRE =
+ cast<MCSymbolRefExpr>(MCI->getOperand(NameOpIdx).getExpr());
+ OS << "[%" << SRE->getSymbol().getName();
+ } else {
+ OS << "[%arg" << ArgIdxOp.getImm();
+ }
+
+ if (OffsetOp.isImm()) {
+ int64_t Offset = OffsetOp.getImm();
+ if (Offset != 0)
+ OS << llvm::format("%+" PRId64, Offset);
+ } else {
+ assert(OffsetOp.isReg() && "Register expected");
+ OS << " + ";
+ printRegName(OS, OffsetOp.getReg());
+ printSwizzle(MCI, (OpNo + 1), OS);
+ }
+ OS << "]";
+}
+
+void PISAInstPrinter::printSymbolName(raw_ostream &OS, StringRef Name,
+ const MCAsmInfo *MAI) {
+ // This is a modification of MCSymbol::print() that prints non-printable
+ // characters differently.
+ if (!MAI || MAI->isValidUnquotedName(Name)) {
+ OS << Name;
+ return;
+ }
+
+ if (MAI && !MAI->supportsNameQuoting())
+ report_fatal_error("Symbol name with unsupported characters");
+
+ OS << '"';
+ for (char C : Name) {
+ if (C == '\n')
+ OS << "\\n";
+ else if (C == '"')
+ OS << "\\\"";
+ else if (isPrint(C))
+ OS << C;
+ else
+ OS << '\\' << hexdigit(C >> 4) << hexdigit(C & 0x0F);
+ }
+ OS << '"';
+}
+
+void PISAInstPrinter::printEnumOption(EnumOptionClass OptClass,
+ bool IsMandatoryField,
+ unsigned DefaultVal, const MCInst *MCI,
+ unsigned OpNo, raw_ostream &OS) {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+
+ unsigned Val = MCOp.getImm();
+ if (!IsMandatoryField && Val == DefaultVal)
+ return;
+
+ const EnumOptionEntry *Entry = lookupEnumOptionByValue(OptClass, Val);
+ assert(Entry && "Enum value not found from lookup table");
+ if (!Entry)
+ return;
+
+ StringRef OptStrRef = getEnumOptionEntryStr(Entry->OptStr);
+ if (!OptStrRef.empty())
+ OS << "." << OptStrRef;
+}
+
+PISAInstPrinter::printOpndFn
+PISAInstPrinter::printBoolOptionOpnd(PISA::BoolOptionID OptID) {
+ return [=](const MCInst *MCI, unsigned OpNo, raw_ostream &OS) -> void {
+ const MCOperand &MCOp = MCI->getOperand(OpNo);
+ if (MCOp.getImm()) {
+ StringRef OptStr =
+ getBoolOptionTableEntryStr(lookupBoolOptionByID(OptID)->OptStr);
+ OS << "." << OptStr;
+ }
+ };
}
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
index 11ab2987397bb..9094807ab82ca 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
@@ -9,24 +9,106 @@
#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAINSTPRINTER_H
#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAINSTPRINTER_H
+#include "MCTargetDesc/PISABaseInfo.h"
+#include "MCTargetDesc/PISAEnum.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/MC/MCInstPrinter.h"
namespace llvm {
-
class PISAInstPrinter : public MCInstPrinter {
+
public:
using MCInstPrinter::MCInstPrinter;
+ typedef std::function<void(const MCInst *, unsigned, raw_ostream &)>
+ printOpndFn;
+ bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+ void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+ unsigned OpIdx, unsigned PrintMethodIdx,
+ raw_ostream &OS);
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
void printRegName(raw_ostream &OS, MCRegister Reg) override;
- void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O,
+ const char *Modifier = nullptr);
+ printOpndFn printBoolOptionOpnd(PISA::BoolOptionID OptID);
+ void printEnumOption(PISA::EnumOptionClass OptClass, bool IsMandatoryField,
+ unsigned DefaultVal, const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS);
+
+ printOpndFn printEnumOption(PISA::EnumOptionClass OptClass,
+ bool IsMandatoryField = false,
+ unsigned DefaultVal = 0) {
+ return [=](const MCInst *MCI, unsigned OpNo, raw_ostream &OS) -> void {
+ switch (OptClass) {
+ default:
+ printEnumOption(OptClass, IsMandatoryField, DefaultVal, MCI, OpNo, OS);
+ }
+ };
+ }
+
+ void printImm1Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printImm8Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printImm16Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printImm32Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printImm64Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printImm128Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printFpImm16Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printFpImm32Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+ void printFpImm64Opnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+
+ void printRegOpnd(unsigned RCID, const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS);
+ auto printRegOpnd(unsigned RCID) {
+ return [=](const MCInst *MCI, unsigned OpNo, raw_ostream &OS) {
+ printRegOpnd(RCID, MCI, OpNo, OS);
+ };
+ }
+
+ void printBrTargetOpnd(const MCInst *MCI, uint64_t Address, unsigned OpNo,
+ raw_ostream &OS);
+ void printLocalVariableOpnd(const MCInst *MCI, unsigned OpNo,
+ raw_ostream &OS);
+ void printGlobalVariableOpnd(const MCInst *MCI, uint64_t Address,
+ unsigned OpNo, raw_ostream &OS);
+
+ void printMemScopeOpnd(const MCInst *MCI, unsigned OpNo, raw_ostream &OS);
+
+ void negateRepr(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void printStringImm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void printAddrOffsetImm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void printBfnOpcode(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void printMemOperand(const MCInst *MI, int OpNo, raw_ostream &O,
+ const char *Modifier = nullptr);
+ void printMemSeqOperand(StringRef Pattern, const MCInst *MI, int OpNo,
+ raw_ostream &OS, const char *Modifier);
+ auto printMemSeqOperand(StringRef Pattern) {
+ return [=](const MCInst *MI, int OpNo, raw_ostream &OS,
+ const char *Modifier = nullptr) {
+ printMemSeqOperand(Pattern, MI, OpNo, OS, Modifier);
+ };
+ }
+ void printParamMemOperand(const MCInst *MI, int OpNo, raw_ostream &O);
+
+ // print FunctionCallTarget operand (reg or function name) and the sub-sequent
+ // function arguments
+ void printFunctionCallTargetOpnd(const MCInst *MI, unsigned OpNo,
+ raw_ostream &O);
// Autogenerated by tblgen.
std::pair<const char *, uint64_t>
getMnemonic(const MCInst &MI) const override;
void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
static const char *getRegisterName(MCRegister Reg);
+ static void printSymbolName(raw_ostream &OS, StringRef Name,
+ const MCAsmInfo *MAI);
+
+private:
+ // Caching STI for member functions to check features.
+ const MCSubtargetInfo *STI = nullptr;
+
+ void printSwizzle(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+ void swizzleRepr(raw_ostream &O, unsigned SwizzleVal);
};
} // namespace llvm
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.cpp
new file mode 100644
index 0000000000000..e431b48250c7d
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.cpp
@@ -0,0 +1,31 @@
+//===-- PISAMCExpr.cpp - Handle custom MCExprs ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAMCExpr.h"
+#include "PISAInstPrinter.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/Support/Format.h"
+using namespace llvm;
+
+const PISAGlobalAddressMCExpr *
+PISAGlobalAddressMCExpr::create(const MCSymbol &Symbol, MCContext &Ctx) {
+ return new (Ctx) PISAGlobalAddressMCExpr(Symbol);
+}
+
+PISAGlobalAddressMCExpr::PISAGlobalAddressMCExpr(const MCSymbol &Sym)
+ : Symbol(&Sym) {
+ assert(Symbol);
+}
+
+void PISAGlobalAddressMCExpr::printImpl(raw_ostream &OS,
+ const MCAsmInfo *MAI) const {
+ OS << "@";
+ PISAInstPrinter::printSymbolName(OS, Symbol->getName(), MAI);
+}
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.h
new file mode 100644
index 0000000000000..6d195f4da8144
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCExpr.h
@@ -0,0 +1,54 @@
+//===-- PISAMCExpr.h ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// PISA special MCTargetExpr class to model floating point immediates
+// Modeled after ARMMCExpr
+
+#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAMCEXPR_H
+#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAMCEXPR_H
+
+#include "llvm/ADT/APFloat.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCSymbol.h"
+#include <utility>
+
+namespace llvm {
+
+/// MCExpr for Global Address.
+/// The operand is the address to a global variable (.e.g. @foo).
+/// It required a prefix "@" for the symbol name
+class PISAGlobalAddressMCExpr : public MCTargetExpr {
+
+private:
+ const MCSymbol *Symbol;
+ explicit PISAGlobalAddressMCExpr(const MCSymbol &Symbol);
+
+public:
+ static const PISAGlobalAddressMCExpr *create(const MCSymbol &Symbol,
+ MCContext &Ctx);
+
+ const MCSymbol &getSymbol() const { return *Symbol; }
+
+ void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+
+ bool evaluateAsRelocatableImpl(MCValue &Res,
+ const MCAssembler *Asm) const override {
+ return false;
+ }
+
+ void visitUsedExpr(MCStreamer &Streamer) const override {}
+ MCFragment *findAssociatedFragment() const override { return nullptr; }
+
+ static bool classof(const MCExpr *E) {
+ return E->getKind() == MCExpr::Target;
+ }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAMCEXPR_H
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
index 5dab8ea1bba59..3ffd668f3d05a 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
@@ -9,14 +9,19 @@
#include "PISAMCTargetDesc.h"
#include "PISAInstPrinter.h"
#include "PISAMCAsmInfo.h"
+#include "PISATargetStreamer.h"
#include "TargetInfo/PISATargetInfo.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/TargetRegistry.h"
+#include "llvm/TargetParser/SubtargetFeature.h"
#define GET_INSTRINFO_MC_DESC
#define ENABLE_INSTR_PREDICATE_VERIFIER
+#define GET_INSTRINFO_OPERAND_ENUM
+#define GET_INSTRINFO_NAMED_OPS
#include "PISAGenInstrInfo.inc"
#define GET_SUBTARGETINFO_MC_DESC
@@ -41,7 +46,7 @@ static MCRegisterInfo *createPISAMCRegisterInfo(const Triple &TT) {
static MCSubtargetInfo *createPISAMCSubtargetInfo(const Triple &TT,
StringRef CPU, StringRef FS) {
- return createPISAMCSubtargetInfoImpl(TT, CPU, /*TuneCPU=*/CPU, FS);
+ return createPISAMCSubtargetInfoImpl(TT, CPU, /*TuneCPU*/ CPU, FS);
}
static MCInstPrinter *createPISAMCInstPrinter(const Triple &T,
@@ -55,10 +60,18 @@ static MCInstPrinter *createPISAMCInstPrinter(const Triple &T,
// NOLINTNEXTLINE(readability-identifier-naming)
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePISATargetMC() {
- Target *T = &getThePISATarget();
- RegisterMCAsmInfo<PISAMCAsmInfo> X(*T);
- TargetRegistry::RegisterMCInstrInfo(*T, createPISAMCInstrInfo);
- TargetRegistry::RegisterMCRegInfo(*T, createPISAMCRegisterInfo);
- TargetRegistry::RegisterMCSubtargetInfo(*T, createPISAMCSubtargetInfo);
- TargetRegistry::RegisterMCInstPrinter(*T, createPISAMCInstPrinter);
+ [[maybe_unused]] static bool Initialized = []() {
+ Target *T = &getThePISATarget();
+ RegisterMCAsmInfo<PISAMCAsmInfo> X(*T);
+ TargetRegistry::RegisterMCInstrInfo(*T, createPISAMCInstrInfo);
+ TargetRegistry::RegisterMCRegInfo(*T, createPISAMCRegisterInfo);
+ TargetRegistry::RegisterMCSubtargetInfo(*T, createPISAMCSubtargetInfo);
+ TargetRegistry::RegisterMCInstPrinter(*T, createPISAMCInstPrinter);
+ TargetRegistry::RegisterAsmTargetStreamer(*T, createPISAAsmTargetStreamer);
+ TargetRegistry::RegisterObjectTargetStreamer(
+ *T, createPISAObjectTargetStreamer);
+ TargetRegistry::RegisterNullTargetStreamer(*T,
+ createPISANullTargetStreamer);
+ return true;
+ }();
}
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
index cb47d07cf4565..625d0f16077cf 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
@@ -14,13 +14,37 @@
#include <memory>
namespace llvm {
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstPrinter;
class MCInstrInfo;
+class MCObjectTargetWriter;
class MCRegisterInfo;
+class MCStreamer;
class MCSubtargetInfo;
+class MCTargetOptions;
+class MCTargetStreamer;
class Target;
+class formatted_raw_ostream;
+
+namespace PISA {
+enum OperandType : unsigned {
+ OPERAND_NEGATE = MCOI::OPERAND_FIRST_TARGET,
+ OPERAND_SWIZZLE,
+};
+} // namespace PISA
+
+MCTargetStreamer *createPISAAsmTargetStreamer(MCStreamer &S,
+ formatted_raw_ostream &OS,
+ MCInstPrinter *InstPrint);
+MCTargetStreamer *createPISAObjectTargetStreamer(MCStreamer &S,
+ const MCSubtargetInfo &STI);
+MCTargetStreamer *createPISANullTargetStreamer(MCStreamer &S);
+
} // namespace llvm
-// Defines symbolic names for PISA registers. This defines a mapping from
+// Defines symbolic names for PISA registers. This defines a mapping from
// register name to register number.
#define GET_REGINFO_ENUM
#include "PISAGenRegisterInfo.inc"
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
new file mode 100644
index 0000000000000..25364bda0ab87
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
@@ -0,0 +1,183 @@
+//===-- PISARegEncoder.cpp - Encode PISA virtual registers ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISARegEncoder.h"
+
+using namespace llvm;
+using namespace PISA;
+
+const char *RegEncoder::getPrefixFromBank(RegBank Bank) {
+
+ const auto NumBanks = static_cast<size_t>(RegBank::NUM_BANK);
+
+ const char *const BankPrefixes[NumBanks] = {
+ "%p", "%b", "%h", "%w", "%d", "%q", "%v2b", "%v2h", "%v2w",
+ "%v2d", "%v3b", "%v3h", "%v3w", "%v3d", "%v4b", "%v4h", "%v4w", "%v4d",
+ "%v5w", "%v6w", "%v7w", "%v8w", "%v16w", "%v32w", "%v64w"};
+
+ auto Index = static_cast<size_t>(Bank);
+ if (Index >= NumBanks)
+ llvm_unreachable("Unknown register bank!");
+
+ return BankPrefixes[Index];
+}
+
+RegEncoder::RegBank RegEncoder::getRegBank(unsigned NumElts, unsigned EltSize) {
+ switch (NumElts) {
+ case 1:
+ switch (EltSize) {
+ case 1:
+ return RegBank::Reg1;
+ case 8:
+ return RegBank::Reg8;
+ case 16:
+ return RegBank::Reg16;
+ case 32:
+ return RegBank::Reg32;
+ case 64:
+ return RegBank::Reg64;
+ case 128:
+ return RegBank::Reg128;
+ default:
+ llvm_unreachable("Unknown element size!");
+ }
+ break;
+ case 2:
+ switch (EltSize) {
+ case 8:
+ return RegBank::RegV2_8;
+ case 16:
+ return RegBank::RegV2_16;
+ case 32:
+ return RegBank::RegV2_32;
+ case 64:
+ return RegBank::RegV2_64;
+ default:
+ llvm_unreachable("Unknown element size!");
+ }
+ break;
+ case 3:
+ switch (EltSize) {
+ case 8:
+ return RegBank::RegV3_8;
+ case 16:
+ return RegBank::RegV3_16;
+ case 32:
+ return RegBank::RegV3_32;
+ case 64:
+ return RegBank::RegV3_64;
+ default:
+ llvm_unreachable("Unknown element size!");
+ }
+ break;
+ case 4:
+ switch (EltSize) {
+ case 8:
+ return RegBank::RegV4_8;
+ case 16:
+ return RegBank::RegV4_16;
+ case 32:
+ return RegBank::RegV4_32;
+ case 64:
+ return RegBank::RegV4_64;
+ default:
+ llvm_unreachable("Unknown element size!");
+ }
+ break;
+ case 5:
+ if (EltSize == 32)
+ return RegBank::RegV5_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 6:
+ if (EltSize == 32)
+ return RegBank::RegV6_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 7:
+ if (EltSize == 32)
+ return RegBank::RegV7_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 8:
+ if (EltSize == 32)
+ return RegBank::RegV8_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 16:
+ if (EltSize == 32)
+ return RegBank::RegV16_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 32:
+ if (EltSize == 32)
+ return RegBank::RegV32_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ case 64:
+ if (EltSize == 32)
+ return RegBank::RegV64_32;
+ else
+ llvm_unreachable("Unknown element size!");
+ break;
+ default:
+ llvm_unreachable("Unknown number of elements!");
+ return RegBank::NUM_BANK;
+ }
+}
+
+// 3 bits used for type, 5 bits used for bank, 24 bits used for index
+static constexpr unsigned NumTypeBits = 3;
+static constexpr unsigned NumBankBits = 5;
+static constexpr unsigned NumRegBits = 24;
+static_assert(RegEncoder::NUM_TYPE < 5, "need to update encoding");
+static_assert(static_cast<unsigned>(RegEncoder::RegBank::NUM_BANK) < 32,
+ "need to update encoding");
+
+RegEncoder::RegType RegEncoder::getRegType(uint8_t TSFlags) {
+ unsigned TypeBitsMask = (1U << NumTypeBits) - 1;
+ return static_cast<RegEncoder::RegType>(TSFlags & TypeBitsMask);
+}
+
+RegEncoder::RegType RegEncoder::getRegType(const TargetRegisterClass *RC) {
+ return getRegType(RC->TSFlags);
+}
+
+RegEncoder::RegBank RegEncoder::getRegBank(uint8_t TSFlags) {
+ unsigned BankBitsMask = (1U << NumBankBits) - 1;
+ return static_cast<RegEncoder::RegBank>(TSFlags & BankBitsMask);
+}
+
+unsigned RegEncoder::encodeVirtualRegister(unsigned Idx, RegBank Bank,
+ RegType Type) {
+ // Check that NumRegBits is sufficient to encode the register Idx
+ auto RegBank = static_cast<unsigned>(Bank);
+ assert(Idx < (1U << NumRegBits) &&
+ "Register index exceeds virtual register encoding limit");
+ return Register::index2VirtReg((Type << (NumRegBits + NumBankBits)) |
+ RegBank << (NumRegBits) |
+ (Idx & ((1U << NumRegBits) - 1)));
+}
+
+std::pair<const char *, unsigned>
+RegEncoder::decodeVirtualRegister(MCRegister Reg) {
+ auto BankBits = (Reg >> NumRegBits) & ((1U << NumBankBits) - 1);
+ const char *Prefix = getPrefixFromBank(getRegBank(BankBits));
+ unsigned Num = Reg & ((1U << NumRegBits) - 1);
+ return std::make_pair(Prefix, Num);
+}
+
+bool RegEncoder::isVirtualRegNo(unsigned RegNo) {
+ return getRegType(RegNo >> (NumRegBits + NumBankBits)) != NONE;
+}
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.h b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.h
new file mode 100644
index 0000000000000..a4778cae1ed92
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.h
@@ -0,0 +1,73 @@
+//===-- PISARegEncoder.h - Encode PISA virtual registers ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAREGENCODER_H
+#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAREGENCODER_H
+
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+
+namespace llvm {
+namespace PISA {
+
+class RegEncoder {
+public:
+ enum RegType {
+ NONE,
+ // Keep in sync with TSFlags in PISARegisterInfo.td
+ REG,
+ PRED,
+ NUM_TYPE
+ };
+
+ enum class RegBank : unsigned {
+ Reg1,
+ Reg8,
+ Reg16,
+ Reg32,
+ Reg64,
+ Reg128,
+ RegV2_8,
+ RegV2_16,
+ RegV2_32,
+ RegV2_64,
+ RegV3_8,
+ RegV3_16,
+ RegV3_32,
+ RegV3_64,
+ RegV4_8,
+ RegV4_16,
+ RegV4_32,
+ RegV4_64,
+ RegV5_32,
+ RegV6_32,
+ RegV7_32,
+ RegV8_32,
+ RegV16_32,
+ RegV32_32,
+ RegV64_32,
+ NUM_BANK
+ };
+
+ static RegBank getRegBank(unsigned NumElts, unsigned EltSize);
+ static const char *getPrefixFromBank(RegBank Bank);
+ static bool isVirtualRegNo(unsigned RegNo);
+ static unsigned encodeVirtualRegister(unsigned Idx, RegBank Bank,
+ RegType Type);
+ static std::pair<const char *, unsigned>
+ decodeVirtualRegister(MCRegister Reg);
+
+protected:
+ static RegEncoder::RegBank getRegBank(uint8_t TSFlags);
+ static RegEncoder::RegType getRegType(uint8_t TSFlags);
+ static RegEncoder::RegType getRegType(const TargetRegisterClass *RC);
+};
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISAREGENCODER_H
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
new file mode 100644
index 0000000000000..1443af35549b9
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
@@ -0,0 +1,572 @@
+//===-- PISATargetStreamer.cpp - PISATargetStreamer class -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISATargetStreamer.h"
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "MCTargetDesc/PISARegEncoder.h"
+#include "PISAInstPrinter.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGenTypes/LowLevelType.h"
+#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+using namespace llvm;
+
+PISA::StorageSpace PISA::mapAddrSpaceToStorageSpace(unsigned AS) {
+ switch (PISAAS::AddressSpace(AS)) {
+ case PISAAS::AddressSpace::PRIVATE:
+ return PISA::StorageSpace::PRIVATE;
+ case PISAAS::AddressSpace::GLOBAL:
+ return PISA::StorageSpace::GLOBAL;
+ case PISAAS::AddressSpace::CONSTANT:
+ return PISA::StorageSpace::CONSTANT;
+ case PISAAS::AddressSpace::SHARED:
+ return PISA::StorageSpace::SHARED;
+ case PISAAS::AddressSpace::GENERIC:
+ return PISA::StorageSpace::GENERIC;
+ }
+ llvm_unreachable("Unexpected address space!");
+}
+
+static StringRef getStorageSpaceRepr(PISA::StorageSpace SS) {
+ switch (SS) {
+ case PISA::StorageSpace::PRIVATE:
+ return "private";
+ case PISA::StorageSpace::GLOBAL:
+ return "global";
+ case PISA::StorageSpace::CONSTANT:
+ return "const";
+ case PISA::StorageSpace::SHARED:
+ return "shared";
+ case PISA::StorageSpace::GENERIC:
+ return "generic";
+ }
+ llvm_unreachable("Invalid storage space!");
+}
+
+static StringRef getParamASRepr(PISAAS::AddressSpace AS) {
+ switch (AS) {
+ case PISAAS::AddressSpace::GLOBAL:
+ return "global";
+ case PISAAS::AddressSpace::CONSTANT:
+ return "const";
+ case PISAAS::AddressSpace::SHARED:
+ return "shared";
+ case PISAAS::AddressSpace::GENERIC:
+ return "generic";
+ default:
+ break;
+ }
+ llvm_unreachable("Invalid parameter address space!");
+}
+
+StringRef PISA::getLinkageTyName(PISA::LinkageTy Linkage) {
+ switch (Linkage) {
+ case DEFAULT:
+ return "default";
+ case EXPORT:
+ return "export";
+ case IMPORT:
+ return "import";
+ }
+ llvm_unreachable("Invalid linkage type!");
+}
+
+static StringRef getCallingConvRepr(CallingConv::ID CC) {
+ switch (CC) {
+ case CallingConv::PISA_KERNEL:
+ return ".kernel";
+ default:
+ return ".function";
+ }
+}
+
+static std::string getKernelAttributeRepr(const KernelAttribute &Attr) {
+
+ DenseMap<PISA::KernelAttributeType, StringRef> AvailableMetadataNodeTypes = {
+ {PISA::KernelAttributeType::REQD_WORK_GROUP_SIZE,
+ ".reqd_work_group_size"},
+ {PISA::KernelAttributeType::VEC_TYPE_HINT, ".vec_type_hint"}};
+
+ std::string Result;
+ auto It = AvailableMetadataNodeTypes.find(Attr.KernelAttrType);
+ if (It != AvailableMetadataNodeTypes.end()) {
+ Result = It->second;
+ }
+
+ switch (Attr.KernelAttrType) {
+ case llvm::PISA::KernelAttributeType::REQD_WORK_GROUP_SIZE: {
+ Result += "(";
+ const auto &Values = std::get<std::vector<uint32_t>>(Attr.KernelAttrValues);
+ if (!Values.empty()) {
+ Result +=
+ std::accumulate(std::next(Values.begin()), Values.end(),
+ std::to_string(Values.front()),
+ [](const std::string &Acc, uint32_t El) {
+ return Acc + std::string(", ") + std::to_string(El);
+ });
+ }
+ Result += ")";
+ } break;
+ case llvm::PISA::KernelAttributeType::VEC_TYPE_HINT: {
+ Result += "(";
+ const auto &Arg = std::get<std::string>(Attr.KernelAttrValues);
+ Result += Arg;
+ Result += ")";
+ } break;
+ }
+ return Result;
+}
+
+static void emitTypeString(uint32_t TypeSizeInBits, uint32_t NumElts,
+ raw_ostream &OS) {
+ assert(NumElts > 0);
+ if (NumElts == 1 && TypeSizeInBits == 1) {
+ OS << ".pred";
+ return;
+ }
+
+ if (NumElts > 1)
+ OS << ".v" << NumElts;
+ OS << "." << TypeSizeInBits << "b";
+}
+
+static void emitTypeString(const LLT &T, raw_ostream &OS) {
+ auto TypeBitSize = T.getScalarType().getSizeInBits();
+ auto NumElts = T.isVector() ? T.getNumElements() : 1;
+ emitTypeString(TypeBitSize, NumElts, OS);
+}
+
+namespace {
+
+class PISATargetAsmStreamer final : public PISATargetStreamer {
+ formatted_raw_ostream &OS;
+ const MCAsmInfo *MAI = nullptr;
+ void printName(raw_ostream &OS, StringRef Name) {
+ PISAInstPrinter::printSymbolName(OS, Name, MAI);
+ }
+
+public:
+ PISATargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS)
+ : PISATargetStreamer(S), OS(OS), MAI(&S.getContext().getAsmInfo()) {}
+ ~PISATargetAsmStreamer() override;
+
+ StringRef emitHeader(const PISA::HeaderDcl &HD) override {
+ if (HD.Version != 0)
+ OS << ".version " << (HD.Version / 100) << "." << (HD.Version % 100)
+ << ";\n";
+ if (!HD.Target.empty())
+ OS << ".target " << HD.Target << ";\n";
+ return HD.Target;
+ }
+
+ void emitGlobalVariable(const PISA::GlobalVariableDcl &GV) override {
+ if (GV.Dcl.Linkage != PISA::LinkageTy::DEFAULT)
+ OS << "." << getLinkageTyName(GV.Dcl.Linkage) << " ";
+
+ // globals must have .const or .global addrspace
+ switch (GV.Dcl.SS) {
+ case PISA::StorageSpace::GLOBAL:
+ OS << ".global ";
+ break;
+ case PISA::StorageSpace::CONSTANT:
+ OS << ".const ";
+ break;
+ default:
+ llvm_unreachable("Invalid storage space for global variables");
+ }
+
+ auto PrintOffset = [](int64_t Val, raw_ostream &OS) {
+ // print imm offset only when it's not zero
+ if (Val > 0) {
+ OS << "+" << format("%" PRId64, Val);
+ } else if (Val < 0) {
+ if (Val == std::numeric_limits<int64_t>::min())
+ OS << "-" << format("%" PRIu64, Val);
+ else
+ OS << "-" << format("%" PRId64, -Val);
+ }
+ };
+
+ OS << " .align " << GV.Dcl.Alignment.value() << " ";
+
+ if (!GV.Dcl.Section.empty())
+ OS << ".section(\"" << GV.Dcl.Section << "\") ";
+
+ if (!GV.Dcl.HostAccessName.empty())
+ OS << ".host_access(\"" << GV.Dcl.HostAccessName << "\") ";
+
+ OS << "@";
+ printName(OS, GV.Dcl.Name);
+
+ if (GV.Init.Initializer.empty()) {
+ OS << "[" << GV.Dcl.Size << "]";
+ } else {
+ OS << " = { ";
+ for (auto [i, X] : llvm::enumerate(GV.Init.Initializer)) {
+ OS << ((i == 0) ? "" : ", ");
+ if (auto Iter = GV.Init.Exprs.find(i); Iter != GV.Init.Exprs.end()) {
+ auto &Entry = Iter->second;
+ if (auto *G = std::get_if<PISA::VariableInit::GlobalExpr>(&Entry)) {
+ OS << "." << X.Type.getSizeInBits() << "b ";
+ OS << "@" << G->Name;
+ PrintOffset(G->Offset, OS);
+ } else if (auto *Z = std::get_if<PISA::VariableInit::Zeros>(&Entry)) {
+ OS << ".zero " << Z->N;
+ } else {
+ llvm_unreachable("unknown construct!");
+ }
+ } else {
+ TypeSize SizeInBits = X.Type.getSizeInBits();
+ OS << "." << SizeInBits << "b ";
+ OS << format_hex(X.Value & maskTrailingOnes<uint64_t>(SizeInBits), 1);
+ }
+ }
+ OS << " }";
+ }
+
+ OS << ";\n";
+ }
+
+ void emitFunctionSignature(const PISA::FunctionSignature &Sig) override {
+ if (Sig.DN.CC != CallingConv::PISA_KERNEL) {
+ switch (Sig.DN.Linkage) {
+ case PISA::LinkageTy::EXPORT:
+ OS << ".export ";
+ break;
+ case PISA::LinkageTy::IMPORT:
+ OS << ".import ";
+ break;
+ default:
+ break;
+ }
+ }
+
+ OS << getCallingConvRepr(Sig.DN.CC) << " ";
+
+ for (auto KernelAttr : Sig.DN.KernelAttrs) {
+ OS << getKernelAttributeRepr(KernelAttr) << " ";
+ }
+
+ if (Sig.DN.CC != CallingConv::PISA_KERNEL) {
+ if (Sig.DN.RetLLT.isValid())
+ emitTypeString(Sig.DN.RetLLT, OS);
+ else
+ OS << "void";
+ OS << " ";
+ }
+
+ OS << "@";
+ printName(OS, Sig.DN.Name);
+
+ OS << "(";
+ if (Sig.DN.CC != CallingConv::PISA_KERNEL) {
+ const char *Sep = "";
+ for (auto &Param : Sig.FunctionParams) {
+ OS << Sep << ".reg ";
+ emitTypeString(Param.Ty, OS);
+ OS << " " << Param.Prefix << Param.Idx;
+ Sep = ", ";
+ }
+ } else {
+ const char *Sep = "";
+ unsigned I = 0;
+ for (auto &Param : Sig.KernelParams) {
+ OS << Sep << ".param[" << Param.Size << "] ";
+ if (Param.hasAlign())
+ OS << ".align(" << Param.Align << ") ";
+ if (Param.hasAS())
+ OS << ".addrspace("
+ << getParamASRepr(static_cast<PISAAS::AddressSpace>(Param.AS))
+ << ") ";
+ if (Param.hasPtrAlign())
+ OS << ".ptr_align(" << Param.PtrAlign << ") ";
+ if (!Param.ArgName.empty())
+ OS << "%" << Param.ArgName;
+ else
+ OS << "%arg" << I;
+ I++;
+ Sep = ", ";
+ }
+ }
+
+ OS << ")";
+
+ OS << "\n";
+ }
+
+ void emitFunctionDeclaration(const PISA::FunctionDeclaration &Dcl) override {
+ assert(Dcl.DN.CC != CallingConv::PISA_KERNEL);
+
+ if (Dcl.DN.Linkage != PISA::LinkageTy::IMPORT) {
+ return;
+ }
+
+ OS << ".import ";
+ OS << getCallingConvRepr(Dcl.DN.CC) << " ";
+
+ if (Dcl.DN.RetLLT.isValid()) {
+ emitTypeString(Dcl.DN.RetLLT, OS);
+ } else
+ OS << "void";
+
+ OS << " @";
+ printName(OS, Dcl.DN.Name);
+
+ OS << "(";
+ const char *Sep = "";
+ for (auto &Param : Dcl.FunctionParams) {
+ OS << Sep << ".reg ";
+ emitTypeString(Param.Ty, OS);
+ Sep = ", ";
+ }
+ OS << ");";
+ }
+
+ void addRegDcl(unsigned NumElts, unsigned EltSize, unsigned RegType,
+ RegNamesRange RegNames, bool MustAdd) override {
+ llvm_unreachable("Never used.");
+ }
+
+ void emitRegDcls(const PISA::RegDcls &Dcls,
+ const PISA::DataTypes &DTs) override {
+ for (auto &[Key, TI] : DTs) {
+ // Skip if 0 registers of this type were declared
+ if (TI.RegStart == TI.RegCounter)
+ continue;
+
+ auto [NumElts, BitWidth, RegType] = Key;
+
+ switch (RegType) {
+ case PISA::RegEncoder::PRED:
+ OS << "\t.pred ";
+ break;
+ case PISA::RegEncoder::REG:
+ OS << "\t.reg ";
+ emitTypeString(BitWidth, NumElts, OS);
+ OS << " ";
+ break;
+ default:
+ llvm_unreachable("unknown reg type!");
+ }
+
+ // Do not use range syntax if only 5 or fewer registers were declared
+ if (TI.RegCounter - TI.RegStart <= 5) {
+ for (unsigned I = TI.RegStart; I < TI.RegCounter; ++I) {
+ if (I != TI.RegStart)
+ OS << ", ";
+ OS << TI.Prefix << I;
+ }
+ OS << ";\n";
+ } else {
+ OS << TI.Prefix << "<" << TI.RegStart << "~" << (TI.RegCounter)
+ << ">;\n";
+ }
+ }
+ }
+
+ void addLocalVariableDecl(const PISA::LocalVariableDcl &V) override {
+ OS << "\t." << getStorageSpaceRepr(V.SS) << " ";
+ OS << ".align " << V.Alignment.value() << " ";
+ OS << "@";
+ // If PISA comes from PISA reader StackIndex won't be set, so we fall
+ // back and use already parsed name.
+ if (V.StackIndex == -1) {
+ OS << V.Name;
+ } else {
+ OS << "R" << V.StackIndex;
+ }
+ OS << '[' << V.Size << "];\n";
+ }
+
+ void emitLocalVariableDcls(const PISA::LocalVariableDcls &Dcls) override {
+ for (auto &V : Dcls.Vars)
+ addLocalVariableDecl(V);
+ }
+
+ void emitFuncBodyStart() override { OS << "{\n"; }
+ void emitFuncBodyEnd() override { OS << "}\n"; }
+
+ bool isAsmStreamer() const override { return true; }
+ int getCurLine() const override {
+ // getLine() in OS is 0 based
+ return OS.getLine() + 1;
+ }
+};
+
+class PISATargetELFStreamer final : public PISATargetStreamer {
+public:
+ PISATargetELFStreamer(MCStreamer &S) : PISATargetStreamer(S) {}
+ ~PISATargetELFStreamer() override;
+
+ StringRef emitHeader(const PISA::HeaderDcl &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitGlobalVariable(const PISA::GlobalVariableDcl &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitFunctionSignature(const PISA::FunctionSignature &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitFunctionDeclaration(const PISA::FunctionDeclaration &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitRegDcls(const PISA::RegDcls &, const PISA::DataTypes &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitLocalVariableDcls(const PISA::LocalVariableDcls &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitFuncBodyStart() override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void emitFuncBodyEnd() override { llvm_unreachable("Not implemented yet!"); }
+
+ void addRegDcl(unsigned NumElts, unsigned EltSize, unsigned RegType,
+ RegNamesRange RegNames, bool MustAdd = true) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+ void addLocalVariableDecl(const PISA::LocalVariableDcl &) override {
+ llvm_unreachable("Not implemented yet!");
+ }
+};
+
+class PISATargetNullStreamer final : public PISATargetStreamer {
+public:
+ PISATargetNullStreamer(MCStreamer &S) : PISATargetStreamer(S) {}
+ ~PISATargetNullStreamer() override;
+
+ StringRef emitHeader(const PISA::HeaderDcl &) override { return ""; }
+ void emitGlobalVariable(const PISA::GlobalVariableDcl &) override {}
+ void emitFunctionSignature(const PISA::FunctionSignature &) override {}
+ void emitFunctionDeclaration(const PISA::FunctionDeclaration &) override {}
+ void emitRegDcls(const PISA::RegDcls &, const PISA::DataTypes &) override {}
+ void emitLocalVariableDcls(const PISA::LocalVariableDcls &) override {}
+ void emitFuncBodyStart() override {}
+ void emitFuncBodyEnd() override {}
+
+ void addRegDcl(unsigned NumElts, unsigned EltSize, unsigned RegType,
+ RegNamesRange RegNames, bool MustAdd = true) override {}
+ void addLocalVariableDecl(const PISA::LocalVariableDcl &) override {}
+};
+
+} // namespace
+
+PISATargetStreamer::~PISATargetStreamer() = default;
+PISATargetAsmStreamer::~PISATargetAsmStreamer() = default;
+PISATargetELFStreamer::~PISATargetELFStreamer() = default;
+PISATargetNullStreamer::~PISATargetNullStreamer() = default;
+
+MCTargetStreamer *
+llvm::createPISAAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+ MCInstPrinter *InstPrinter) {
+ return new PISATargetAsmStreamer(S, OS);
+}
+
+MCTargetStreamer *
+llvm::createPISAObjectTargetStreamer(MCStreamer &S,
+ const MCSubtargetInfo &STI) {
+ const Triple &TT = STI.getTargetTriple();
+ if (TT.isOSBinFormatELF())
+ return new PISATargetELFStreamer(S);
+ return nullptr;
+}
+
+MCTargetStreamer *llvm::createPISANullTargetStreamer(MCStreamer &S) {
+ return new PISATargetNullStreamer(S);
+}
+
+DataType DataTypes::getTypeFromLLT(LLT Ty) {
+ unsigned NumElts = Ty.isScalar() ? 1 : Ty.getNumElements();
+ unsigned EltSize = Ty.getScalarSizeInBits();
+ unsigned RegType =
+ EltSize == 1 ? PISA::RegEncoder::PRED : PISA::RegEncoder::REG;
+ return DataType{NumElts, EltSize, RegType};
+}
+
+LLT DataTypes::getLLTFromType(const DataType &Ty) {
+ if (Ty.NumElts == 1) {
+ return LLT::integer(Ty.EltSize);
+ }
+ return LLT::vector(ElementCount::getFixed(Ty.NumElts), Ty.EltSize);
+}
+
+std::string DataTypes::getPrefixFromLLT(LLT Ty) {
+ unsigned NumElts = Ty.isScalar() ? 1 : Ty.getNumElements();
+ unsigned EltSize = Ty.getScalarSizeInBits();
+ std::string Prefix = "%";
+
+ const DenseMap<unsigned, StringRef> VectorPrefixes = {
+ {1, ""}, {2, "v2"}, {3, "v3"}, {4, "v4"}, {5, "v5"}, {6, "v6"},
+ {7, "v7"}, {8, "v8"}, {16, "v16"}, {32, "v32"}, {64, "v64"}};
+
+ if (auto It = VectorPrefixes.find(NumElts); It != VectorPrefixes.end())
+ Prefix += It->second;
+ else
+ llvm_unreachable("Unsupported PISA vector size");
+
+ const DenseMap<unsigned, StringRef> ScalarPrefixes = {
+ {1, "p"}, // Predicate
+ {8, "b"}, // Byte
+ {16, "h"}, // Half-word
+ {32, "w"}, // Word
+ {64, "d"}, // Double-word
+ {128, "q"} // Quad-word
+ };
+
+ if (auto It = ScalarPrefixes.find(EltSize); It != ScalarPrefixes.end())
+ Prefix += It->second;
+ else
+ llvm_unreachable("Unsupported PISA scalar size");
+ return Prefix;
+}
+
+DataType DataTypes::getTypeFromPrefix(std::string Prefix) {
+ auto Result =
+ llvm::StringSwitch<std::tuple<unsigned, unsigned, unsigned>>(Prefix)
+ .Case("%p", {1, 1, PISA::RegEncoder::PRED})
+ .Case("%b", {1, 8, PISA::RegEncoder::REG})
+ .Case("%h", {1, 16, PISA::RegEncoder::REG})
+ .Case("%w", {1, 32, PISA::RegEncoder::REG})
+ .Case("%d", {1, 64, PISA::RegEncoder::REG})
+ .Case("%q", {1, 128, PISA::RegEncoder::REG})
+ .Case("%v2b", {2, 8, PISA::RegEncoder::REG})
+ .Case("%v3b", {3, 8, PISA::RegEncoder::REG})
+ .Case("%v4b", {4, 8, PISA::RegEncoder::REG})
+ .Case("%v2h", {2, 16, PISA::RegEncoder::REG})
+ .Case("%v3h", {3, 16, PISA::RegEncoder::REG})
+ .Case("%v4h", {4, 16, PISA::RegEncoder::REG})
+ .Case("%v2w", {2, 32, PISA::RegEncoder::REG})
+ .Case("%v3w", {3, 32, PISA::RegEncoder::REG})
+ .Case("%v4w", {4, 32, PISA::RegEncoder::REG})
+ .Case("%v5w", {5, 32, PISA::RegEncoder::REG})
+ .Case("%v6w", {6, 32, PISA::RegEncoder::REG})
+ .Case("%v7w", {7, 32, PISA::RegEncoder::REG})
+ .Case("%v8w", {8, 32, PISA::RegEncoder::REG})
+ .Case("%v16w", {16, 32, PISA::RegEncoder::REG})
+ .Case("%v32w", {32, 32, PISA::RegEncoder::REG})
+ .Case("%v64w", {64, 32, PISA::RegEncoder::REG})
+ .Case("%v2d", {2, 64, PISA::RegEncoder::REG})
+ .Case("%v3d", {3, 64, PISA::RegEncoder::REG})
+ .Case("%v4d", {4, 64, PISA::RegEncoder::REG})
+ .Default({0, 0, 0});
+
+ if (std::get<0>(Result) == 0)
+ llvm_unreachable("Unknown type prefix");
+
+ auto [NumElts, EltSize, RegType] = Result;
+ return DataType{NumElts, EltSize, RegType};
+}
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.h b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.h
new file mode 100644
index 0000000000000..2610a83f36615
--- /dev/null
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.h
@@ -0,0 +1,286 @@
+//===-- PISATargetStreamer.h - PISA Target Streamer -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISATARGETSTREAMER_H
+#define LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISATARGETSTREAMER_H
+
+#include "PISADefines.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/CodeGenTypes/LowLevelType.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCStreamer.h"
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace llvm {
+
+namespace PISA {
+
+enum LinkageTy { DEFAULT, EXPORT, IMPORT };
+StringRef getLinkageTyName(LinkageTy Linkage);
+
+enum class StorageSpace : unsigned {
+ GENERIC,
+ GLOBAL,
+ CONSTANT,
+ SHARED,
+ PRIVATE,
+};
+StorageSpace mapAddrSpaceToStorageSpace(unsigned AS);
+
+enum class KernelAttributeType {
+ REQD_WORK_GROUP_SIZE,
+ VEC_TYPE_HINT,
+};
+
+struct KernelAttribute {
+ KernelAttributeType KernelAttrType;
+ std::variant<std::vector<uint32_t>, std::string, std::vector<std::string>,
+ uint32_t>
+ KernelAttrValues;
+};
+
+// PISA function and directive name
+struct FunctionDirectiveAndName {
+ // Calling convention.
+ CallingConv::ID CC;
+ // External linkage.
+ LinkageTy Linkage;
+ // Function name.
+ std::string Name;
+ // An optional return LLT for non-kernel functions.
+ LLT RetLLT;
+ // An optional list of kernel attributes.
+ SmallVector<KernelAttribute> KernelAttrs;
+};
+
+struct FunctionDeclParam {
+ LLT Ty;
+};
+
+struct FunctionDeclaration {
+ FunctionDirectiveAndName DN;
+ SmallVector<FunctionDeclParam> FunctionParams;
+};
+
+struct FunctionParameter {
+ LLT Ty;
+ std::string Prefix;
+ unsigned Idx;
+};
+
+struct KernelParameter {
+ unsigned Size = 0;
+ unsigned Align = 0;
+ unsigned PtrAlign = 0;
+ unsigned AS = ~0;
+ std::string ArgName;
+ std::string TypeName;
+ std::string TypeQualifier;
+
+ bool hasAlign() const { return Align != 0; }
+ bool hasPtrAlign() const { return PtrAlign != 0; }
+ bool hasAS() const { return AS != (unsigned)~0; }
+};
+// PISA header info
+struct HeaderDcl {
+ unsigned Version;
+ SmallString<16> Target;
+};
+// PISA function signature
+struct FunctionSignature {
+ FunctionDirectiveAndName DN;
+ // FIXME: Need to unify kernel and function parameters.
+ SmallVector<FunctionParameter> FunctionParams;
+ SmallVector<KernelParameter> KernelParams;
+ unsigned SourceLine = 0;
+};
+// PISA function register declarations
+struct RegDcls {
+ MapVector<std::tuple</*NumElts=*/unsigned, /*BitWidth=*/unsigned,
+ /*Type=*/unsigned>,
+ std::vector<std::pair</*Prefix=*/std::string, /*Id=*/unsigned>>>
+ Regs;
+};
+
+// PISA register declaration types
+struct DataType {
+ unsigned NumElts;
+ unsigned EltSize;
+ unsigned RegType;
+ DataType(unsigned N, unsigned B, unsigned R)
+ : NumElts(N), EltSize(B), RegType(R) {}
+};
+struct TypeInfo {
+ unsigned RegStart;
+ unsigned RegCounter;
+ LLT Ty;
+ std::string Prefix;
+};
+class DataTypes {
+private:
+ MapVector<std::tuple</*NumElts=*/unsigned, /*BitWidth=*/unsigned,
+ /*Type=*/unsigned>,
+ TypeInfo>
+ TypeInfos;
+
+ std::tuple<unsigned, unsigned, unsigned> tupleDT(DataType DT) {
+ return std::make_tuple(DT.NumElts, DT.EltSize, DT.RegType);
+ }
+
+public:
+ static DataType getTypeFromLLT(LLT Ty);
+ static LLT getLLTFromType(const DataType &Ty);
+ static std::string getPrefixFromLLT(LLT Ty);
+ static DataType getTypeFromPrefix(std::string Prefix);
+
+ auto begin() const { return TypeInfos.begin(); }
+ auto end() const { return TypeInfos.end(); }
+
+ TypeInfo &getInfo(LLT Ty) { return getInfo(getTypeFromLLT(Ty)); }
+ TypeInfo &getInfo(unsigned NumElts, unsigned EltSize, unsigned RegType) {
+ return getInfo(DataType(NumElts, EltSize, RegType));
+ }
+ TypeInfo &getInfo(DataType DT) {
+ auto *It = TypeInfos.find(tupleDT(DT));
+ if (It == TypeInfos.end())
+ llvm_unreachable("Expect that requested DataType is already present in "
+ "the TypeInfos map");
+ return It->second;
+ }
+
+ TypeInfo &emplaceInfo(unsigned NumElts, unsigned EltSize, unsigned RegType) {
+ return emplaceInfo(DataType(NumElts, EltSize, RegType));
+ }
+ TypeInfo &emplaceInfo(LLT Ty) { return emplaceInfo(getTypeFromLLT(Ty)); };
+ TypeInfo &emplaceInfo(DataType DT) {
+ LLT Ty = getLLTFromType(DT);
+ auto [It, Inserted] = TypeInfos.try_emplace(
+ tupleDT(DT), TypeInfo{0, 0, Ty, getPrefixFromLLT(Ty)});
+ return It->second;
+ }
+
+ void insertInfo(unsigned NumElts, unsigned EltSize, unsigned RegType,
+ TypeInfo TI) {
+ insertInfo(DataType(NumElts, EltSize, RegType), std::move(TI));
+ }
+ void insertInfo(DataType DT, TypeInfo TI) {
+ auto [It, Inserted] = TypeInfos.try_emplace(tupleDT(DT), std::move(TI));
+ if (!Inserted)
+ llvm_unreachable("Expect that inserted DataType is not already present "
+ "in the TypeInfos map");
+ }
+
+ // Once all function parameter declarations have been processed, call this
+ // to set all RegStart to the register after the last parameter of that Type
+ // Ex. if last parameter of Type Double-Word is %d3, then RegStart for the
+ // Double-Word TypeInfo should be 4. Note that RegCounter is not changed
+ // (in this example, it should still be 4 for d0-d3).
+ void finalizeFuncParams() {
+ for (auto &[_, TI] : TypeInfos)
+ TI.RegStart = TI.RegCounter;
+ }
+};
+
+struct VariableDcl {
+ // External linkage.
+ LinkageTy Linkage;
+ StorageSpace SS;
+ LLT Type;
+ Align Alignment;
+ // Variable name.
+ std::string Name;
+ uint64_t Size = 0;
+ int StackIndex = -1;
+ std::string Section;
+ std::string HostAccessName;
+};
+
+struct VariableInit {
+ struct InitElement {
+ LLT Type;
+ uint64_t Value;
+ };
+ SmallVector<InitElement> Initializer;
+ struct GlobalExpr {
+ // e.g., "@f+8"
+ std::string Name;
+ int64_t Offset = 0;
+ };
+ struct Zeros {
+ // e.g., .zeros 7
+ uint64_t N = 0;
+ };
+ using SpecialEntry = std::variant<GlobalExpr, Zeros>;
+ // This notes that the index in the initializer is actually the given
+ // global or ".zero" directive rather than the immediate value there.
+ DenseMap<uint64_t, SpecialEntry> Exprs;
+};
+
+// Variable declaration and initialization
+struct VariableDclInit {
+ VariableDcl Dcl;
+ VariableInit Init;
+};
+
+// Local variable declaration in private and shared space
+using LocalVariableDcl = VariableDcl;
+struct LocalVariableDcls {
+ SmallVector<LocalVariableDcl> Vars;
+};
+
+// Global variable declaration in global and const space
+using GlobalVariableDcl = VariableDclInit;
+struct GlobalVariableDcls {
+ SmallVector<GlobalVariableDcl> Vars;
+};
+
+} // namespace PISA
+
+class PISATargetStreamer : public MCTargetStreamer {
+public:
+ PISATargetStreamer(MCStreamer &S) : MCTargetStreamer(S) {}
+ ~PISATargetStreamer() override;
+
+ /// Emit header directives (.version, .target). Returns the resolved target
+ /// CPU string, which may differ from the initial CPU if a .target directive
+ /// was present in the input.
+ virtual StringRef emitHeader(const PISA::HeaderDcl &) = 0;
+ virtual void emitGlobalVariable(const PISA::GlobalVariableDcl &) = 0;
+ virtual void emitFunctionSignature(const PISA::FunctionSignature &) = 0;
+ virtual void emitFunctionDeclaration(const PISA::FunctionDeclaration &) = 0;
+ virtual void emitRegDcls(const PISA::RegDcls &, const PISA::DataTypes &) = 0;
+ virtual void emitLocalVariableDcls(const PISA::LocalVariableDcls &) = 0;
+ virtual void emitFuncBodyStart() = 0;
+ virtual void emitFuncBodyEnd() = 0;
+
+ // Register names stored as <Prefix, Id>
+ using RegName = std::tuple<std::string, unsigned>;
+
+ // Array of register names for immutable APIs.
+ using RegNamesRange = ArrayRef<RegName>;
+
+ // If MustAdd is true, addRegDcl would assert if This reg is already added.
+ virtual void addRegDcl(unsigned NumElts, unsigned EltSize, unsigned RegType,
+ RegNamesRange RegNames, bool MustAdd = true) = 0;
+ virtual void addLocalVariableDecl(const PISA::LocalVariableDcl &) = 0;
+
+ void changeSection(const MCSection *CurSection, MCSection *Section,
+ uint32_t SubSection, raw_ostream &OS) override {}
+
+ // Only Asm stream subclass returns true
+ virtual bool isAsmStreamer() const { return false; }
+ virtual int getCurLine() const { return -1; }
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_MCTARGETDESC_PISATARGETSTREAMER_H
diff --git a/llvm/lib/Target/PISA/PISA.td b/llvm/lib/Target/PISA/PISA.td
index 71277b826162b..0a88485ca6420 100644
--- a/llvm/lib/Target/PISA/PISA.td
+++ b/llvm/lib/Target/PISA/PISA.td
@@ -6,29 +6,37 @@
//
//===----------------------------------------------------------------------===//
+include "llvm/TableGen/SearchableTable.td"
include "llvm/Target/Target.td"
+// Global setting
+def Global {
+ string NS = "PISA";
+}
+
+#define PISA
+
include "PISAFeatures.td"
include "PISARegisterInfo.td"
+include "PISARegisterBanks.td"
include "PISAInstrInfo.td"
+include "PISACombine.td"
-// Map the pointer-like operands of the target-independent pseudo instructions
-// (STACKMAP, PATCHPOINT, ...) onto a concrete PISA register class.
defm : RemapAllTargetPseudoPointerOperands<Reg64b>;
-
def PISAInstrInfo : InstrInfo;
class Proc<string Name, list<SubtargetFeature> Features>
- : Processor<Name, NoItineraries, Features>;
+ : Processor<Name, NoItineraries, Features>;
-// Valid -mcpu values.
-def : Proc<"100", [Feature100]>;
+// valid -mcpu values
+def : Proc<"100", [Feature100]>;
def PISAInstPrinter : AsmWriter {
- string AsmWriterClassName = "InstPrinter";
+ string AsmWriterClassName = "InstPrinter";
bit isMCAsmWriter = 1;
}
+
def PISA : Target {
let InstructionSet = PISAInstrInfo;
let AssemblyWriters = [PISAInstPrinter];
diff --git a/llvm/lib/Target/PISA/PISACacheCtrlMMRA.h b/llvm/lib/Target/PISA/PISACacheCtrlMMRA.h
new file mode 100644
index 0000000000000..e46b25419d83d
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISACacheCtrlMMRA.h
@@ -0,0 +1,92 @@
+//===-- PISACacheCtrlMMRA.h -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISACACHECTRLMMRA_H
+#define LLVM_LIB_TARGET_PISA_PISACACHECTRLMMRA_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <optional>
+
+namespace llvm {
+namespace PISA {
+
+inline constexpr StringRef CacheCtrlMMRAPrefix = "pisa.cache.ctrl";
+
+// Reads the cache-ctrl integer from the "pisa.cache.ctrl" MMRA tag on I.
+// Returns std::nullopt if the tag is absent or malformed. If more than
+// one distinct cache-ctrl value is attached (e.g. a passthrough merge of
+// two memory ops with different cache hints by an upstream pass such as
+// the LoadStoreVectorizer), emits a warning through the LLVMContext
+// listing the conflicting values and keeps one of them so the compiler
+// can still produce valid output.
+inline std::optional<unsigned> getCacheCtrlFromMMRA(const Instruction &I) {
+ MMRAMetadata MMRA(I);
+ SmallVector<unsigned, 2> Values;
+ for (const auto &[Prefix, Suffix] : MMRA) {
+ if (Prefix != CacheCtrlMMRAPrefix)
+ continue;
+ unsigned Value;
+ if (Suffix.getAsInteger(10, Value))
+ return std::nullopt;
+ if (!is_contained(Values, Value))
+ Values.push_back(Value);
+ }
+ if (Values.empty())
+ return std::nullopt;
+ if (Values.size() > 1) {
+ SmallString<128> Msg;
+ raw_svector_ostream OS(Msg);
+ OS << "instruction has conflicting pisa.cache.ctrl MMRA tags: {";
+ interleaveComma(Values, OS);
+ OS << "}; keeping " << Values.front();
+ I.getContext().diagnose(
+ DiagnosticInfoGeneric(&I, Twine(StringRef(Msg)), DS_Warning));
+ }
+ return Values.front();
+}
+
+// Sets the "pisa.cache.ctrl" MMRA tag on I to Value. Preserves all
+// other MMRA tags already on I, replacing any prior cache-ctrl tag.
+inline void setCacheCtrlMMRA(Instruction &I, unsigned Value) {
+ LLVMContext &Ctx = I.getContext();
+
+ SmallVector<MMRAMetadata::TagT, 4> Tags;
+ MMRAMetadata Existing(I);
+ for (const auto &Tag : Existing) {
+ if (Tag.first != CacheCtrlMMRAPrefix)
+ Tags.push_back(Tag);
+ }
+
+ SmallString<8> Buf;
+ Tags.emplace_back(CacheCtrlMMRAPrefix, Twine(Value).toStringRef(Buf));
+
+ I.setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, Tags));
+}
+
+// Copies the "pisa.cache.ctrl" MMRA tag from From to To if present,
+// preserving all other MMRA tags already on To.
+inline void copyCacheCtrlMMRA(const Instruction &From, Instruction &To) {
+ if (auto Value = getCacheCtrlFromMMRA(From))
+ setCacheCtrlMMRA(To, *Value);
+}
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISACACHECTRLMMRA_H
diff --git a/llvm/lib/Target/PISA/PISACombine.td b/llvm/lib/Target/PISA/PISACombine.td
new file mode 100644
index 0000000000000..1cd8fa459fdb6
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISACombine.td
@@ -0,0 +1,1075 @@
+//=- PISACombine.td - Define PISA Combine Rules --------------*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/GlobalISel/Combine.td"
+
+def extra_commute_iconstant_to_rhs : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_UADDO, G_SADDO,
+ G_UMULO, G_SMULO, G_UMULH, G_SMULH, G_UADDSAT,
+ G_SADDSAT, G_SMULFIX, G_UMULFIX, G_SMULFIXSAT,
+ G_UMULFIXSAT):$root, [{
+ return getIConstantVRegVal(${root}->getOperand(1).getReg(), MRI).has_value() &&
+ !getIConstantVRegVal(${root}->getOperand(2).getReg(), MRI).has_value();
+ }]),
+ (apply [{
+ Observer.changingInstr(*${root});
+ Register LHSReg = ${root}->getOperand(1).getReg();
+ Register RHSReg = ${root}->getOperand(2).getReg();
+ ${root}->getOperand(1).setReg(RHSReg);
+ ${root}->getOperand(2).setReg(LHSReg);
+ Observer.changedInstr(*${root});
+ }])
+>;
+
+def extra_commute_fconstant_to_rhs : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINNUM_IEEE,
+ G_FMAXNUM_IEEE, G_FMINIMUM, G_FMAXIMUM,
+ G_FADD, G_FMUL):$root, [{
+ return getFConstantVRegValWithLookThrough(${root}->getOperand(1).getReg(), MRI).has_value() &&
+ !getFConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI).has_value();
+ }]),
+ (apply [{
+ Observer.changingInstr(*${root});
+ Register LHSReg = ${root}->getOperand(1).getReg();
+ Register RHSReg = ${root}->getOperand(2).getReg();
+ ${root}->getOperand(1).setReg(RHSReg);
+ ${root}->getOperand(2).setReg(LHSReg);
+ Observer.changedInstr(*${root});
+ }])
+>;
+
+def extra_commute_intrinsic_iconstant_to_rhs : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_INTRINSIC):$root, [{
+ auto IntrinsicID = cast<GIntrinsic>(*${root}).getIntrinsicID();
+ auto isFloatArg = false;
+ switch (IntrinsicID) {
+ default:
+ return false;
+ case Intrinsic::pisa_fadd:
+ case Intrinsic::pisa_fmul:
+ isFloatArg = true;
+ break;
+ }
+ return isFloatArg ?
+ (getFConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI).has_value() &&
+ !getFConstantVRegValWithLookThrough(${root}->getOperand(3).getReg(), MRI).has_value()) :
+ (getIConstantVRegVal(${root}->getOperand(2).getReg(), MRI).has_value() &&
+ !getIConstantVRegVal(${root}->getOperand(3).getReg(), MRI).has_value());
+ }]),
+ (apply [{
+ Observer.changingInstr(*${root});
+ Register LHSReg = ${root}->getOperand(2).getReg();
+ Register RHSReg = ${root}->getOperand(3).getReg();
+ ${root}->getOperand(2).setReg(RHSReg);
+ ${root}->getOperand(3).setReg(LHSReg);
+ Observer.changedInstr(*${root});
+ }])
+>;
+
+// AND rd1, rs1, C1
+// AND rd2, rd1, C1 - remove copy
+def extra_remove_redundant_constant : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_AND, G_OR):$root,
+ [{
+ auto C1 = getIConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI);
+ if (C1.has_value()) {
+ auto *DefMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ if (DefMI && (${root}->getOpcode() == DefMI->getOpcode())) {
+ auto C2 = getIConstantVRegValWithLookThrough(DefMI->getOperand(2).getReg(), MRI);
+ if (C2.has_value()) {
+ ${matchinfo} = DefMI->getOperand(0).getReg();
+ return C1->Value == C2->Value;
+ }
+ }
+ }
+ return false;
+ }]),
+ (apply
+ [{
+ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo});
+ }])
+>;
+
+// G_[F]CONSTANT i16 x
+// G_[F]CONSTANT i16 y
+// G_[F]CONSTANT i16 z
+// G_[F]CONSTANT i16 w
+// G_BUILD_VECTOR x, y, z, w
+// => G_CONSTANT i64 [wzyx]
+// => G_BITCAST
+def build_vector_with_constants : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{
+ auto Type = MRI.getType(${root}->getOperand(0).getReg());
+ if (Type.isPointerVector())
+ return false;
+ auto Size = Type.getSizeInBits();
+ if ((Size <= 64) && isPowerOf2_32(Size)) {
+ for (unsigned I=1; I < ${root}->getNumOperands(); I++) {
+ auto C = getAnyConstantVRegValWithLookThrough(${root}->getOperand(I).getReg(), MRI);
+ if (!C.has_value()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }]),
+ (apply
+ [{
+ applyBuildVectorWithConstants(*${root});
+ }])
+>;
+
+// i56 = G_LOAD x
+// i32 = G_TRUNC i56
+// => i32 = G_LOAD x
+def truncated_load : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_TRUNC):$root,
+ [{
+ auto *DefMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ auto Type = MRI.getType(DefMI->getOperand(0).getReg());
+ auto Size = Type.getSizeInBits();
+ auto IsVector = Type.isVector();
+ return !isPowerOf2_32(Size) && !IsVector && (DefMI->getOpcode() == TargetOpcode::G_LOAD);
+ }]),
+ (apply
+ [{
+ applyTruncatedLoad(*${root});
+ }])
+>;
+
+// i24 x = G_TRUNC i32
+// G_STORE i24 x
+// => <4 x i8> y = BITCAST x
+// <3 x i8> z = SHUFFLE_VECTOR y, <0,1,2>
+// G_STORE <3 x i8> z
+def truncated_store : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_STORE):$root,
+ [{
+ return matchTruncatedStore(*${root});
+ }]),
+ (apply
+ [{
+ applyTruncatedStore(*${root});
+ }])
+>;
+
+// i24 = G_LOAD x
+// i32 = G_ZEXT i24
+// =>
+// i32 loadRes = G_LOAD x
+// i32 = AND loadRes, 0x00FFFFFF
+def extended_load : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_ZEXT):$root,
+ [{
+ return matchExtendedLoad(*${root});
+ }]),
+ (apply
+ [{
+ applyExtendedLoad(*${root});
+ }])
+>;
+
+// i24 = G_LOAD x
+// =>
+// %3:_(s24) = G_ZEXTLOAD %0:reg32b(p0)
+// %7:_(s32) = G_CONSTANT i32 2
+// %6:reg32b(p0) = G_PTR_ADD %0:reg32b, %7:_(s32)
+// %9:_(s24) = G_ZEXTLOAD %6:reg32b(p0)
+// %11:_(s24) = G_CONSTANT i24 16
+// %10:_(s24) = G_SHL %9:_, %11:_(s24)
+// %12:_(s24) = G_OR %3:_, %10:_
+// Similar transformation is done for G_STORE.
+def expand_nonpowerof2_load_store : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_LOAD, G_STORE):$root,
+ [{
+ return matchExpandNonPowerOf2LoadStore(*${root});
+ }]),
+ (apply
+ [{
+ applyExpandNonPowerOf2LoadStore(*${root});
+ }])
+>;
+
+// If the value we store resulted from a G_LOAD that the rule above expanded,
+// then we can use the individual load values directly instead of merging into
+// one integer, and then splitting it again. Size modifications between the
+// loads and store (i.e. SEXT/ZEXT/TRUNC) are also supported by
+// truncating/extending the relevant load results (or, in the case of
+// truncation, ignoring some loads entirely)
+//
+// Below is a simple example where we simply load and store an i56:
+//
+// bb.1.entry:
+// %0:reg32b(p0) = functionParameter_32b 0
+// -> %3:_(s56) = G_ZEXTLOAD %0:reg32b(p0) ::
+// (load (s32) from %ir.dst)
+// %4:_(s56) = G_CONSTANT i56 0
+// %7:_(s32) = G_CONSTANT i32 4
+// %6:reg32b(p0) = G_PTR_ADD %0:reg32b, %7:_(s32)
+// -> %9:_(s56) = G_ZEXTLOAD %6:reg32b(p0) ::
+// (load (s16) from %ir.dst + 4, align 4)
+// %11:_(s56) = G_CONSTANT i56 32
+// %10:_(s56) = G_SHL %9:_, %11:_(s56)
+// %12:_(s56) = G_OR %3:_, %10:_
+// %14:_(s32) = G_CONSTANT i32 6
+// %13:reg32b(p0) = G_PTR_ADD %0:reg32b, %14:_(s32)
+// -> %16:_(s56) = G_ZEXTLOAD %13:reg32b(p0) ::
+// (load (s8) from %ir.dst + 6, align 2, basealign 4)
+// %18:_(s56) = G_CONSTANT i56 48
+// %17:_(s56) = G_SHL %16:_, %18:_(s56)
+// %19:_(s56) = G_OR %12:_, %17:_
+// G_STORE %19:_(s56), %0:reg32b(p0) ::
+// (store (s56) into %ir.dst, align 4)
+// ret
+def loadVector_matchinfo : GIDefMatchData<"SmallVector<std::pair<MachineInstr *, unsigned>, 8>">;
+def MI_matchinfo : GIDefMatchData<"MachineInstr *">;
+def simplify_nonpowerof2_load_store_chain : GICombineRule<
+ (defs root:$root, loadVector_matchinfo:$matchinfo, MI_matchinfo:$MI_matchinfo),
+ (match (wip_match_opcode G_STORE):$root,
+ [{
+ return matchSimplifyNonPowerOf2LoadStoreChain(*${root}, ${matchinfo}, ${MI_matchinfo});
+ }]),
+ (apply
+ [{
+ applySimplifyNonPowerOf2LoadStoreChain(*${root}, ${matchinfo}, ${MI_matchinfo});
+ }])
+>;
+
+// i1 C = G_CMP ne i? A, 0
+// i? S = G_SELECT i1 C, i? LHS, i? RHS
+// => S = sel.? LHS, RHS, A
+// ... or ...
+// i1 C = G_CMP eq i? A, 0
+// i? S = G_SELECT i1 C, i? LHS, i? RHS
+// => S = sel.? RHS, LHS, A
+def compare_select_reg : GICombineRule<
+ (defs root:$SelectMI),
+ (match (wip_match_opcode G_SELECT):$SelectMI,
+ [{
+ return matchCompareSelect(${SelectMI});
+ }]),
+ (apply
+ [{
+ applyCompareSelect(${SelectMI});
+ }])
+>;
+
+// arcp G_FDIV %0, %1
+// =>
+// %rcp = arcp G_INTRINSIC intrinsic(@llvm.pisa.frcp), %1
+// %result = arcp G_FMUL %0, %rcp
+def fdiv_to_rcp_fmul : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FDIV):$root,
+ [{ return matchFDivToRcpFMul(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
+>;
+
+//%2:registers(s32) = G_SHL %0:reg32b, 2
+//%4:registers(s32) = G_ADD %2:registers, %3:registers
+// => %4:registers(s32) = G_PISA_SMAD %0:reg32b, 4, %3:registers
+def shl_add_to_mad : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_ADD):$root,
+ [{ return matchShlAddToMad(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+
+// i32 %lo = G_TRUNC i32 a 16
+// i32 %shift = G_LSHR i32 a 16
+// %hi = G_TRUNC i32 %shift to i16
+// => %1 = G_BITCAST i32 a to <2 x 16>
+// => %hi = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 1
+// => %lo = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 0
+def trunc_shift_to_vector_extract : GICombineRule<
+ (defs root:$TruncMI),
+ (match (wip_match_opcode G_TRUNC):$TruncMI,
+ [{ return matchTruncatedShift(*${TruncMI}); }]),
+ (apply
+ [{ applyTruncatedShift(*${TruncMI}); }])
+>;
+
+// %5 = G_CONSTANT i16 x
+// %6 = G_CONSTANT i16 y
+// %7 = G_AND %5:_, %6:_
+// %2 = G_TRUNC %7 to i8
+// => %2 = COPY (i8)(x & y)
+def const_and_trunc_to_copy : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_TRUNC):$root,
+ [{
+ auto *AndMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ if (AndMI->getOpcode() == TargetOpcode::G_AND) {
+ auto C1 = getIConstantVRegValWithLookThrough(AndMI->getOperand(1).getReg(), MRI);
+ auto C2 = getIConstantVRegValWithLookThrough(AndMI->getOperand(2).getReg(), MRI);
+ if (C1.has_value() && C2.has_value()) {
+ auto Value = C1->Value & C2->Value;
+ return (Value.getZExtValue() >> MRI.getType(${root}->getOperand(0).getReg()).getSizeInBits()) == 0;
+ }
+ }
+ return false;
+ }]),
+ (apply
+ [{
+ auto *AndMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ auto C1 = getIConstantVRegValWithLookThrough(AndMI->getOperand(1).getReg(), MRI);
+ auto C2 = getIConstantVRegValWithLookThrough(AndMI->getOperand(2).getReg(), MRI);
+ auto Value = (C1->Value & C2->Value).trunc(MRI.getType(${root}->getOperand(0).getReg()).getSizeInBits());
+ Helper.replaceInstWithConstant(*${root}, Value);
+ }])
+>;
+
+// G_AND (G_ZEXT x), C -> G_ZEXT (G_AND x, trunc(C)).
+// CodeGenPrepare can form the widened mask, but keeping the mask narrow lets
+// address folding see the original arithmetic shape.
+def zext_and_to_and_zext : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_AND):$root,
+ [{ return matchZExtAndToAndZExt(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %114:_(<5 x s32>) = G_EXTRACT_SUBVECTOR %113:_(<8 x s32>), 0
+// %65:_(s32) = G_EXTRACT_VECTOR_ELT %114:_(<5 x s32>), %66:_(s32)
+// => %65:_(s32) = G_EXTRACT_VECTOR_ELT %113:_(<8 x s32>), %66:_(s32)
+
+def extract_subvector_to_extract_elt : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
+ [{
+ auto *SubvectorMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ if (!SubvectorMI) {
+ return false;
+ }
+ if (SubvectorMI->getOpcode() != TargetOpcode::G_EXTRACT_SUBVECTOR) {
+ return false;
+ }
+ if (SubvectorMI->getOperand(2).isReg()){
+ // Skip to avoid needing to generate add instruction for index adjustment.
+ return false;
+ }
+ auto C = getIConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI);
+ if (!C.has_value()){
+ if (SubvectorMI->getOperand(2).getImm() != 0){
+ // Skip to avoid needing to generate add instruction for index adjustment.
+ return false;
+ }
+ }
+ LLVM_DEBUG(dbgs() << " Success: extract_subvector_to_extract_elt rule matches!\n");
+ return true;
+ }]),
+ (apply
+ [{
+ auto *SubvectorMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ Register SrcReg = SubvectorMI->getOperand(1).getReg();
+ Register IdxReg = ${root}->getOperand(2).getReg();
+ Register DstReg = ${root}->getOperand(0).getReg();
+ auto C = getIConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI);
+ if (C.has_value() && SubvectorMI->getOperand(2).isImm()){
+ uint64_t NewIdx = C->Value.getZExtValue() + SubvectorMI->getOperand(2).getImm();
+ B.buildExtractVectorElementConstant(DstReg, SrcReg, NewIdx);
+ }
+ else {
+ B.buildExtractVectorElement(DstReg, SrcReg, IdxReg);
+ }
+ ${root}->eraseFromParent();
+ }])
+>;
+
+// %28:_(<8 x i32>) = G_BUILD_VECTOR %91:_(i32), %92:_(i32), %93:_(i32), %94:_(i32), %95:_(i32), %96:_(i32), %97:_(i32), %98:_(i32)
+// %37:_(<4 x i32>) = G_EXTRACT_SUBVECTOR %28:_(<8 x i32>), 0
+// => %37:_(<4 x i32>) = G_BUILD_VECTOR %91:_(i32), %92:_(i32), %93:_(i32), %94:_(i32)
+def extract_subvector_build_vector_matchdata : GIDefMatchData<"SmallVector<Register, 8>">;
+def extract_subvector_build_vector : GICombineRule<
+ (defs root:$root, extract_subvector_build_vector_matchdata:$matchinfo),
+ (match (wip_match_opcode G_EXTRACT_SUBVECTOR):$root,
+ [{ return matchExtractSubvectorBuildVector(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyExtractSubvectorBuildVector(*${root}, ${matchinfo}); }])
+>;
+
+// %29:_(<64 x s32>) = IMPLICIT_DEF
+// %30:_(<64 x s32>) = G_INSERT_SUBVECTOR %29:_, %7:_(<8 x s32>), 0
+// %31:_(<64 x s32>) = G_INSERT_SUBVECTOR %30:_, %10:_(<8 x s32>), 8
+// %32:_(<64 x s32>) = G_INSERT_SUBVECTOR %31:_, %13:_(<8 x s32>), 16
+// %33:_(<64 x s32>) = G_INSERT_SUBVECTOR %32:_, %16:_(<8 x s32>), 24
+// %34:_(<64 x s32>) = G_INSERT_SUBVECTOR %33:_, %19:_(<8 x s32>), 32
+// %35:_(<64 x s32>) = G_INSERT_SUBVECTOR %34:_, %22:_(<8 x s32>), 40
+// %36:_(<64 x s32>) = G_INSERT_SUBVECTOR %35:_, %25:_(<8 x s32>), 48
+// %37:_(<64 x s32>) = G_INSERT_SUBVECTOR %36:_, %28:_(<8 x s32>), 56
+// %4:_(<32 x s32>) = G_EXTRACT_SUBVECTOR %37:_(<64 x s32>), 32
+// =>
+// %29:_(<32 x s32>) = IMPLICIT_DEF
+// %34:_(<32 x s32>) = G_INSERT_SUBVECTOR %33:_, %19:_(<8 x s32>), 0
+// %35:_(<32 x s32>) = G_INSERT_SUBVECTOR %34:_, %22:_(<8 x s32>), 8
+// %36:_(<32 x s32>) = G_INSERT_SUBVECTOR %35:_, %25:_(<8 x s32>), 16
+// %37:_(<32 x s32>) = G_INSERT_SUBVECTOR %36:_, %28:_(<8 x s32>), 24
+// %4:_(<32 x s32>) = COPY %37:_(<32 x s32>)
+def extract_subvector_partial_matchdata : GIDefMatchData<"SmallVector<MachineInstr *, 8>">;
+def extract_subvector_partial : GICombineRule<
+ (defs root:$root, extract_subvector_partial_matchdata:$matchinfo),
+ (match (wip_match_opcode G_EXTRACT_SUBVECTOR):$root,
+ [{ return matchExtractSubvectorPartial(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyExtractSubvectorPartial(*${root}, ${matchinfo}); }])
+>;
+
+// A G_BUILD_VECTOR whose elements are consecutive lanes of a single wider
+// source vector (extracted via G_UNMERGE_VALUES at lanes base, base+1) is a
+// sub-vector slice of that source. Rewrite it to G_EXTRACT_SUBVECTOR so ISel
+// emits a direct composite sub-register COPY (.xy / .zw) that register
+// coalescing folds, instead of an element-wise gather. This is what lets the
+// .zw high half become copy-free (the .xy low half already coalesced).
+// %70,%71,%72,%73 = G_UNMERGE_VALUES %0(<4 x s16>)
+// %23:_(<2 x s16>) = G_BUILD_VECTOR %72, %73 ; lanes 2,3 of %0
+// => %23:_(<2 x s16>) = G_EXTRACT_SUBVECTOR %0, 2
+def build_vector_from_unmerge_lanes_matchdata : GIDefMatchData<"std::tuple<Register, int64_t>">;
+def build_vector_from_unmerge_lanes : GICombineRule<
+ (defs root:$root, build_vector_from_unmerge_lanes_matchdata:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchBuildVectorFromUnmergeLanes(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyBuildVectorFromUnmergeLanes(*${root}, ${matchinfo}); }])
+>;
+
+// A G_BUILD_VECTOR that concatenates whole sub-vectors (each source's lanes
+// appear in order) is a G_CONCAT_VECTORS. Rewrite it so ISel writes each
+// source into its composite sub-register slice (.xy / .zw) directly, instead
+// of an element-wise rebuild.
+// %56,%57 = G_UNMERGE_VALUES %14(<2 x s16>) ; %36,%37 = G_UNMERGE_VALUES %29
+// %2:_(<4 x s16>) = G_BUILD_VECTOR %56, %57, %36, %37
+// => %2:_(<4 x s16>) = G_CONCAT_VECTORS %14, %29
+def build_vector_concat_subvectors_matchdata : GIDefMatchData<"SmallVector<Register, 4>">;
+def build_vector_concat_subvectors : GICombineRule<
+ (defs root:$root, build_vector_concat_subvectors_matchdata:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchBuildVectorConcatSubvectors(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyBuildVectorConcatSubvectors(*${root}, ${matchinfo}); }])
+>;
+
+// %6 = G_CONSTANT i32 3
+// %2 = G_BUILD_VECTOR %23, %25, %27, %29, %30
+// %3 = G_EXTRACT_VECTOR_ELT %2, %6
+// => %3 = COPY %29
+def build_vector_extract_to_copy : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
+ [{
+ auto *BuildMI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ if (BuildMI->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
+ auto C = getIConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI);
+ return C.has_value();
+ }
+ return false;
+ }]),
+ (apply
+ [{
+ auto [BuildMI, RegIdx] = PISA::getDefIgnoringBitcasts(${root}->getOperand(1).getReg(), MRI);
+ auto C = getIConstantVRegValWithLookThrough(${root}->getOperand(2).getReg(), MRI);
+ auto DstReg = ${root}->getOperand(0).getReg();
+ auto OpndReg = BuildMI->getOperand(C->Value.getZExtValue() + 1).getReg();
+ if (DstReg != OpndReg)
+ B.buildCopy(DstReg, OpndReg);
+ ${root}->eraseFromParent();
+ }])
+>;
+
+// s16 %1 = G_CONSTANT i16 1
+// s32 %2 = G_CONSTANT i32 2
+// s16 = G_SHL %1, %2
+// => s16 = G_CONSTANT i16 4
+// note: occurs when dealing with vectors of i1
+def shifts_of_constants : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_SHL, G_LSHR, G_ASHR):$root,
+ [{
+ auto *Src0MI = getDefIgnoringCopies(${root}->getOperand(1).getReg(), MRI);
+ auto *Src1MI = getDefIgnoringCopies(${root}->getOperand(2).getReg(), MRI);
+ return (Src0MI->getOpcode() == TargetOpcode::G_CONSTANT)
+ && (Src1MI->getOpcode() == TargetOpcode::G_CONSTANT);
+ }]),
+ (apply
+ [{
+ applyShiftOfConstants(${root});
+ }])
+>;
+
+// %5:_(s32) = G_CONSTANT i32 3
+// %4:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.lane.id)
+// %6:_(s32) = nuw nsw G_SHL %4:_, %5:_(s32)
+// %7:_(s64) = nneg G_ZEXT %6:_(s32)
+// %31:_(s64) = G_CONSTANT i64 2
+// %9:_(s64) = nuw nsw G_SHL %7:_, %31:_(s64)
+// => %4:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.lane.id)
+// %5:_(s32) = G_CONSTANT i32 5
+// %6:_(s32) = nuw nsw G_SHL %4:_, %5:_(s32)
+// %7:_(s64) = nneg G_ZEXT %6:_(s32)
+def lane_id_shl_chain : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_SHL):$root,
+ [{ return matchLaneIdLeftShiftChain(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// A(16) = G_EXTRACT_VECTOR_ELT ARG(32), 0
+// B(16) = G_EXTRACT_VECTOR_ELT ARG(32), 1
+// C(<2x16>) = G_BUILD_VECTOR A, B
+// D(32) = G_BITCAST C(<2x16>)
+// => D(32) = COPY ARG(32)
+def remove_redundant_moves_pre : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_BITCAST):$root,
+ [{ return matchRedundantMovesPre(*${root}); }]),
+ (apply
+ [{ applyRedundantMovesPre(*${root}); }])
+>;
+// A(<2x16>) = G_BITCAST ARG(32)
+// B(16), C(16) = G_UNMERGE_VALUES A(<2x16)
+// D(<2x16>) = G_BUILD_VECTOR B, C
+// E(32) = G_BITCAST D(<2x16>)
+// => E(32) = COPY ARG(32)
+def remove_redundant_moves_post : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_BITCAST):$root,
+ [{ return matchRedundantMovesPost(*${root}); }]),
+ (apply
+ [{ applyRedundantMovesPost(*${root}); }])
+>;
+
+// %4:_(<32 x s8>) = G_BITCAST %3:_(<4 x s64>)
+// %5:_(<8 x s32>) = G_BITCAST %4:_(<32 x s8>)
+// => %5:_(<8 x s32>) = G_BITCAST %3:_(<4 x s64>)
+def redundant_bitcasts : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_BITCAST):$root,
+ [{
+ auto DstReg = ${root}->getOperand(0).getReg();
+ auto SrcReg = ${root}->getOperand(1).getReg();
+ auto *SrcMI = getDefIgnoringCopies(SrcReg, MRI);
+ // do not remove casts between integers and floating point
+ if (MRI.getType(DstReg).getScalarType().isFloat() != MRI.getType(SrcReg).getScalarType().isFloat())
+ return false;
+ // favor <? x s32> as we can use extract/insert ops
+ return (SrcMI->getOpcode() == TargetOpcode::G_BITCAST)
+ && (MRI.getType(DstReg).getScalarSizeInBits() == 32);
+ }]),
+ (apply
+ [{
+ auto DstReg = ${root}->getOperand(0).getReg();
+ auto SrcReg = ${root}->getOperand(1).getReg();
+ auto *SrcMI = getDefIgnoringCopies(SrcReg, MRI);
+ auto OrigSrcReg = SrcMI->getOperand(1).getReg();
+ if (MRI.getType(DstReg) == MRI.getType(OrigSrcReg))
+ B.buildCopy(DstReg, OrigSrcReg);
+ else
+ B.buildBitcast(DstReg, OrigSrcReg);
+ ${root}->eraseFromParent();
+ }])
+>;
+
+// A(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16), 0
+// B(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16), 1
+// ...
+// X(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16>), N-1
+// Y(<N x s16>) = G_BUILD_VECTOR A, B, ..., X
+// => Y(<N x s16>) = COPY ARG(<N x s16>)
+def extract_all_build_vector : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchExtractAllToBuildVector(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyExtractAllToBuildVector(*${root}, ${matchinfo}); }])
+>;
+
+// Fold int2ptr(ptr2int(x)) -> x
+// - LLVM version does not consider difference
+// in size between PTR and integer
+def p2i_to_i2p_fixed: GICombineRule<
+ (defs root:$dst, register_matchinfo:$matchinfo),
+ (match (G_PTRTOINT $int, $ptr),
+ (G_INTTOPTR $dst, $int):$root,
+ [{
+ auto PTy = MRI.getType(${ptr}.getReg());
+ auto TTy = MRI.getType(${int}.getReg());
+ auto DTy = MRI.getType(${dst}.getReg());
+ if (PTy.getScalarSizeInBits() != TTy.getScalarSizeInBits())
+ return false;
+ if (TTy.getScalarSizeInBits() != DTy.getScalarSizeInBits())
+ return false;
+ // A COPY between pointers of different address spaces is illegal
+ if (PTy != DTy)
+ return false;
+ ${matchinfo} = ${ptr}.getReg();
+ return true;
+ }]),
+ (apply
+ [{
+ B.buildCopy(${dst}, ${ptr});
+ ${root}->eraseFromParent();
+ }])
+>;
+
+def or_and_to_bfi : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_OR):$root,
+ [{ return matchOrAndToBfi(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// s32 %0 = G_CONSTANT i32 32
+// s32 %1 = G_SUB i32 %0, %width
+// s32 %2 = G_SUB i32 %1, %offset
+// s32 %3 = G_SHL i32 %data, %2
+// s32 %4 = G_LSHR i32 %3, %1
+// => bfe %data %width %offset
+def shift_sub_to_bfe : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_LSHR, G_ASHR):$root,
+ [{ return matchShiftSubToBfe(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// s32 %maskedwidth = G_AND %width, 63
+// s32 %maskedoffset = G_AND %offset, 31
+// bfe %data %maskedwidth %maskedoffset
+// =>
+// bfe %data %width %offset
+def and_bitfield_to_bitfield : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_INTRINSIC, G_UBFX, G_SBFX):$root,
+ [{ return matchAndBitfieldToBitfield(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// This transforms an unoptimal pattern for fcmp + zext i1->i8
+// %14:_(<2 x s16>) = G_BITCAST %CmpRes:_(s32)
+// %20:_(s16), %21:_(s16) = G_UNMERGE_VALUES %14:_(<2 x s16>)
+// %22:_(s8) = G_TRUNC %20:_(s16)
+// %23:_(s8) = G_TRUNC %21:_(s16)
+// %17:_(<2 x s8>) = G_BUILD_VECTOR %22:_(s8), %23:_(s8)
+// %28:_(s16) = G_CONSTANT i16 257
+// %19:_(<2 x s8>) = G_BITCAST %28:_(s16)
+// %24:_(s16) = G_BITCAST %17:_(<2 x s8>)
+// %25:_(s16) = G_BITCAST %19:_(<2 x s8>)
+// %26:_(s16) = G_AND %24:_, %25:_
+// %res:_(<2 x s8>) = G_BITCAST %26:_(s16)
+// => bfe %1, %CmpRes, 1, 0
+// bfe %2, %CmpRes, 1, 16
+// bfi %3, %1, %2, 8
+// trunc.16.32 %res, %3
+def v2_i1_zext_to_bfe_bfi : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_BITCAST):$root,
+ [{ return matchV2i1ZextToBfeBfi(*${root}, ${matchinfo}); }]),
+ (apply [{ applyV2i1ZextToBfeBfi(*${root}, ${matchinfo}); }])>;
+
+// frcp(sqrt(x)) -> frsqrt(x)
+// sqrt(frcp(x)) -> frsqrt(x)
+// frcp(fabs(sqrt(x))) -> fabs(frsqrt(x))
+def rcp_sqrt_to_rsqrt : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_INTRINSIC, G_FSQRT, G_FDIV):$root,
+ [{ return matchRcpSqrtToRsqrt(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// s32 %floor = G_FFLOOR %x
+// s32 %dst = G_FSUB %x, %floor
+// => s32 %dst = frc %x
+// The plain G_FSUB fold requires contract on both the fsub and the floor. Also
+// matches the pisa.fsub intrinsic form, but only for round.towardzero with no
+// saturation.
+def sub_floor_to_frc : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FSUB, G_INTRINSIC):$root,
+ [{ return matchSubFloorToFrc(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// sequence repeated multiple times ...
+// %39:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<32 x s8>), [0..31]
+// %41:_(s32) = G_ZEXT %39:_(s8)
+// %42:_(s32) = nuw nsw G_SHL %41:_, [0,8,16,24]
+// %43:_(s32) = disjoint G_OR %38:_, %42:_
+// followed by ...
+// %157:_(<8 x s32>) = G_BUILD_VECTOR %43:_(s32), ...
+// => %157:_(<8 x s32>) = G_BITCAST %4:_(<32 x s8>)
+def extract_insert_to_bitcast : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchExtractInsertToBitcast(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyExtractInsertToBitcast(*${root}, ${matchinfo}); }])
+>;
+
+// %374:_(s16) = G_CONSTANT i16 1
+// %375:_(s16) = G_AND %321:_, %374:_
+// %421:_(s16) = G_CONSTANT i16 0
+// %422:_(s16) = G_CONSTANT i16 1
+// %405:_(s16) = G_PISA_SELECT %375:_, %422:_, %421:_
+// => %374:_(s16) = G_CONSTANT i16 1
+// => %375:_(s16) = G_AND %321:_, %374:_
+// => %405:_(s16) = COPY %375
+def and_select : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_PISA_SELECT):$root,
+ [{ return matchAndSelect(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyAndSelect(*${root}, ${matchinfo}); }])
+>;
+
+// %23:_(<4 x s8>) = G_BITCAST %1:_(s32)
+// %2:_(s8) = G_EXTRACT_VECTOR_ELT %23:_(<4 x s8>), %25:_(s32)
+// %7:_(s32) = G_LSHR %1:_, %6:_(s32)
+// %8:_(s8) = G_TRUNC %7:_(s32)
+// %12:_(s32) = G_LSHR %1:_, %11:_(s32)
+// %13:_(s8) = G_TRUNC %12:_(s32)
+// %24:_(s32) = G_CONSTANT i32 3
+// %18:_(s8) = G_EXTRACT_VECTOR_ELT %23:_(<4 x s8>), %24:_(s32)
+// %19:_(<4 x s8>) = G_BUILD_VECTOR %2:_(s8), %8:_(s8), %13:_(s8), %18:_(s8)
+// => %19:_(<4 x s8>) = G_BITCAST %1:_(s32)
+def extract_build_vector_to_bitcast : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchExtractBuildVectorToBitcast(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyExtractBuildVectorToBitcast(*${root}, ${matchinfo}); }])
+>;
+
+// Reduce predicates in s32.
+// %12:_(s1) = G_CONSTANT i1 true
+// %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %4:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %5:_(s32)
+// %6:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %7:_(s32)
+// %8:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %9:_(s32)
+// %10:_(s1) = G_AND %4:_, %6:_
+// %11:_(s1) = G_AND %10:_, %8:_
+// %13:_(s1) = G_ICMP intpred(eq), %11:_(s1), %12:_
+// => %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %18:_(<3 x s32>) = G_SEXT %3:_(<3 x s1>)
+// %20:_(s32) = G_CONSTANT i32 0
+// %19:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %20:_(s32)
+// %23:_(s32) = G_CONSTANT i32 1
+// %21:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %23:_(s32)
+// %22:_(s32) = G_AND %19:_, %21:_
+// %26:_(s32) = G_CONSTANT i32 2
+// %24:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %26:_(s32)
+// %25:_(s32) = G_AND %22:_, %24:_
+// %27:_(s32) = G_CONSTANT i32 -1
+// %13:_(s1) = G_ICMP intpred(eq), %25:_(s32), %27:_
+// Supports patterns with sext too:
+// %13:_(s8) = G_CONSTANT i8 -1
+// %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %4:_(<3 x s8>) = G_SEXT %3:_(<3 x s1>)
+// %5:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %6:_(s32)
+// %7:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %8:_(s32)
+// %9:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %10:_(s32)
+// %11:_(s8) = G_AND %5:_, %7:_
+// %12:_(s8) = G_AND %11:_, %9:_
+// %14:_(s1) = G_ICMP intpred(eq), %12:_(s8), %13:_
+def reduce_predicates : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_AND, G_OR, G_XOR):$root,
+ [{ return matchReducePredicates(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %23:_(s32) = G_CONSTANT i32 8
+// %17:_(s8) = G_TRUNC %1:reg32b(s32)
+// %24:_(s32) = G_LSHR %1:reg32b, %23:_(s32)
+// %25:_(s8) = G_TRUNC %24:_(s32)
+// %28:_(s8) = G_TRUNC %2:reg32b(s32)
+// %31:_(s32) = G_LSHR %2:reg32b, %23:_(s32)
+// %32:_(s8) = G_TRUNC %31:_(s32)
+///%149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
+// %141:_(s32) = G_BITCAST %149:_(<4 x s8>)
+// => %A = G_AND %1, 0xFFFF
+// => %B = G_SHL %2, 16
+// => %141 = G_OR %A, %B
+def reg2_matchdata : GIDefMatchData<"Reg2MatchInfo">;
+def build_reg_from_2 : GICombineRule<
+ (defs root:$root, reg2_matchdata:$matchinfo),
+ (match (wip_match_opcode G_BITCAST):$root,
+ [{ return matchBuildRegFrom2(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyBuildRegFrom2(*${root}, ${matchinfo}); }])
+>;
+
+// %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
+// %22:_(s32) = G_CONSTANT i32 0
+// %23:_(s32) = G_CONSTANT i32 -1
+// %10:_(s32) = G_SELECT %9:_(s1), %23:_, %22:_
+// %11:_(s32) = G_CONSTANT i32 -1
+// %12:_(s32) = G_XOR %10:_, %11:_
+// %13:_(s32) = G_CONSTANT i32 -1
+// %6:_(s1) = G_ICMP intpred(eq), %12:_(s32), %13:_
+// G_BRCOND %6:_(s1), %bb.4
+// G_BR %bb.3
+// => %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
+// => G_BRCOND %6:_(s1), %bb.3
+// => G_BR %bb.4
+def opt_fcmp_brcond_by_inverting_cond_matchdata : GIDefMatchData<"std::tuple<MachineInstr *, Register>">;
+def opt_fcmp_brcond_by_inverting_cond : GICombineRule<
+ (defs root:$root, opt_fcmp_brcond_by_inverting_cond_matchdata:$matchinfo),
+ (match (wip_match_opcode G_BR):$root,
+ [{ return matchFCmpInvertedCond(*${root}, ${matchinfo}); }]),
+ (apply [{ applyFCmpInvertedCond(*${root}, ${matchinfo}); }])
+>;
+
+def cmp_and_all_ones : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_ICMP):$root,
+ [{ return matchCmpAndAllOnes(*${root}, VT, ${matchinfo}); }]),
+ (apply
+ [{ applyCmpAndAllOnes(*${root}, ${matchinfo}); }])
+>;
+
+def sink_trunc_matchdata : GIDefMatchData<"std::tuple<Register, int64_t>">;
+def sink_trunc : GICombineRule<
+ (defs root:$root, sink_trunc_matchdata:$matchinfo),
+ (match (wip_match_opcode G_AND, G_OR, G_XOR):$root,
+ [{ return matchSinkTrunc(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applySinkTrunc(*${root}, ${matchinfo}); }])
+>;
+
+def trunc_trunc : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_TRUNC):$root,
+ [{ return matchTruncTrunc(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyTruncTrunc(*${root}, ${matchinfo}); }])
+>;
+
+// %1:registers(s32) = G_ABS %0:reg32b
+// %4:registers(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired),
+// IRedOp::UMAX, %1:registers(s32), %2:reg32b(s32), %3:reg32b(s32)
+// => %1:registers(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired),
+// IRedOp::ABSMAX, %0:registers(s32), %2:reg32b(s32), %3:reg32b(s32)
+def abs_redmax_to_redabsmax : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_INTRINSIC_CONVERGENT):$root,
+ [{ return matchAbsRedMaxToRedAbsMax(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %9:_(s1) = G_CONSTANT i1 true
+// %8:_(s1) = G_ICMP intpred(eq), %15:_(s32), %22:_
+// %10:_(s1) = G_ICMP intpred(eq), %8:_(s1), %9:_
+// %11:_(s32) = G_SEXT %10:_(s1)
+// => %8:_(s1) = G_ICMP intpred(eq), %15:_(s32), %22:_
+// %11:_(s32) = G_SEXT %8:_(s1)
+// or
+// %12:_(s1) = G_CONSTANT i1 true
+// %11:_(s1) = G_AND %10:_, %35:_
+// %13:_(s1) = G_ICMP intpred(eq), %11:_(s1), %12:_
+// %14:_(s32) = G_SEXT %13:_(s1)
+// => %11:_(s1) = G_AND %10:_, %35:_
+// %14:_(s32) = G_SEXT %11:_(s1)
+def cmp_int1 : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_ICMP):$root,
+ [{ return matchCmpInt1(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %1:_(i1) = G_TRUNC %0:_(wide)
+// %2:_(N) = G_SELECT %1:_(i1), N 1, N 0
+// => %3:_(wide) = G_AND %0, 1
+// %2:_(N) = G_TRUNC %3
+// Avoids introducing an illegal i1 compare when lowering bool-to-int.
+def select_trunc_one_zero : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_SELECT):$root,
+ [{ return matchSelectTruncOneZero(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %45:_(s32) = G_CONSTANT i32 -1
+// %60:_(s32) = G_CONSTANT i32 0
+// %46:_(s32) = G_SELECT %10:_(s1), %45:_, %60:_
+// %47:_(s32) = G_SELECT %35:_(s1), %45:_, %60:_
+// %48:_(s32) = G_AND %46:_, %47:_
+// %59:_(s32) = G_CONSTANT i32 31
+// %58:_(s32) = G_SHL %48:_, %59:_(s32)
+// %14:_(s32) = G_ASHR %58:_, %59:_(s32)
+// => %14:_(s32) = G_AND %46:_, %47:_
+def shift_true_false : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_ASHR):$root,
+ [{ return matchShiftTrueFalse(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+// %54:_(s8), %55:_(s8), %56:_(s8), %57:_(s8) = G_UNMERGE_VALUES %21:_(<4 x s8>)
+// %39:_(s16) = G_ANYEXT %54:_(s8)
+// %40:_(s16) = G_ANYEXT %55:_(s8)
+// %41:_(s16) = G_ADD %39:_, %40:_
+// %36:_(s16) = G_ANYEXT %56:_(s8)
+// %37:_(s16) = G_ANYEXT %57:_(s8)
+// %38:_(s16) = G_ADD %36:_, %37:_
+// %35:_(s16) = G_ADD %41:_, %38:_
+// => %40:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.dp4a.uu), 0, %21, 0x01010101, 0
+// %35:_(s16) = G_TRUNC %40:_(s32)
+def add_i8_reduction : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_ADD):$root,
+ [{ return matchAddInt8Reduction(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
+
+// %370:_(i16), %371:_(i16) = G_UNMERGE_VALUES %366:_(<2 x i16>)
+// %378:_(f16) = G_BITCAST %370:_(i16)
+// %379:_(f16) = G_BITCAST %371:_(i16)
+// %37:_(<2 x f16>) = G_BUILD_VECTOR %378:_(f16), %379:_(f16)
+// => %37:_(<2 x f16>) = G_BITCAST %366:_(<2 x i16>)
+def unmerge_bitcast_buildvector_to_bitcast : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchUnmergeBitcastBuildVectorToBitcast(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyUnmergeBitcastBuildVectorToBitcast(*${root}, ${matchinfo}); }])>;
+
+// G_FENCE ordering_A, scope_X
+// ... pure computation (no memory ops) ...
+// G_FENCE ordering_B, scope_X
+// => coalesce into a single G_FENCE with the merged AtomicOrdering
+// (e.g. release+acquire -> acq_rel, anything+seq_cst -> seq_cst)
+def redundant_fence : GICombineRule<
+ (defs root:$root, MI_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FENCE):$root,
+ [{ return matchRedundantFence(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyRedundantFence(*${root}, ${matchinfo}); }])
+>;
+
+// G_FENCE ordering, scope_X-addrspace_A
+// ... pure computation (no memory ops) ...
+// G_FENCE ordering, scope_X-addrspace_B
+// => G_FENCE ordering, scope_X-generic
+// Merges adjacent fences with same ordering and same base scope but different
+// address spaces into a single generic fence.
+def merge_adjacent_fences : GICombineRule<
+ (defs root:$root, MI_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FENCE):$root,
+ [{ return matchMergeAdjacentFences(*${root}, ${matchinfo}); }]),
+ (apply
+ [{ applyMergeAdjacentFences(*${root}, ${matchinfo}); }])
+>;
+
+// p2i_to_i2p do not account for pointer-size or address-space difference
+def pisa_identity_combines : GICombineGroup<
+ !listremove(identity_combines.Rules, [p2i_to_i2p])>;
+// avoid G_SELECT %2:_(i1), -1, 0 => G_SEXT %2:_(i1)
+def pisa_select_combines : GICombineGroup<
+ !listremove(select_combines.Rules, [match_selects])>;
+// avoid G_BUILD_VECTOR => G_CONCAT_VECTOR
+// avoid x,y<dead>=unmerge(z) => x=trunc(z): causes G_TRUNC on wide types
+def pisa_merge_combines : GICombineGroup<
+ !listremove(merge_combines.Rules, [combine_build_unmerge, unmerge_dead_to_trunc])>;
+// avoid G_BUILD_VECTOR(undef,undef) => G_IMPLICIT_DEF
+// avoid G_BITCAST/G_TRUNC(undef) => G_IMPLICIT_DEF: post-legalizer must not
+// introduce G_IMPLICIT_DEF for types the legalizer already scalarized
+// (e.g., <2 x i32> resulting from a bitcast of a legal i64 undef)
+def pisa_undef_combines : GICombineGroup<
+ !listremove(undef_combines.Rules, [propagate_undef_all_ops,
+ unary_undef_to_undef])>;
+// narrow_binop_* are conflicting with hoist_logic_op_with_same_opcode_hands, remove them to avoid loop.
+def pisa_cast_combines: GICombineGroup<
+ !listremove(cast_combines.Rules, [
+ narrow_binop_and,
+ narrow_binop_or,
+ narrow_binop_xor])>;
+
+// Rules removed from both pre- and post-legalizer combiners.
+def pisa_removed_combines : GICombineGroup<[
+ identity_combines, // use pisa_identity_combines
+ select_combines, // use pisa_select_combines
+ cast_combines, // use pisa_cast_combines
+ merge_combines, // use pisa_merge_combines
+ undef_combines, // use pisa_undef_combines
+ combines_for_extload, // prevents post-legalizer re-folding of G_LOAD+G_SEXT → G_SEXTLOAD
+ div_rem_to_divrem, // G_[US]DIVREM not supported
+ intrem_combines, // use [su]div.rem instruction
+ opt_brcond_by_inverting_cond,
+ trunc_shift,
+ fold_binop_into_select, // breaks sext(fcmp) AND patterns needed for bfn matching
+ freeze_combines // push_freeze_to_prevent_poison_from_propagating can push G_FREEZE through G_SELECT creating illegal G_FREEZE(i1) after legalization; exclude entire group and re-add freeze_of_non_undef_non_poison explicitly below
+]>;
+
+// Additional rules removed only from the post-legalizer combiner.
+// - intdiv_combines(sdiv_by_const) emits G_SMULH/G_UMULH
+def pisa_post_removed_combines : GICombineGroup<[
+ commute_shift,
+ intdiv_combines
+]>;
+
+// Fold min/max with an identity-element constant:
+// umin(x, UINT_MAX) -> x, umax(x, 0) -> x
+// smin(x, INT_MAX) -> x, smax(x, INT_MIN) -> x
+// The legalizer introduces such patterns when expanding G_VECREDUCE_U/SMIN/MAX
+// (padding lanes are filled with the identity element).
+def minmax_identity_fold : GICombineRule<
+ (defs root:$root, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_UMIN, G_UMAX, G_SMIN, G_SMAX):$root,
+ [{ return matchMinMaxIdentityFold(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
+>;
+
+// Fix G_SHL/G_LSHR/G_ASHR shift amounts to i32 post-legalization.
+// PISA only legalizes shifts with i32 shift amounts ({I16,I32}, {I32,I32},
+// {I64,I32}), but post-legalizer rules such as mul_to_shl may introduce a
+// new shift whose amount has the value type (e.g. i64). This rule truncates
+// or zero-extends such shift amounts back to i32.
+def pisa_fix_shift_amt : GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_SHL, G_LSHR, G_ASHR):$root,
+ [{ return matchFixIllegalShiftAmt(*${root}); }]),
+ (apply [{ applyFixIllegalShiftAmt(*${root}); }])>;
+
+let CombineAllMethodName = "tryCombineAllImpl" in {
+
+def PISAPreLegalizerCombiner: GICombiner<
+ "PISAPreLegalizerCombinerImpl",
+ !listremove(
+ !listconcat(all_combines.Rules, [
+ pisa_identity_combines, pisa_cast_combines, pisa_select_combines, pisa_merge_combines, pisa_undef_combines,
+ truncated_load, truncated_store, trunc_shift_to_vector_extract,
+ extended_load, simplify_nonpowerof2_load_store_chain, expand_nonpowerof2_load_store,
+ extract_insert_to_bitcast, redundant_bitcasts, remove_redundant_moves_pre,
+ rcp_sqrt_to_rsqrt, sub_floor_to_frc, extract_build_vector_to_bitcast,
+ reduce_predicates, cmp_int1, select_trunc_one_zero,
+ // match_selects must run in the pre-legalizer (after select_trunc_one_zero
+ // so trunc-based patterns take priority) to convert
+ // G_SELECT(i1, true, x) → G_OR(i1, x) via tryFoldBoolSelectToLogic.
+ // After legalization this OR becomes a real G_OR that PISABFNMatcher uses
+ // to produce bfn.0xa8. It is excluded from pisa_select_combines
+ // (post-legalizer only) to prevent select(i1, wide -1, wide 0) → sext(i1).
+ match_selects,
+ lane_id_shl_chain, zext_and_to_and_zext,
+ freeze_of_non_undef_non_poison]),
+ pisa_removed_combines.Rules)>;
+
+def PISAPostLegalizerCombiner: GICombiner<
+ "PISAPostLegalizerCombinerImpl",
+ !listremove(
+ !listconcat(all_combines.Rules, [
+ pisa_identity_combines, pisa_cast_combines, pisa_select_combines, pisa_merge_combines, pisa_undef_combines,
+ extra_commute_iconstant_to_rhs, extra_commute_fconstant_to_rhs, extra_commute_intrinsic_iconstant_to_rhs,
+ extra_remove_redundant_constant, build_vector_with_constants, compare_select_reg,
+ shl_add_to_mad, fdiv_to_rcp_fmul, const_and_trunc_to_copy,
+ build_vector_extract_to_copy, extract_subvector_to_extract_elt, extract_subvector_build_vector, extract_subvector_partial, build_vector_from_unmerge_lanes, build_vector_concat_subvectors, shifts_of_constants,
+ remove_redundant_moves_post, p2i_to_i2p_fixed, extract_all_build_vector, unmerge_bitcast_buildvector_to_bitcast,
+ redundant_bitcasts, or_and_to_bfi, shift_sub_to_bfe, v2_i1_zext_to_bfe_bfi, and_bitfield_to_bitfield,
+ and_select, build_reg_from_2, opt_fcmp_brcond_by_inverting_cond,
+ cmp_and_all_ones, sink_trunc, trunc_trunc, abs_redmax_to_redabsmax,
+ shift_true_false, add_i8_reduction, redundant_fence,
+ freeze_of_non_undef_non_poison, pisa_fix_shift_amt, minmax_identity_fold,
+ merge_adjacent_fences]),
+ !listconcat(pisa_removed_combines.Rules, pisa_post_removed_combines.Rules))>;
+}
diff --git a/llvm/lib/Target/PISA/PISADefines.h b/llvm/lib/Target/PISA/PISADefines.h
new file mode 100644
index 0000000000000..42887a9c86db4
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISADefines.h
@@ -0,0 +1,52 @@
+//===-- PISADefines.h - Common defines for PISA ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISADEFINES_H
+#define LLVM_LIB_TARGET_PISA_PISADEFINES_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include <cstdint>
+
+namespace llvm {
+namespace PISA {
+
+enum class Swizzle : unsigned { X, Y, Z, W, XYZW, XY, ZW, NONE };
+constexpr unsigned SimdSize = 32;
+
+// Keep these in sync with PISAInstrInfo.td!
+enum class LdCacheCtrl : unsigned {
+ Default = 0,
+ L1c = 1,
+ L1uc = 2,
+ L1s = 3,
+ L2c = 4,
+ L2uc = 5,
+ L3c = 6,
+ L3uc = 7,
+ ri = 8
+};
+
+enum class StCacheCtrl : unsigned {
+ Default = 0,
+ L1uc = 1,
+ L1wb = 2,
+ L1wt = 3,
+ L1s = 4,
+ L2uc = 5,
+ L2wb = 6,
+ L3uc = 7,
+ L3wb = 8
+};
+
+enum class AtomCacheCtrl : unsigned { Default = 0, uc = 2, L2wb = 4, L3wb = 3 };
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISADEFINES_H
diff --git a/llvm/lib/Target/PISA/PISAGenericOpcodes.td b/llvm/lib/Target/PISA/PISAGenericOpcodes.td
new file mode 100644
index 0000000000000..f3579d60d86d3
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAGenericOpcodes.td
@@ -0,0 +1,99 @@
+//===-- PISAGenericOpcodes.td - PISA Generic Opcodes -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+class PISAGenericInstruction : GenericInstruction {
+ let Namespace = Global.NS;
+}
+
+def G_PISA_PARAM_SLOT : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins untyped_imm_0:$slot, untyped_imm_0:$offset,
+ variable_ops);
+ let hasSideEffects = false;
+}
+
+// Target-specific generic instruction
+class PISAUnaryInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1);
+ let hasSideEffects = false;
+}
+class PISABinaryInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2);
+ let hasSideEffects = false;
+ let isCommutable = true;
+}
+class PISABinaryInstructionRndSat : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2, untyped_imm_0:$src3, untyped_imm_0:$src4);
+ let hasSideEffects = false;
+ let isCommutable = false;
+}
+class PISABinaryInstructionNanp : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2, untyped_imm_0:$nanp);
+ let hasSideEffects = false;
+ let isCommutable = true;
+}
+class PISATernaryInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2, type0:$src3);
+ let hasSideEffects = false;
+ let isCommutable = false;
+}
+class PISAReductionInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins untyped_imm_0:$op, type0:$src1, type1:$src2, type0:$src3);
+ let hasSideEffects = false;
+ let isCommutable = false;
+}
+class PISAReductionInstructionNanp : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins untyped_imm_0:$op, type0:$src1, type1:$src2, type0:$src3, untyped_imm_0:$nanp);
+ let hasSideEffects = false;
+ let isCommutable = false;
+}
+class PISATernaryInstructionRndSat : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2, type0:$src3, untyped_imm_0:$src4, untyped_imm_0:$src5);
+ let hasSideEffects = false;
+ let isCommutable = false;
+}
+class PISACompareInstruction : PISABinaryInstruction {
+ let isCommutable = false;
+}
+
+
+class PISAConvertInstWithRnd : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$src1, type2:$src2);
+ let hasSideEffects = false;
+}
+
+class PISACasInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins untyped_imm_0:$src1, type0:$src2, type0:$src3, untyped_imm_0:$src4);
+ let hasSideEffects = false;
+ let isCommutable = false;
+ let mayLoad = true;
+ let mayStore = true;
+}
+class PISAFAtomInstruction : PISAGenericInstruction {
+ let OutOperandList = (outs type0:$oldval);
+ let InOperandList = (ins ptype1:$addr, type0:$val);
+ let hasSideEffects = false;
+ let isCommutable = false;
+ let mayLoad = true;
+ let mayStore = true;
+}
+
+// sel
+def G_PISA_SELECT : PISATernaryInstruction;
+def pisa_select : SDNode<"PISAISD::SELECT", SDTSelect>;
+def : GINodeEquiv<G_PISA_SELECT, pisa_select>;
diff --git a/llvm/lib/Target/PISA/PISAHelpers.h b/llvm/lib/Target/PISA/PISAHelpers.h
new file mode 100644
index 0000000000000..95cb1b5aa36f9
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAHelpers.h
@@ -0,0 +1,38 @@
+//===-- PISAHelpers.h - Common helpers for PISA ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAHELPERS_H
+#define LLVM_LIB_TARGET_PISA_PISAHELPERS_H
+
+#include "PISADefines.h"
+
+#include "llvm/Support/ErrorHandling.h"
+
+namespace llvm::PISA {
+
+inline unsigned getSwizzleElemCount(PISA::Swizzle Swizzle) {
+ switch (Swizzle) {
+ case PISA::Swizzle::NONE:
+ return 0;
+ case PISA::Swizzle::X:
+ case PISA::Swizzle::Y:
+ case PISA::Swizzle::Z:
+ case PISA::Swizzle::W:
+ return 1;
+ case PISA::Swizzle::XY:
+ case PISA::Swizzle::ZW:
+ return 2;
+ case PISA::Swizzle::XYZW:
+ return 4;
+ }
+ llvm_unreachable("Unknown swizzle!");
+}
+
+} // namespace llvm::PISA
+
+#endif // LLVM_LIB_TARGET_PISA_PISAHELPERS_H
diff --git a/llvm/lib/Target/PISA/PISAInstrFormats.td b/llvm/lib/Target/PISA/PISAInstrFormats.td
new file mode 100644
index 0000000000000..f1b5a8a2646a6
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAInstrFormats.td
@@ -0,0 +1,834 @@
+//===-- PISAInstrFormats.td - PISA Instruction Formats -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "PISAGenericOpcodes.td"
+include "llvm/Target/PISA/MnemonicOperand.td"
+
+class PISAAsmOperand<string am, string sig=am> : AsmOperandClass {
+ let Name = am;
+ string Signature = sig;
+ let ParserMethod = "parse"#Signature;
+ let PredicateMethod = "is"#Signature;
+}
+
+// This operand class provides a simple interface for quickly describing
+// the syntax of memory operands that are formed by a sequence
+// For example, specifying "R,i" for the args means that
+// the operand would look something like: [%r0, 5].
+class PISAMemSeqAsmOperand<string am, string args, string uniq> : AsmOperandClass {
+ let Name = am#uniq;
+ let ParserMethod = "parse"#am#"(\""#args#"\")";
+ let PredicateMethod = "is"#am;
+ let RenderMethod = "add"#am#"Operands";
+}
+
+class PISAOperandName<string name> {
+ string pm = "print"#name;
+ AsmOperandClass pmc = PISAAsmOperand<name>;
+ string em = "encode"#name;
+ string dm = "decode"#name;
+}
+
+class PISAAsmRegisterOperandPMC<RegisterClass rc, string sig>
+ : PISAAsmOperand<rc#"Opnd", sig> {
+ let RenderMethod = "add"#Signature;
+}
+
+class PISARegisterOperand<RegisterClass rc>
+ : RegisterOperand<rc> {
+ defvar name = "RegOpnd";
+ defvar RCIDStr = "PISA::"#!cast<string>(rc)#"RegClassID";
+ string Signature = name#"("#RCIDStr#")";
+
+ let PrintMethod = "print"#Signature;
+ let EncoderMethod = "encode"#Signature;
+ let DecoderMethod = "decode"#Signature;
+ let ParserMatchClass = PISAAsmRegisterOperandPMC<rc, Signature>;
+}
+
+class PISALocalVariableOperand
+ : Operand<i32>, PISAOperandName<"LocalVariableOpnd"> {
+ let PrintMethod = pm;
+ let ParserMatchClass = pmc;
+ let EncoderMethod = em;
+ let DecoderMethod = dm;
+}
+
+def FrameIndex : PISALocalVariableOperand;
+
+class PISAImmediateOperand<ValueType ty, string name>
+ : Operand<ty>, PISAOperandName<name> {
+ let OperandType = "OPERAND_IMMEDIATE";
+ let PrintMethod = pm;
+ let ParserMatchClass = pmc;
+ let EncoderMethod = em;
+ let DecoderMethod = dm;
+}
+
+class PISAImmediateRangeOperand<ValueType ty, string name, int min, int max>
+ : PISAImmediateOperand<ty, name>,
+ TImmLeaf<ty, "return Imm >= " # min # " && Imm <= " # max # ";">
+{}
+
+class PISABfnOpcodeOperand<ValueType ty, string name>
+ : PISAImmediateOperand<ty, name> {
+ let PrintMethod = "printBfnOpcode";
+}
+
+
+class PISASymbolOperand<ValueType ty, string name>
+ : Operand<ty>, PISAOperandName<name> {
+ let OperandType = "OPERAND_PCREL";
+ let PrintMethod = pm;
+ let ParserMatchClass = pmc;
+ let EncoderMethod = em;
+ let DecoderMethod = dm;
+}
+
+// Immediate address offset operand for the load/store
+class AddrOffImm<ValueType VT> : Operand<VT>{
+ let OperandType = "OPERAND_IMMEDIATE";
+ let PrintMethod = "printAddrOffsetImm";
+}
+
+def PISAGlobalVariableOpnd : PISASymbolOperand<OtherVT, "GlobalVariableOpnd">;
+
+// function name as operand (addrof)
+def FunctionOpnd : PISASymbolOperand<OtherVT, "FunctionOpnd"> {
+ // print the same as a global variable operand, i.e. "@func"
+ let PrintMethod = "printGlobalVariableOpnd";
+}
+
+class IsQualifier<string Name, string QName> {
+ defvar QIndex = !find(Name, QName#".");
+ defvar QIndexAtEnd = !if(!gt(!size(Name), !size(QName)), !sub(!size(Name), !size(QName)), 0);
+ defvar IsAtEnd = !if(!eq(QIndex, -1), !eq(!substr(Name, QIndexAtEnd, !size(QName)), QName), 0);
+ bit ret = !or(!ne(QIndex, -1), IsAtEnd);
+}
+
+class ExtractVectorIndex<string name> {
+ int ret = !foldl(-1, !range(0, 64), acc, idx,
+ !if(IsQualifier<name, "."#idx>.ret, idx, acc));
+}
+
+class _PISAInst<string Name = "UNKNOWN"> : Instruction {
+ field bits<16> Inst;
+
+ let Namespace = "PISA";
+ let DecoderNamespace = "PISA";
+
+ // !cast<void>(directive)
+ let UseNamedOperandTable = 1;
+}
+
+class PISAInst<dag outs, dag ins, string asmstr, list<dag> pattern = []>
+ : _PISAInst<asmstr> {
+ dag OutOperandList = outs;
+ dag InOperandList = ins;
+ let AsmString = asmstr;
+ let Pattern = pattern;
+}
+
+// pisaVT should be a container class only instead of a ValueType.
+class pisaVT<ValueType vt, string d, RegisterClass rc, bit isHasRC = 1> {
+ ValueType VT = vt;
+ string directive = d;
+ string opcode = !subst(".", "_", !if(vt.isFP, directive, !subst("b", "", directive)));
+ string name = !subst("_", "", !cast<string>(NAME));
+
+ string bitwidth = vt.Size#b;
+ // mdirective specifies directive for memory operations (e.g. .32b for .f type)
+ string mdirective = !if(!and(vt.isFP, !not(vt.isVector)), "."#bitwidth, directive);
+ RegisterClass RC = rc;
+ DAGOperand ImmOpnd;
+ DAGOperand RegOpnd = !if(isHasRC, PISARegisterOperand<rc>, ?);
+}
+
+// Subclasses of pisaIntVT, pisaFloatVT, pisaPtrVT, pisaVectorVT, and pisaVoidVT
+// are inherited from pisaVT.
+// The main goal is to add the functionailty of !isa<subclass>,
+// e.g., !isa<pisaIntVT>(_i16) return 1 to tell that it is an int type.
+class pisaIntVT<ValueType vt, string d, RegisterClass rc> :
+ pisaVT<vt, d, rc> {
+ let ImmOpnd = PISAImmediateOperand <vt, "Imm"#vt.Size#"Opnd">;
+}
+
+class pisaFloatVT<ValueType vt, string d, RegisterClass rc> :
+ pisaVT<vt, d, rc> {
+ let ImmOpnd = PISAImmediateOperand <vt, "FpImm"#vt.Size#"Opnd">;
+}
+
+class pisaPtrVT<ValueType vt, string d, RegisterClass rc> :
+ pisaVT<vt, d, rc>;
+
+class pisaVectorVT<ValueType vt, string d, RegisterClass rc> :
+ pisaVT<vt, d, rc>;
+
+class pisaVoidVT<ValueType vt, string d, RegisterClass rc> :
+ pisaVT<vt, d, rc, 0>;
+
+def p0 : PtrValueType<i64, 0>;
+def p1 : PtrValueType<i64, 1>;
+def p2 : PtrValueType<i64, 2>;
+def p3 : PtrValueType<i32, 3>;
+def p4 : PtrValueType<i32, 4>;
+
+// <ValueType, directive, RegisterClass>;
+def _void : pisaVoidVT<isVoid, "", ? >;
+def _i1 : pisaIntVT<i1, ".1b", Pred >;
+def _i8 : pisaIntVT<i8, ".8b", Reg8b >;
+def _i16 : pisaIntVT<i16, ".16b", Reg16b >;
+def _i32 : pisaIntVT<i32, ".32b", Reg32b >;
+def _i64 : pisaIntVT<i64, ".64b", Reg64b >;
+def _i128 : pisaIntVT<i128, ".128b", Reg128b >;
+def _f16 : pisaFloatVT<f16, ".hf", Reg16b >;
+def _bf16 : pisaFloatVT<bf16, ".bf", Reg16b >;
+def _f32 : pisaFloatVT<f32, ".f", Reg32b >;
+def _f64 : pisaFloatVT<f64, ".df", Reg64b >;
+def _p0 : pisaPtrVT<p0, ".64b", Reg64b >;
+def _p1 : pisaPtrVT<p1, ".64b", Reg64b >;
+def _p2 : pisaPtrVT<p2, ".64b", Reg64b >;
+def _p3 : pisaPtrVT<p3, ".32b", Reg32b >;
+def _p4 : pisaPtrVT<p4, ".32b", Reg32b >;
+def _v2i8 : pisaVectorVT<v2i8, ".v2.8b", RegV2_8b >;
+def _v3i8 : pisaVectorVT<v3i8, ".v3.8b", RegV3_8b >;
+def _v4i8 : pisaVectorVT<v4i8, ".v4.8b", RegV4_8b >;
+def _v2i16 : pisaVectorVT<v2i16, ".v2.16b", RegV2_16b >;
+def _v3i16 : pisaVectorVT<v3i16, ".v3.16b", RegV3_16b >;
+def _v4i16 : pisaVectorVT<v4i16, ".v4.16b", RegV4_16b >;
+def _v2i32 : pisaVectorVT<v2i32, ".v2.32b", RegV2_32b >;
+def _v3i32 : pisaVectorVT<v3i32, ".v3.32b", RegV3_32b >;
+def _v4i32 : pisaVectorVT<v4i32, ".v4.32b", RegV4_32b >;
+def _v5i32 : pisaVectorVT<v5i32, ".v5.32b", RegV5_32b >;
+def _v6i32 : pisaVectorVT<v6i32, ".v6.32b", RegV6_32b >;
+def _v7i32 : pisaVectorVT<v7i32, ".v7.32b", RegV7_32b >;
+def _v8i32 : pisaVectorVT<v8i32, ".v8.32b", RegV8_32b >;
+def _v16i32 : pisaVectorVT<v16i32, ".v16.32b", RegV16_32b >;
+def _v32i32 : pisaVectorVT<v32i32, ".v32.32b", RegV32_32b >;
+def _v64i32 : pisaVectorVT<v64i32, ".v64.32b", RegV64_32b >;
+def _v2i64 : pisaVectorVT<v2i64, ".v2.64b", RegV2_64b >;
+def _v3i64 : pisaVectorVT<v3i64, ".v3.64b", RegV3_64b >;
+def _v4i64 : pisaVectorVT<v4i64, ".v4.64b", RegV4_64b >;
+def _v2f16 : pisaVectorVT<v2f16, ".v2.16b", RegV2_16b >;
+def _v3f16 : pisaVectorVT<v3f16, ".v3.16b", RegV3_16b >;
+def _v4f16 : pisaVectorVT<v4f16, ".v4.16b", RegV4_16b >;
+def _v2bf16 : pisaVectorVT<v2bf16, ".v2.16b", RegV2_16b >;
+def _v3bf16 : pisaVectorVT<v3bf16, ".v3.16b", RegV3_16b >;
+def _v4bf16 : pisaVectorVT<v4bf16, ".v4.16b", RegV4_16b >;
+def _v2f32 : pisaVectorVT<v2f32, ".v2.32b", RegV2_32b >;
+def _v3f32 : pisaVectorVT<v3f32, ".v3.32b", RegV3_32b >;
+def _v4f32 : pisaVectorVT<v4f32, ".v4.32b", RegV4_32b >;
+def _v5f32 : pisaVectorVT<v5f32, ".v5.32b", RegV5_32b >;
+def _v6f32 : pisaVectorVT<v6f32, ".v6.32b", RegV6_32b >;
+def _v7f32 : pisaVectorVT<v7f32, ".v7.32b", RegV7_32b >;
+def _v8f32 : pisaVectorVT<v8f32, ".v8.32b", RegV8_32b >;
+def _v2f64 : pisaVectorVT<v2f64, ".v2.64b", RegV2_64b >;
+def _v3f64 : pisaVectorVT<v3f64, ".v3.64b", RegV3_64b >;
+def _v4f64 : pisaVectorVT<v4f64, ".v4.64b", RegV4_64b >;
+
+class PISAValueTypes {
+ list<pisaVT> Integer = [_i8, _i16, _i32, _i64];
+ list<pisaVT> Float = [_bf16, _f16, _f32, _f64];
+ list<pisaVT> Ptr = [_p0, _p1, _p2, _p3, _p4];
+ list<pisaVT> Vector = [_v2i8, _v2i16, _v2i32, _v2i64,
+ _v3i8, _v3i16, _v3i32, _v3i64,
+ _v4i8, _v4i16, _v4i32, _v4i64,
+ _v8i32];
+ list<pisaVT> LargeVector = [_v5i32, _v6i32, _v7i32, _v16i32, _v32i32, _v64i32];
+ // types that can be used in mov instruction
+ // - register moves are limited to 128 bits
+ list<pisaVT> MovImm = !listconcat(Integer, Float, [_i128]);
+ list<pisaVT> MovReg = !filter(VT, !listconcat(MovImm, Vector), !le(VT.VT.Size, 128));
+ // types that can be used as function parameters or return values
+ list<pisaVT> CallReturn = !listconcat(Integer, Vector, LargeVector);
+ // types that can be used in ld/st instructions
+ list<pisaVT> LoadStore = [_i8, _i16, _i32, _i64,
+ _v2i8, _v2i16, _v2i32, _v2i64,
+ _v3i32, _v3i64,
+ _v4i8, _v4i16, _v4i32, _v4i64,
+ _v8i32];
+ // types that can be used in ld.param instructions
+ list<pisaVT> LoadParam = !listconcat(LoadStore, LargeVector);
+ // types that can be used in atomic load/store
+ list<pisaVT> AtomicLoadStore = [_i16, _i32, _i64, _i128];
+ // types that can be used in lifetime.start
+ list<pisaVT> LifetimeTypes = !listconcat([_i1, _i8, _i16, _i32, _i64, _i128],
+ Vector, LargeVector);
+
+}
+def VTs: PISAValueTypes;
+
+class getVectorType<int numElements, int bitSize> {
+ pisaVT VT = !if(!eq(numElements, 1),
+ !cast<pisaVT>("_i"#bitSize),
+ !cast<pisaVT>("_v"#numElements#"i"#bitSize));
+}
+
+class FenceFlushOp {}
+def evict : FenceFlushOp;
+def inval : FenceFlushOp;
+def clean : FenceFlushOp;
+
+class FenceScope {}
+def subgroup : FenceScope;
+def workgroup : FenceScope;
+def gpu : FenceScope;
+def system : FenceScope;
+
+class AddressSpace<int n, pisaVT vt, string _name = NAME> {
+ int num = n;
+ pisaVT VT = vt;
+ PtrValueType PVT = PtrValueType<vt.VT, n>; // !cast<PtrValueType>("p"#num);
+ DAGOperand RC = vt.RegOpnd;
+ string name = _name;
+ string directive = !if(!empty(name), "", "."#name);
+}
+def private : AddressSpace<4, _i32>;
+def global : AddressSpace<1, _i64>;
+def const : AddressSpace<2, _i64>;
+def shared : AddressSpace<3, _i32>;
+def generic : AddressSpace<0, _i64>;
+def default : AddressSpace<0, _i64, "">;
+// FIXME: Rename address space as storage space.
+// Special storage spaces.
+
+class StoreAddrspacePat<AddressSpace AS, PatFrag pat> :
+ PatFrag<(ops node:$data, node:$ptr), (pat node:$data, node:$ptr)> {
+ let IsStore = true;
+ let IsUnindexed = true;
+ let IsNonExtLoad = true;
+ let AddressSpaces = [ AS.num ];
+}
+
+class LoadAddrspacePat<AddressSpace AS, PatFrag pat> :
+ PatFrag<(ops node:$ptr), (pat node:$ptr)> {
+ let IsLoad = true;
+ let IsUnindexed = true;
+ let IsNonExtLoad = true;
+ let AddressSpaces = [ AS.num ];
+}
+
+
+class UnaryAtomAddrspacePat<AddressSpace Addrspace, PatFrag pat>
+ : PatFrag<(ops node:$ptr), (pat node:$ptr)> {
+ let AddressSpaces = [Addrspace.num];
+ let IsAtomic = 1;
+}
+
+class BinaryAtomAddrspacePat<AddressSpace Addrspace, PatFrag pat>
+ : PatFrag<(ops node:$ptr, node:$other), (pat node:$ptr, node:$other)> {
+ let AddressSpaces = [ Addrspace.num ];
+ let IsAtomic = 1;
+}
+
+// Variant of BinaryAtomAddrspacePat that also checks the result register type
+// in GlobalISel, needed to disambiguate bf16 from f16 (both are 16-bit).
+class BinaryAtomAddrspaceMemVTPat<AddressSpace Addrspace, PatFrag pat,
+ string MemVTCheck>
+ : PatFrag<(ops node:$ptr, node:$other), (pat node:$ptr, node:$other)> {
+ let AddressSpaces = [ Addrspace.num ];
+ let IsAtomic = 1;
+ let GISelPredicateCode = [{ return }] # MemVTCheck # [{; }];
+}
+
+// match cmpxchg instruction
+class TernaryAtomAddrspacePat<AddressSpace Addrspace, SDPatternOperator pat> :
+ PatFrag<(ops node:$ptr, node:$opnd1, node:$opnd2), (pat node:$ptr, node:$opnd1, node:$opnd2)> {
+ let AddressSpaces = [ Addrspace.num ];
+ let IsAtomic = 1;
+ let IsLoad = true;
+ let IsStore = true;
+}
+
+/********************************* PISAInst class hierarchy *********************************/
+// +--------------------------------+
+// | _PISAInst: |
+// | defines Inst, Namespace, |
+// (Layer 1) | DecoderNamespace, |
+// | UseNamedOperandTable |
+// +--------------------------------+
+// | | |
+// +---------------+ | +------------------+
+// v v v
+// +--------------------------+ +--------------------------+ +----------------------------+
+// | PISAInst1: | | PISAInst2: | | PISAInst3: |
+// | defines srcRC, | | defines src0RC, src1RC,| | defines src0RC, src1RC, |
+// | dstRC, | | dstRC, | | src2RC, dstRC, |
+// | AsmString, | | AsmString, | | AsmString, |
+// | OutOperandList | | OutOperandList | | OutOperandList |
+// +--------------------------+ +--------------------------+ +----------------------------+
+// | | |
+// v v v
+// +--------------------------+ +--------------------------+ +----------------------------+
+// | PISAInst_r, | | PISAInst_rr, PISAInst_ri,| | PISAInst_rrr, PISAInst_irr,|
+// | PISAInst_i: | | PISAInst_ir, PISAInst_ii:| | PISAInst_rri, PISAInst_iri,|
+// | defines InOperandList, | | defines InOperandList, | | PISAInst_rir, PISAInst_iir,|
+// | Imm opd | | Imm opd | | PISAInst_rii, PISAInst_irr,|
+// +--------------------------+ +--------------------------+ | PISAInst_iii: |
+// | | | defines InOperandList, |
+// | | | Imm opd |
+// | | +----------------------------+
+// | | |
+// | (Classes below are Layer 4) | |
+// | (x is either r or i) | |
+// v v v
+// +--------------------------+ +--------------------------+ +----------------------------+
+// | UnaryInst_x: | | BinInst_xx: | | TernaryInst_xxx |
+// | defines Pattern | | defines Pattern | | defines Pattern |
+// | | | | | |
+// | See also - | | See also - | | See also - |
+// | FP2IntRndmodeInst_r | | CmpBinaryInstReg_rx | | MadInst_rxx |
+// | Int2FPRndmodeInst_r | | CmpBinaryInstPred_rx | | |
+// | | | | | |
+// +--------------------------+ +--------------------------+ +----------------------------+
+//
+// Description: This is the class hierarchy for PISAInst. Each layer defines different fields
+// to share common code. The normal usage is to inherit a class from Layer 3
+// with desired patterns like rr, ri, etc, and to define its own 'Pattern' for
+// DAG selection. All these classes are for non-memory operations. We need to
+// define another formats for memop.
+//
+
+class PISAInst1<string name, pisaVT dstVT, pisaVT srcVT> : _PISAInst<name> {
+ DAGOperand srcRC = srcVT.RegOpnd;
+ DAGOperand dstRC = dstVT.RegOpnd;
+ let AsmString = !if(!isa<pisaVoidVT>(dstVT),
+ name#" \t$src;",
+ name#" \t$dst, $src;");
+ let OutOperandList = !if(!isa<pisaVoidVT>(dstVT),
+ (outs),
+ (outs dstRC:$dst));
+}
+
+class PISAInst_r<string name, pisaVT dstVT, pisaVT srcVT> : PISAInst1<name, dstVT, srcVT> {
+ let InOperandList = (ins srcRC:$src);
+}
+
+class PISAInst_i<string name, pisaVT dstVT, pisaVT srcVT> : PISAInst1<name, dstVT, srcVT> {
+ SDNode immVal = !if(!isa<pisaFloatVT>(srcVT), fpimm, imm);
+ DAGOperand immOpd = srcVT.ImmOpnd;
+ let InOperandList = (ins immOpd:$src);
+}
+
+class PISAInst2<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT> : _PISAInst<name> {
+ DAGOperand src0RC = src0VT.RegOpnd;
+ DAGOperand src1RC = src1VT.RegOpnd;
+ DAGOperand dstRC = dstVT.RegOpnd;
+ let AsmString = !if(!isa<pisaVoidVT>(dstVT),
+ name#" \t$src0, $src1;",
+ name#" \t$dst, $src0, $src1;");
+ let OutOperandList = !if(!isa<pisaVoidVT>(dstVT),
+ (outs),
+ (outs dstRC:$dst));
+}
+
+class PISAInst_rr<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT> : PISAInst2<name, dstVT, src0VT, src1VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1);
+}
+
+class PISAInst_ri<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT> : PISAInst2<name, dstVT, src0VT, src1VT> {
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1);
+}
+
+class PISAInst_ir<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT> : PISAInst2<name, dstVT, src0VT, src1VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1);
+}
+
+class PISAInst_ii<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT> : PISAInst2<name, dstVT, src0VT, src1VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1);
+}
+
+class PISAInst3<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : _PISAInst<name> {
+ DAGOperand src0RC = src0VT.RegOpnd;
+ DAGOperand src1RC = src1VT.RegOpnd;
+ DAGOperand src2RC = src2VT.RegOpnd;
+ DAGOperand dstRC = dstVT.RegOpnd;
+ let AsmString = !if(!isa<pisaVoidVT>(dstVT),
+ name#" \t$src0, $src1, $src2;",
+ name#" \t$dst, $src0, $src1, $src2;");
+ let OutOperandList = !if(!isa<pisaVoidVT>(dstVT),
+ (outs),
+ (outs dstRC:$dst));
+}
+
+class PISAInst_rrr<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, src2RC:$src2);
+}
+
+class PISAInst_rri<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal2 = !if(!isa<pisaFloatVT>(src2VT), fpimm, imm);
+ DAGOperand immOpd2 = src2VT.ImmOpnd;
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, immOpd2:$src2);
+}
+
+class PISAInst_rir<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1, src2RC:$src2);
+}
+
+class PISAInst_rii<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ SDNode immVal2 = !if(!isa<pisaFloatVT>(src2VT), fpimm, imm);
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ DAGOperand immOpd2 = src2VT.ImmOpnd;
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1, immOpd2:$src2);
+}
+
+class PISAInst_irr<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1, src2RC:$src2);
+}
+
+class PISAInst_iri<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ SDNode immVal2 = !if(!isa<pisaFloatVT>(src2VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ DAGOperand immOpd2 = src2VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1, immOpd2:$src2);
+}
+
+class PISAInst_iir<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1, src2RC:$src2);
+}
+
+class PISAInst_iii<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT> : PISAInst3<name, dstVT, src0VT, src1VT, src2VT> {
+ SDNode immVal0 = !if(!isa<pisaFloatVT>(src0VT), fpimm, imm);
+ SDNode immVal1 = !if(!isa<pisaFloatVT>(src1VT), fpimm, imm);
+ SDNode immVal2 = !if(!isa<pisaFloatVT>(src2VT), fpimm, imm);
+ DAGOperand immOpd0 = src0VT.ImmOpnd;
+ DAGOperand immOpd1 = src1VT.ImmOpnd;
+ DAGOperand immOpd2 = src2VT.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1, immOpd2:$src2);
+}
+
+class PISAInst4<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT, pisaVT src3VT> : _PISAInst<name> {
+ DAGOperand src0RC = src0VT.RegOpnd;
+ DAGOperand src1RC = src1VT.RegOpnd;
+ DAGOperand src2RC = src2VT.RegOpnd;
+ DAGOperand src3RC = src3VT.RegOpnd;
+ DAGOperand dstRC = dstVT.RegOpnd;
+ let AsmString = !if(!isa<pisaVoidVT>(dstVT),
+ name#" \t$src0, $src1, $src2, $src3;",
+ name#" \t$dst, $src0, $src1, $src2, $src3;");
+ let OutOperandList = !if(!isa<pisaVoidVT>(dstVT),
+ (outs),
+ (outs dstRC:$dst));
+}
+
+class PISAInst_rrrr<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT, pisaVT src3VT> : PISAInst4<name, dstVT, src0VT, src1VT, src2VT, src3VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3);
+}
+
+class PISAInst5<string name, pisaVT dstVT, pisaVT src0VT, pisaVT src1VT, pisaVT src2VT, pisaVT src3VT, pisaVT src4VT> : _PISAInst<name> {
+ DAGOperand src0RC = src0VT.RegOpnd;
+ DAGOperand src1RC = src1VT.RegOpnd;
+ DAGOperand src2RC = src2VT.RegOpnd;
+ DAGOperand src3RC = src3VT.RegOpnd;
+ DAGOperand src4RC = src4VT.RegOpnd;
+ DAGOperand dstRC = dstVT.RegOpnd;
+ let AsmString = !if(!isa<pisaVoidVT>(dstVT),
+ name#" \t$src0, $src1, $src2, $src3, $src4;",
+ name#" \t$dst, $src0, $src1, $src2, $src3, $src4;");
+ let OutOperandList = !if(!isa<pisaVoidVT>(dstVT),
+ (outs),
+ (outs dstRC:$dst));
+}
+
+/****************************** End of PISAInst class hierarchy *****************************/
+
+/******************************* PISAMemInst class hierarchy ********************************/
+// +--------------------------------+
+// | _PISAInst: |
+// | defines Inst, Namespace, |
+// (Layer 1) | DecoderNamespace, |
+// | UseNamedOperandTable |
+// +--------------------------------+
+// | | |
+// +---------------+ | +------------------+
+// v v v
+// +--------------------------+ +--------------------------+ +--------------------------------+
+// | PISAStore: | | PISALoad: | | PISABinaryAtom: |
+// | defines InOperandList, | | defines InOperandList, | | PISATernaryAtom: |
+// | OutOperandList,| | OutOperandList,| | defines InOperandList, |
+// | AsmString, | | AsmString, | | OutOperandList, |
+// | Pattern | | Pattern | | AsmString, |
+// | | | | | Pattern |
+// +--------------------------+ +--------------------------+ +--------------------------------+
+//
+//
+// Description: This is the class hierarchy for PISAMemInst. The classes shown above define basic
+// structures of PISAMemInst for load, store and atomic operations, like inputs,
+// outputs, printing formats, patterns, etc. Each memory operation can have different
+// operand types for the addressing mode. We use MemProperty class to define
+// variants of formats for the operand types in a form of PISAMem_xx, where x
+// can be either v(Var), r(Reg), or i(Imm) for the corresponding operand.
+//
+
+def ADDR_rr : ComplexPattern<iPTR, 2, "SelectAddr_rr", [], [], 2>;
+def ADDR_ri : ComplexPattern<iPTR, 2, "SelectAddr_ri", [], [], 1>;
+
+class MEM_rr<AddressSpace AS> : Operand<i64> {
+ let MIOperandInfo = (ops AS.RC, AS.RC);
+ let PrintMethod = "printMemOperand";
+ let ParserMatchClass = PISAAsmOperand<"MemRR">;
+}
+
+class MEM_ri<AddressSpace AS> : Operand<i64> {
+ let MIOperandInfo = (ops AS.RC, i64imm);
+ let PrintMethod = "printMemOperand";
+ let ParserMatchClass = PISAAsmOperand<"MemRI">;
+}
+
+// A special memory operand for the kernel parameter *only*.
+def MEM_ii : Operand<i64>, ComplexPattern<i64, 2, "selectParamSlot_ii", [], []> {
+ let MIOperandInfo = (ops i32imm, i64imm);
+ let PrintMethod = "printParamMemOperand";
+ let ParserMatchClass = PISAAsmOperand<"MemII">;
+}
+
+def MEM_ir : Operand<i64>, ComplexPattern<i64, 2, "selectParamSlot_ir", [], []> {
+ let MIOperandInfo = (ops i32imm, Reg32b);
+ let PrintMethod = "printParamMemOperand";
+ let ParserMatchClass = PISAAsmOperand<"MemIR">;
+}
+
+def gi_ADDR_rr : GIComplexOperandMatcher<s64, "SelectAddr_rr">, GIComplexPatternEquiv<ADDR_rr>;
+def gi_ADDR_ri : GIComplexOperandMatcher<s64, "SelectAddr_ri">, GIComplexPatternEquiv<ADDR_ri>;
+def gi_PARAM_SLOT_ii : GIComplexOperandMatcher<s64, "selectParamSlot_ii">, GIComplexPatternEquiv<MEM_ii>;
+def gi_PARAM_SLOT_ir : GIComplexOperandMatcher<s64, "selectParamSlot_ir">, GIComplexPatternEquiv<MEM_ir>;
+
+class MemProperty<AddressSpace as, dag i, ComplexPattern pattern> {
+ dag Ins = i;
+ ComplexPattern Pattern = pattern;
+ AddressSpace AS = as;
+}
+
+class PISAMem_rr<AddressSpace AS> : MemProperty<AS, (ins MEM_rr<AS>:$addr), ADDR_rr>;
+class PISAMem_ri<AddressSpace AS> : MemProperty<AS, (ins MEM_ri<AS>:$addr), ADDR_ri>;
+
+// atomics syncscope("<target-scope>") definitions
+defvar AtomicScopeControlDefault = 255;
+defm AtomicScopeControl : EnumOptionOpndWithDefaultOpsDef<"AtomicScopeControl", false, AtomicScopeControlDefault>;
+class AtomicScopeControlEntry<string name, string mnemonic, bits<8> val>
+ : EnumOptionEntry<AtomicScopeControlClassID, name, val, "AtomicScopeControl", mnemonic>,
+ GenericEnum {
+ let FilterClass = "AtomicScopeControlEntry";
+}
+multiclass AtomicScopeControlDef<string mnemonic, bits<8> value> {
+ defvar entry = !if(!empty(mnemonic), "NONE", !toupper(!subst(".", "_", mnemonic)));
+ defvar EnumName = AtomicScopeControl.EnumPrefix # "_" # entry;
+ def EnumName : AtomicScopeControlEntry<entry, mnemonic, value>;
+}
+
+// Must be in sync with llvm::pisa::MemoryScope definitions in
+// llvm/include/llvm/IR/PISAIntrinsicUtils.h
+defm NONE : AtomicScopeControlDef<"", AtomicScopeControlDefault>;
+defm SYSTEM : AtomicScopeControlDef<"system", 0>;
+defm GPU : AtomicScopeControlDef<"gpu", 1>;
+defm WORKGROUP : AtomicScopeControlDef<"workgroup", 2>;
+
+// atomics cache control definitions
+defm AtomicCacheControl : EnumOptionOpndWithDefaultOpsDef<"AtomicCacheControl", false, 0>;
+class AtomicCacheControlEntry<string name, string mnemonic, bits<8> val>
+ : EnumOptionEntry<AtomicCacheControlClassID, name, val, "AtomicCacheControl", mnemonic>,
+ GenericEnum {
+ let FilterClass = "AtomicCacheControlEntry";
+}
+multiclass AtomicCacheControlDef<string mnemonic, bits<8> value> {
+ defvar entry = !if(!empty(mnemonic), "NONE", !toupper(!subst(".", "_", mnemonic)));
+ defvar EnumName = AtomicCacheControl.EnumPrefix # "_" # entry;
+ def EnumName : AtomicCacheControlEntry<entry, mnemonic, value>;
+}
+defm NONE : AtomicCacheControlDef<"", 0>;
+defm L1UC_L2UC_L3UC : AtomicCacheControlDef<"uc", 2>;
+defm L1UC_L2UC_L3WB : AtomicCacheControlDef<"L3wb", 3>;
+defm L1UC_L2WB_L3UC : AtomicCacheControlDef<"L2wb", 4>;
+
+// Fence memory ordering definitions (matches llvm::AtomicOrdering)
+defm FenceMemOrder : EnumOptionOpndWithDefaultOpsDef<"FenceMemOrder", false, 6>;
+class FenceMemOrderEntry<string name, string mnemonic, bits<8> val>
+ : EnumOptionEntry<FenceMemOrderClassID, name, val, "FenceMemOrder", mnemonic>,
+ GenericEnum {
+ let FilterClass = "FenceMemOrderEntry";
+}
+multiclass FenceMemOrderDef<string mnemonic, bits<8> value> {
+ defvar entry = !toupper(mnemonic);
+ defvar EnumName = FenceMemOrder.EnumPrefix # "_" # entry;
+ def EnumName : FenceMemOrderEntry<entry, mnemonic, value>;
+}
+defm ACQUIRE : FenceMemOrderDef<"acquire", 4>;
+defm RELEASE : FenceMemOrderDef<"release", 5>;
+defm ACQ_REL : FenceMemOrderDef<"acq_rel", 6>;
+defm SEQ_CST : FenceMemOrderDef<"seq_cst", 7>;
+
+
+// load cache control definitions
+defm LoadCacheControl : EnumOptionOpndWithDefaultOpsDef<"LoadCacheControl", false, 0>;
+class LoadCacheControlEntry<string name, string mnemonic, bits<8> val>
+ : EnumOptionEntry<LoadCacheControlClassID, name, val, "LoadCacheControl", mnemonic>,
+ GenericEnum {
+ let FilterClass = "LoadCacheControlEntry";
+}
+multiclass LoadCacheControlDef<string mnemonic, bits<8> value> {
+ defvar entry = !if(!empty(mnemonic), "NONE", !toupper(!subst(".", "_", mnemonic)));
+ defvar EnumName = LoadCacheControl.EnumPrefix # "_" # entry;
+ def EnumName : LoadCacheControlEntry<entry, mnemonic, value>;
+}
+defm NONE : LoadCacheControlDef<"", 0>;
+defm L1UC_L2UC_L3UC : LoadCacheControlDef<"L1uc.L2uc.L3uc", 2>;
+defm L1UC_L2UC_L3C : LoadCacheControlDef<"L1uc.L2uc.L3c", 3>;
+defm L1UC_L2C_L3UC : LoadCacheControlDef<"L1uc.L2c.L3uc", 4>;
+defm L1UC_L2C_L3C : LoadCacheControlDef<"L1uc.L2c.L3c", 5>;
+defm L1C_L2UC_L3UC : LoadCacheControlDef<"L1c.L2uc.L3uc", 6>;
+defm L1C_L2UC_L3C : LoadCacheControlDef<"L1c.L2uc.L3c", 7>;
+defm L1C_L2C_L3UC : LoadCacheControlDef<"L1c.L2c.L3uc", 8>;
+defm L1C_L2C_L3C : LoadCacheControlDef<"L1c.L2c.L3c", 9>;
+defm L1S_L2UC_L3UC : LoadCacheControlDef<"L1s.L2uc.L3uc", 10>;
+defm L1S_L2UC_L3C : LoadCacheControlDef<"L1s.L2uc.L3c", 11>;
+defm L1S_L2C_L3UC : LoadCacheControlDef<"L1s.L2c.L3uc", 12>;
+defm L1S_L2C_L3C : LoadCacheControlDef<"L1s.L2c.L3c", 13>;
+defm RI : LoadCacheControlDef<"ri", 14>;
+
+// store cache control definitions
+defm StoreCacheControl : EnumOptionOpndWithDefaultOpsDef<"StoreCacheControl", false, 0>;
+class StoreCacheControlEntry<string name, string mnemonic, bits<8> val>
+ : EnumOptionEntry<StoreCacheControlClassID, name, val, "StoreCacheControl", mnemonic>,
+ GenericEnum {
+ let FilterClass = "StoreCacheControlEntry";
+}
+multiclass StoreCacheControlDef<string mnemonic, bits<8> value> {
+ defvar entry = !if(!empty(mnemonic), "NONE", !toupper(!subst(".", "_", mnemonic)));
+ defvar EnumName = StoreCacheControl.EnumPrefix # "_" # entry;
+ def EnumName : StoreCacheControlEntry<entry, mnemonic, value>;
+}
+defm NONE : StoreCacheControlDef<"", 0>;
+defm L1UC_L2UC_L3UC : StoreCacheControlDef<"L1uc.L2uc.L3uc", 2>;
+defm L1UC_L2UC_L3WB : StoreCacheControlDef<"L1uc.L2uc.L3wb", 3>;
+defm L1UC_L2WB_L3UC : StoreCacheControlDef<"L1uc.L2wb.L3uc", 4>;
+defm L1UC_L2WB_L3WB : StoreCacheControlDef<"L1uc.L2wb.L3wb", 5>;
+defm L1WT_L2UC_L3UC : StoreCacheControlDef<"L1wt.L2uc.L3uc", 6>;
+defm L1WT_L2UC_L3WB : StoreCacheControlDef<"L1wt.L2uc.L3wb", 7>;
+defm L1WT_L2WB_L3UC : StoreCacheControlDef<"L1wt.L2wb.L3uc", 8>;
+defm L1WT_L2WB_L3WB : StoreCacheControlDef<"L1wt.L2wb.L3wb", 9>;
+defm L1S_L2UC_L3UC : StoreCacheControlDef<"L1s.L2uc.L3uc", 10>;
+defm L1S_L2UC_L3WB : StoreCacheControlDef<"L1s.L2uc.L3wb", 11>;
+defm L1S_L2WB_L3UC : StoreCacheControlDef<"L1s.L2wb.L3uc", 12>;
+defm L1WB_L2UC_L3UC : StoreCacheControlDef<"L1wb.L2uc.L3uc", 13>;
+defm L1WB_L2WB_L3UC : StoreCacheControlDef<"L1wb.L2wb.L3uc", 14>;
+defm L1WB_L2UC_L3WB : StoreCacheControlDef<"L1wb.L2uc.L3wb", 15>;
+
+
+// load instruction
+class PISALoad<string directive, AddressSpace AS, pisaVT VT, MemProperty M, PatFrag node> : _PISAInst<directive> {
+ DAGOperand dstRC = VT.RegOpnd;
+ dag OutOperandList = (outs dstRC:$dst);
+ dag InOperandList = !con(M.Ins, (ins LoadCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${cachehint}"#VT.mdirective#"\t $dst, $addr;";
+ list<dag> Pattern = [(set VT.VT:$dst, (LoadAddrspacePat<AS, node> M.Pattern:$addr))];
+}
+class PISAAtomicLoad<string directive, AddressSpace AS, pisaVT VT, MemProperty M, PatFrag node> : _PISAInst<directive> {
+ DAGOperand dstRC = VT.RegOpnd;
+ dag OutOperandList = (outs dstRC:$dst);
+ dag InOperandList = !con(M.Ins, (ins AtomicScopeControlOpnd:$scope, AtomicCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${scope}${cachehint}"#VT.mdirective#"\t $dst, $addr;";
+ list<dag> Pattern = [(set VT.VT:$dst, (LoadAddrspacePat<AS, node> M.Pattern:$addr))];
+}
+
+// store instruction
+class PISAStore<string directive, AddressSpace AS, pisaVT VT, MemProperty M, PatFrag node> : _PISAInst<directive> {
+ DAGOperand srcRC = VT.RegOpnd;
+ dag OutOperandList = (outs);
+ dag InOperandList = !con(M.Ins, (ins srcRC:$data, StoreCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${cachehint}"#VT.mdirective#"\t $addr, $data;";
+ list<dag> Pattern = [(set (StoreAddrspacePat<AS, node> VT.VT:$data, M.Pattern:$addr))];
+}
+class PISAAtomicStore<string directive, AddressSpace AS, pisaVT VT, MemProperty M, PatFrag node> : _PISAInst<directive> {
+ DAGOperand srcRC = VT.RegOpnd;
+ dag OutOperandList = (outs);
+ dag InOperandList = !con(M.Ins, (ins srcRC:$data, AtomicScopeControlOpnd:$scope, AtomicCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${scope}${cachehint}"#VT.mdirective#"\t $addr, $data;";
+ list<dag> Pattern = [(set (StoreAddrspacePat<AS, node> VT.VT:$data, M.Pattern:$addr))];
+}
+
+
+// The definitions of $dst, $src0 and $src1 are defined in the PISA SPEC
+// https://intel.github.io/pisa/instructions_memory.html#iatom
+class PISAUnaryAtom<string directive, string op, AddressSpace AS, pisaVT VT,
+ MemProperty M, PatFrag node> : _PISAInst<directive#"."#op#VT.directive> {
+ DAGOperand dstRC = VT.RegOpnd;
+ dag OutOperandList = (outs dstRC:$dst);
+ dag InOperandList = !con(M.Ins, (ins AtomicScopeControlOpnd:$scope, AtomicCacheControlOpnd:$cachehint));
+ string AsmString =
+ directive#"${scope}${cachehint}."#op#VT.directive#"\t $dst, $addr;";
+ list<dag> Pattern = [(set VT.VT:$dst,
+ (UnaryAtomAddrspacePat<AS, node> M.Pattern:$addr))];
+}
+
+class PISABinaryAtom<string directive, string op, AddressSpace AS, pisaVT VT,
+ MemProperty M, PatFrag node> : _PISAInst<directive#"."#op#VT.directive> {
+ DAGOperand dstRC = VT.RegOpnd;
+ DAGOperand srcRC = VT.RegOpnd;
+ dag OutOperandList = (outs dstRC:$dst);
+ dag InOperandList = !con(M.Ins, (ins srcRC:$src0, AtomicScopeControlOpnd:$scope, AtomicCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${scope}${cachehint}."#op#VT.directive#"\t $dst, $addr, $src0;";
+ list<dag> Pattern = [(set VT.VT:$dst,
+ (BinaryAtomAddrspacePat<AS, node> M.Pattern:$addr, VT.VT:$src0))];
+}
+
+// cmpxchg instruction
+class PISATernaryAtom<string directive, string op, AddressSpace AS, pisaVT VT,
+ MemProperty M, SDPatternOperator node> : _PISAInst<directive#"."#op#VT.directive> {
+ DAGOperand dstRC = VT.RegOpnd;
+ DAGOperand srcRC = VT.RegOpnd;
+ dag OutOperandList = (outs dstRC:$dst);
+ dag InOperandList = !con(M.Ins, (ins srcRC:$src0, srcRC:$src1, AtomicScopeControlOpnd:$scope, AtomicCacheControlOpnd:$cachehint));
+ string AsmString = directive#"${scope}${cachehint}."#op#VT.directive#"\t $dst, $addr, $src0, $src1;";
+ list<dag> Pattern = [(set VT.VT:$dst, (TernaryAtomAddrspacePat<AS, node> M.Pattern:$addr, VT.VT:$src0, VT.VT:$src1))];
+}
+
+// Remap to PISA[MemOp]_xx for readability, where x represents either r(Reg), or i(Imm) for the corresponding operand
+class PISALoad_rr <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISALoad <d, AS, VT, PISAMem_rr<AS>, node>;
+class PISAAtomicLoad_rr <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAAtomicLoad <d, AS, VT, PISAMem_rr<AS>, node>;
+class PISAStore_rr <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAStore <d, AS, VT, PISAMem_rr<AS>, node>;
+class PISAAtomicStore_rr <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAAtomicStore <d, AS, VT, PISAMem_rr<AS>, node>;
+class PISAUnaryAtom_rr<string d, string op, AddressSpace AS, pisaVT VT,
+ PatFrag node>
+ : PISAUnaryAtom<d, op, AS, VT, PISAMem_rr<AS>, node>;
+class PISABinaryAtom_rr <string d, string op, AddressSpace AS, pisaVT VT, PatFrag node>
+ : PISABinaryAtom <d, op, AS, VT, PISAMem_rr<AS>, node>;
+class PISATernaryAtom_rr<string d, string op, AddressSpace AS, pisaVT VT, SDPatternOperator node>
+ : PISATernaryAtom<d, op, AS, VT, PISAMem_rr<AS>, node>;
+
+class PISALoad_ri <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISALoad <d, AS, VT, PISAMem_ri<AS>, node>;
+class PISAAtomicLoad_ri <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAAtomicLoad <d, AS, VT, PISAMem_ri<AS>, node>;
+class PISAStore_ri <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAStore <d, AS, VT, PISAMem_ri<AS>, node>;
+class PISAAtomicStore_ri <string d, AddressSpace AS, pisaVT VT, PatFrag node> : PISAAtomicStore <d, AS, VT, PISAMem_ri<AS>, node>;
+class PISAUnaryAtom_ri<string d, string op, AddressSpace AS, pisaVT VT,
+ PatFrag node>
+ : PISAUnaryAtom<d, op, AS, VT, PISAMem_ri<AS>, node>;
+class PISABinaryAtom_ri <string d, string op, AddressSpace AS, pisaVT VT, PatFrag node>
+ : PISABinaryAtom <d, op, AS, VT, PISAMem_ri<AS>, node>;
+class PISATernaryAtom_ri<string d, string op, AddressSpace AS, pisaVT VT, SDPatternOperator node>
+ : PISATernaryAtom<d, op, AS, VT, PISAMem_ri<AS>, node>;
+
+
+/**************************** End of PISAMemInst class hierarchy ****************************/
diff --git a/llvm/lib/Target/PISA/PISAInstrInfo.cpp b/llvm/lib/Target/PISA/PISAInstrInfo.cpp
index c20e1a022db71..d9d219aa1c516 100644
--- a/llvm/lib/Target/PISA/PISAInstrInfo.cpp
+++ b/llvm/lib/Target/PISA/PISAInstrInfo.cpp
@@ -9,11 +9,458 @@
#include "PISAInstrInfo.h"
#include "PISA.h"
#include "PISASubtarget.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/Support/ErrorHandling.h"
#define GET_INSTRINFO_CTOR_DTOR
#include "PISAGenInstrInfo.inc"
using namespace llvm;
+using namespace PISA;
PISAInstrInfo::PISAInstrInfo(const PISASubtarget &STI)
- : PISAGenInstrInfo(STI, RI), RI() {}
+ : PISAGenInstrInfo(STI, RI) {}
+
+bool PISAInstrInfo::isNoEmissionInstr(const MachineInstr &MI) const {
+ // Functions are emitted to C-style function signature.
+ // These instructions are not required to be in the output.
+ return isFunctionParamInstr(MI);
+}
+
+bool PISAInstrInfo::isFunctionParamInstr(const MachineInstr &MI) const {
+ switch (MI.getOpcode()) {
+ case PISA::functionParameter_i8:
+ case PISA::functionParameter_i16:
+ case PISA::functionParameter_i32:
+ case PISA::functionParameter_i64:
+ case PISA::functionParameter_v2i8:
+ case PISA::functionParameter_v3i8:
+ case PISA::functionParameter_v4i8:
+ case PISA::functionParameter_v2i16:
+ case PISA::functionParameter_v3i16:
+ case PISA::functionParameter_v4i16:
+ case PISA::functionParameter_v2i32:
+ case PISA::functionParameter_v3i32:
+ case PISA::functionParameter_v4i32:
+ case PISA::functionParameter_v5i32:
+ case PISA::functionParameter_v6i32:
+ case PISA::functionParameter_v7i32:
+ case PISA::functionParameter_v8i32:
+ case PISA::functionParameter_v16i32:
+ case PISA::functionParameter_v32i32:
+ case PISA::functionParameter_v64i32:
+ case PISA::functionParameter_v2i64:
+ case PISA::functionParameter_v3i64:
+ case PISA::functionParameter_v4i64:
+ return true;
+ default:
+ return false;
+ }
+}
+
+namespace llvm {
+namespace PISA {
+static const MachineOperand &getMO(const MachineInstr &MI, PISA::OpName Name) {
+ int16_t Idx = getNamedOperandIdx(MI.getOpcode(), Name);
+ assert(Idx >= 0 && "name not present!");
+
+ return MI.getOperand(Idx);
+}
+} // namespace PISA
+} // namespace llvm
+
+// See description in TargetInstrInfo.h
+bool PISAInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool /*AllowModify*/) const {
+ // If the block has no terminators, it just falls into the block after it.
+ MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+ if (I == MBB.end() || !isUnpredicatedTerminator(*I))
+ return false;
+
+ // Get the last instruction in the block.
+ MachineInstr *LastInst = &*I;
+
+ // If there is only one terminator instruction, process it.
+ if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
+ if (LastInst->isUnconditionalBranch()) {
+ if (LastInst->getOpcode() != PISA::gotolabel)
+ return true;
+ TBB = LastInst->getOperand(0).getMBB();
+ return false;
+ }
+ if (LastInst->isConditionalBranch()) {
+ if (LastInst->getOpcode() != PISA::predgoto)
+ return true;
+ // Block ends with fall-through condbranch.
+ TBB = getMO(*LastInst, OpName::label).getMBB();
+ // mod, pred
+ Cond.push_back(LastInst->getOperand(0));
+ Cond.push_back(LastInst->getOperand(1));
+ return false;
+ }
+ return true; // Can't handle indirect branch.
+ }
+
+ // Get the instruction before it if it is a terminator.
+ MachineInstr *SecondLastInst = &*I;
+
+ if (!SecondLastInst->isConditionalBranch() ||
+ !LastInst->isUnconditionalBranch() ||
+ // triple terminator?
+ (I != MBB.begin() && isUnpredicatedTerminator(*--I)))
+ return true;
+
+ if (SecondLastInst->getOpcode() != PISA::predgoto ||
+ LastInst->getOpcode() != PISA::gotolabel)
+ return true;
+
+ TBB = getMO(*SecondLastInst, OpName::label).getMBB();
+ FBB = LastInst->getOperand(0).getMBB();
+
+ // mod, pred
+ Cond.push_back(SecondLastInst->getOperand(0));
+ Cond.push_back(SecondLastInst->getOperand(1));
+
+ return false;
+}
+
+// See description in TargetInstrInfo.h
+unsigned PISAInstrInfo::removeBranch(MachineBasicBlock &MBB,
+ int *BytesRemoved) const {
+ assert(!BytesRemoved && "not supported!");
+
+ unsigned Count = 0;
+ for (auto &MI : llvm::make_early_inc_range(MBB.terminators())) {
+ assert(MI.isBranch() && "not a branch?");
+ MI.eraseFromParent();
+ Count++;
+ }
+
+ return Count;
+}
+
+// See description in TargetInstrInfo.h
+unsigned PISAInstrInfo::insertBranch(
+ MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
+
+ assert(!BytesAdded && "not supported!");
+ assert(TBB && "Should have at least one branch!");
+
+ if (!FBB) {
+ if (Cond.empty()) { // Unconditional branch
+ BuildMI(&MBB, DL, get(PISA::gotolabel)).addMBB(TBB);
+ } else { // Conditional branch
+ assert(Cond.size() == 2 && "wrong number of args?");
+ BuildMI(&MBB, DL, get(PISA::predgoto))
+ .add(Cond[0])
+ .add(Cond[1])
+ .addMBB(TBB);
+ }
+ return 1;
+ }
+
+ // Two-way Conditional Branch.
+ assert(Cond.size() == 2 && "wrong number of args?");
+ BuildMI(&MBB, DL, get(PISA::predgoto)).add(Cond[0]).add(Cond[1]).addMBB(TBB);
+ BuildMI(&MBB, DL, get(PISA::gotolabel)).addMBB(FBB);
+ return 2;
+}
+
+bool PISAInstrInfo::reverseBranchCondition(
+ SmallVectorImpl<MachineOperand> &Cond) const {
+ if (Cond.empty())
+ return true;
+
+ uint64_t DoNegate = Cond[0].getImm();
+ Cond[0].setImm(!DoNegate);
+
+ return false;
+}
+
+// We have to override this because LowerCopy() in ExpandPostRAPseudos
+// gets rid of identity copies without checking subregs. It doesn't check
+// subregs because register allocation has already happened (but not for PISA)
+// so there aren't any subregs.
+bool PISAInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
+ if (!MI.isCopy())
+ return false;
+
+ MachineOperand &DstMO = MI.getOperand(0);
+ MachineOperand &SrcMO = MI.getOperand(1);
+
+ // only generate instructions using defined registers
+ if (!SrcMO.isUndef())
+ copyPhysReg(*MI.getParent(), MI, MI.getDebugLoc(), DstMO.getReg(),
+ SrcMO.getReg(), SrcMO.isKill());
+
+ MI.eraseFromParent();
+
+ return true;
+}
+
+bool PISAInstrInfo::isSafeToMove(const MachineInstr &MI,
+ const MachineBasicBlock *MBB,
+ const MachineFunction &MF) const {
+ // Convergent instructions must not be moved across basic blocks because
+ // doing so can change the set of threads executing the instruction,
+ // potentially producing incorrect results.
+ if (MI.isConvergent())
+ return false;
+ return true;
+}
+
+void PISAInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ const DebugLoc &DL, Register DestReg,
+ Register SrcReg, bool KillSrc,
+ bool RenamableDest, bool RenamableSrc) const {
+
+ assert(I->isCopy() && "Copy instruction is expected");
+ auto &MRI = I->getMF()->getRegInfo();
+
+ auto GetSubregRc = [&](MachineOperand &MO) -> const TargetRegisterClass * {
+ unsigned Subreg = MO.getSubReg();
+ Register Reg = MO.getReg();
+ auto *SuperRC = Reg.isPhysical() ? RI.getMinimalPhysRegClass(Reg)
+ : MRI.getRegClass(Reg);
+ if (Subreg == 0)
+ return SuperRC;
+
+ return RI.getSubRegisterClass(SuperRC, Subreg);
+ };
+
+ auto &DstOp = I->getOperand(0);
+ auto &SrcOp = I->getOperand(1);
+ auto *DstSubRC = GetSubregRc(DstOp);
+ auto *SrcSubRC = GetSubregRc(SrcOp);
+
+ unsigned Op = 0;
+
+ const unsigned DstSubNumElts = RI.getNumEltsFromRegClass(DstSubRC);
+ const unsigned DstSubEltSize = RI.getBitSizeFromRegClass(DstSubRC);
+ const unsigned SrcSubNumElts = RI.getNumEltsFromRegClass(SrcSubRC);
+ const unsigned SrcSubEltSize = RI.getBitSizeFromRegClass(SrcSubRC);
+ const bool DstSubIsVector = DstSubNumElts > 1;
+ const bool SrcSubIsVector = SrcSubNumElts > 1;
+ const bool DstSubIsScalar = !DstSubIsVector;
+ const bool SrcSubIsScalar = !SrcSubIsVector;
+
+ if (DstSubIsVector && SrcSubIsVector) {
+ if (DstSubNumElts == 4 && DstSubEltSize == 8 && SrcSubNumElts == 2 &&
+ SrcSubEltSize == 16)
+ Op = PISA::mov_v4i8_v2i16_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 16 && SrcSubNumElts == 4 &&
+ SrcSubEltSize == 8)
+ Op = PISA::mov_v2i16_v4i8_r;
+ else if (DstSubNumElts == 4 && DstSubEltSize == 16 && SrcSubNumElts == 2 &&
+ SrcSubEltSize == 32)
+ Op = PISA::mov_v4i16_v2i32_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 32 && SrcSubNumElts == 4 &&
+ SrcSubEltSize == 16)
+ Op = PISA::mov_v2i32_v4i16_r;
+ else if (DstSubNumElts == 4 && DstSubEltSize == 32 && SrcSubNumElts == 2 &&
+ SrcSubEltSize == 64)
+ Op = PISA::mov_v4i32_v2i64_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 64 && SrcSubNumElts == 4 &&
+ SrcSubEltSize == 32)
+ Op = PISA::mov_v2i64_v4i32_r;
+ else {
+ assert(DstSubNumElts == SrcSubNumElts && "num elts mismatch!");
+ if (DstSubNumElts == 2) {
+ switch (DstSubEltSize) {
+ case 8:
+ Op = PISA::mov_v2i8_r;
+ break;
+ case 16:
+ Op = PISA::mov_v2i16_r;
+ break;
+ case 32:
+ Op = PISA::mov_v2i32_r;
+ break;
+ case 64:
+ Op = PISA::mov_v2i64_r;
+ break;
+ default:
+ llvm_unreachable("unknown elt size!");
+ }
+ } else if (DstSubNumElts == 3) {
+ switch (DstSubEltSize) {
+ case 8:
+ Op = PISA::mov_v3i8_r;
+ break;
+ case 16:
+ Op = PISA::mov_v3i16_r;
+ break;
+ default:
+ llvm_unreachable("unknown elt size!");
+ }
+ } else if (DstSubNumElts == 4) {
+ switch (DstSubEltSize) {
+ case 8:
+ Op = PISA::mov_v4i8_r;
+ break;
+ case 16:
+ Op = PISA::mov_v4i16_r;
+ break;
+ case 32:
+ Op = PISA::mov_v4i32_r;
+ break;
+ default:
+ llvm_unreachable("unknown elt size!");
+ }
+ } else {
+ llvm_unreachable("unknown vec size!");
+ }
+ }
+ } else if (DstSubIsVector && SrcSubIsScalar) {
+ if (DstSubNumElts == 2 && DstSubEltSize == 8 && SrcSubEltSize == 16)
+ Op = PISA::mov_v2i8_i16_r;
+ else if (DstSubNumElts == 4 && DstSubEltSize == 8 && SrcSubEltSize == 32)
+ Op = PISA::mov_v4i8_i32_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 16 && SrcSubEltSize == 32)
+ Op = PISA::mov_v2i16_i32_r;
+ else if (DstSubNumElts == 4 && DstSubEltSize == 16 && SrcSubEltSize == 64)
+ Op = PISA::mov_v4i16_i64_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 32 && SrcSubEltSize == 64)
+ Op = PISA::mov_v2i32_i64_r;
+ else if (DstSubNumElts == 4 && DstSubEltSize == 32 && SrcSubEltSize == 128)
+ Op = PISA::mov_v4i32_i128_r;
+ else if (DstSubNumElts == 2 && DstSubEltSize == 64 && SrcSubEltSize == 128)
+ Op = PISA::mov_v2i64_i128_r;
+ else
+ llvm_unreachable("wrong copy instruction");
+ } else if (DstSubIsScalar && SrcSubIsVector) {
+ if (DstSubEltSize == 16 && SrcSubNumElts == 2 && SrcSubEltSize == 8)
+ Op = PISA::mov_i16_v2i8_r;
+ else if (DstSubEltSize == 32 && SrcSubNumElts == 2 && SrcSubEltSize == 16)
+ Op = PISA::mov_i32_v2i16_r;
+ else if (DstSubEltSize == 32 && SrcSubNumElts == 4 && SrcSubEltSize == 8)
+ Op = PISA::mov_i32_v4i8_r;
+ else if (DstSubEltSize == 64 && SrcSubNumElts == 2 && SrcSubEltSize == 32)
+ Op = PISA::mov_i64_v2i32_r;
+ else if (DstSubEltSize == 64 && SrcSubNumElts == 4 && SrcSubEltSize == 16)
+ Op = PISA::mov_i64_v4i16_r;
+ else if (DstSubEltSize == 128 && SrcSubNumElts == 2 && SrcSubEltSize == 64)
+ Op = PISA::mov_i128_v2i64_r;
+ else if (DstSubEltSize == 128 && SrcSubNumElts == 4 && SrcSubEltSize == 32)
+ Op = PISA::mov_i128_v4i32_r;
+ else
+ llvm_unreachable("wrong copy operation");
+ } else if (DstSubIsScalar && SrcSubIsScalar) {
+ const auto *TRI = static_cast<const PISARegisterInfo *>(
+ I->getMF()->getSubtarget().getRegisterInfo());
+ auto I16 = LLT::integer(16);
+
+ if ((DstSubEltSize == 1) && (SrcSubEltSize == 1)) {
+ // sel.16b %tmp, 1, 0, %p_in
+ // ucmp.ne.16b %p_out, %tmp, 0
+ auto TmpReg = MRI.createGenericVirtualRegister(I16);
+ MRI.setRegClass(TmpReg, TRI->getRegClassFromLLT(I16));
+ Op = PISA::sel_16_iip;
+ BuildMI(MBB, I, DL, get(Op))
+ .addDef(TmpReg)
+ .addReg(SrcOp.getReg())
+ .addImm(1)
+ .addImm(0);
+ Op = PISA::ucmp_ne_16b_pri;
+ BuildMI(MBB, I, DL, get(Op))
+ .addDef(DstOp.getReg())
+ .addReg(TmpReg)
+ .addImm(0);
+ return;
+ }
+ if (DstSubEltSize == 1) {
+ // ucmp.ne.??b %p, %src, 0
+ auto TmpReg = SrcOp.getReg();
+ if (SrcSubEltSize == 8) {
+ TmpReg = MRI.createGenericVirtualRegister(I16);
+ MRI.setRegClass(TmpReg, TRI->getRegClassFromLLT(I16));
+ Op = PISA::zext_16b_8b_r;
+ BuildMI(MBB, I, DL, get(Op)).addDef(TmpReg).addReg(SrcOp.getReg());
+ }
+ switch (SrcSubEltSize) {
+ default:
+ llvm_unreachable("unsupported source size!");
+ case 8:
+ case 16:
+ Op = PISA::ucmp_ne_16b_pri;
+ break;
+ case 32:
+ Op = PISA::ucmp_ne_32b_pri;
+ break;
+ case 64:
+ Op = PISA::ucmp_ne_64b_pri;
+ break;
+ }
+ BuildMI(MBB, I, DL, get(Op))
+ .addDef(DstOp.getReg())
+ .addReg(TmpReg)
+ .addImm(0);
+ return;
+ }
+ if (SrcSubEltSize == 1) {
+ // sel.??b %dst, 1, 0, %p
+ auto TmpReg = DstOp.getReg();
+ if (DstSubEltSize == 8) {
+ TmpReg = MRI.createGenericVirtualRegister(I16);
+ MRI.setRegClass(TmpReg, TRI->getRegClassFromLLT(I16));
+ }
+ switch (DstSubEltSize) {
+ default:
+ llvm_unreachable("unsupported destination size!");
+ case 8:
+ case 16:
+ Op = PISA::sel_16_iip;
+ break;
+ case 32:
+ Op = PISA::sel_32_iip;
+ break;
+ case 64:
+ Op = PISA::sel_64_iip;
+ break;
+ }
+ BuildMI(MBB, I, DL, get(Op))
+ .addDef(TmpReg)
+ .addReg(SrcOp.getReg())
+ .addImm(1)
+ .addImm(0);
+ if (DstSubEltSize == 8) {
+ Op = PISA::trunc_8b_16b_r;
+ BuildMI(MBB, I, DL, get(Op)).addDef(DstOp.getReg()).addReg(TmpReg);
+ }
+ return;
+ }
+ switch (DstSubEltSize) {
+ case 8:
+ Op = PISA::mov_i8_r;
+ break;
+ case 16:
+ Op = PISA::mov_i16_r;
+ break;
+ case 32:
+ Op = PISA::mov_i32_r;
+ break;
+ case 64:
+ Op = PISA::mov_i64_r;
+ break;
+ default:
+ llvm_unreachable("unknown elt size!");
+ }
+ }
+
+ unsigned DstSubreg = DstOp.getSubReg();
+ unsigned SrcSubreg = SrcOp.getSubReg();
+
+ RegState Flags = DstOp.isUndef() ? RegState::Undef : RegState::NoFlags;
+ BuildMI(MBB, I, DL, get(Op))
+ .addDef(DstOp.getReg(), Flags, DstSubreg)
+ .addReg(SrcOp.getReg(), getKillRegState(KillSrc), SrcSubreg);
+}
diff --git a/llvm/lib/Target/PISA/PISAInstrInfo.h b/llvm/lib/Target/PISA/PISAInstrInfo.h
index 0b411221edf54..4e73ee6e19977 100644
--- a/llvm/lib/Target/PISA/PISAInstrInfo.h
+++ b/llvm/lib/Target/PISA/PISAInstrInfo.h
@@ -8,13 +8,15 @@
#ifndef LLVM_LIB_TARGET_PISA_PISAINSTRINFO_H
#define LLVM_LIB_TARGET_PISA_PISAINSTRINFO_H
-
#include "PISARegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#define GET_INSTRINFO_HEADER
#include "PISAGenInstrInfo.inc"
+#define GET_INSTRINFO_OPERAND_ENUM
+#include "PISAGenInstrInfo.inc"
+
namespace llvm {
class PISASubtarget;
@@ -25,6 +27,34 @@ class PISAInstrInfo : public PISAGenInstrInfo {
PISAInstrInfo(const PISASubtarget &STI);
const PISARegisterInfo &getRegisterInfo() const { return RI; }
+ bool isNoEmissionInstr(const MachineInstr &MI) const;
+ bool isFunctionParamInstr(const MachineInstr &MI) const;
+
+ bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool AllowModify = false) const override;
+
+ unsigned removeBranch(MachineBasicBlock &MBB,
+ int *BytesRemoved = nullptr) const override;
+
+ unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
+ MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
+ const DebugLoc &DL,
+ int *BytesAdded = nullptr) const override;
+
+ bool
+ reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
+
+ bool expandPostRAPseudo(MachineInstr &MI) const override;
+
+ bool isSafeToMove(const MachineInstr &MI, const MachineBasicBlock *MBB,
+ const MachineFunction &MF) const override;
+
+ void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
+ const DebugLoc &DL, Register DestReg, Register SrcReg,
+ bool KillSrc, bool RenamableDest = false,
+ bool RenamableSrc = false) const override;
};
} // namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAInstrInfo.td b/llvm/lib/Target/PISA/PISAInstrInfo.td
index de93f39b2dc01..154e1d8c9c6ea 100644
--- a/llvm/lib/Target/PISA/PISAInstrInfo.td
+++ b/llvm/lib/Target/PISA/PISAInstrInfo.td
@@ -1,4 +1,4 @@
-//===-- PISAInstrInfo.td - PISA Instruction defs ----------*- tablegen -*-===//
+//===-- PISAInstrInfo.td - PISA Instruction defs -----------*- tablegen -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,2515 @@
//
//===----------------------------------------------------------------------===//
-class PISAInst<dag outs, dag ins, string asmstr, list<dag> pattern>
- : Instruction {
- let Namespace = "PISA";
- dag OutOperandList = outs;
- dag InOperandList = ins;
- let AsmString = asmstr;
- let Pattern = pattern;
- let hasSideEffects = 0;
+include "PISAInstrFormats.td"
+
+// Templates for imm type checks
+class uimmCheck<int num, ValueType VT>
+ : ImmLeaf<VT, "return isUInt<"#num#">(Imm);">;
+class simmCheck<int num, ValueType VT>
+ : ImmLeaf<VT, "return isInt<"#num#">(Imm);">;
+class immType<bit isSigned, ValueType VT, ValueType extVT = VT> {
+ ImmLeaf check = !if(isSigned, simmCheck<VT.Size, extVT>,
+ uimmCheck<VT.Size, extVT>);
+}
+
+// 'afn' specified for call
+class fastunaryop<SDPatternOperator nodeop> :
+ PatFrag<(ops node:$src), (nodeop node:$src), [{ /* empty */ }]> {
+ let GISelPredicateCode = [{ return useFastFP(MI); }];
+}
+// 'arcp' specified for call, used only for division
+class fastbinop<SDPatternOperator nodeop> :
+ PatFrag<(ops node:$lhs, node:$rhs), (nodeop node:$lhs, node:$rhs), [{ /* empty */ }]> {
+ let GISelPredicateCode = [{ return MI.getFlag(MachineInstr::FmArcp); }];
+}
+
+// operand handling
+class PrintDispatch<string PM, ValueType Ty, string OpType> : OperandWithDefaultOps<Ty, (ops (Ty 0))> {
+ let PrintMethod = PM;
+ let OperandNamespace = "PISA";
+ let OperandType = OpType;
+}
+def Negate : PrintDispatch<"negateRepr", i32, "OPERAND_NEGATE"> {
+ let ParserMatchClass = PISAAsmOperand<"NegateOpnd">;
+}
+
+// call target
+def FunctionCallTargetAsmOperand : AsmOperandClass {
+ let Name = "FunctionCallTargetOpnd";
+ let ParserMethod = "parseFunctionCallTargetOpnd";
+}
+// ... register or globalVar
+def FunctionCallTargetOpnd : Operand<i64> {
+ let PrintMethod="printFunctionCallTargetOpnd";
+ let ParserMatchClass = FunctionCallTargetAsmOperand;
+}
+// ... register
+def IndirectFunctionCallTargetOpnd : PISARegisterOperand<Reg64b> {
+ let PrintMethod="printFunctionCallTargetOpnd";
+}
+
+
+// mapping between LLVM and PISA names
+class AtomicMemOrder<string MO> {
+ string name = !if(!eq(MO,"monotonic"), "relaxed", MO);
+}
+
+// ftz modifier
+let RecomputePerFunction = 1 in {
+ def useFTZ : Predicate<"Subtarget->getTargetLowering()->useFTZ(*MF)">;
+ def noFTZ : Predicate<"!Subtarget->getTargetLowering()->useFTZ(*MF)">;
+}
+class FTZModifier<string ftzName, string ftzOp, Predicate ftzPred> {
+ string name = ftzName;
+ string op = ftzOp;
+ Predicate pred = ftzPred;
+}
+def noneFTZMod : FTZModifier<"", "", Predicate<"">>;
+def omitFTZMod : FTZModifier<"", "", noFTZ>;
+def usesFTZMod : FTZModifier<".ftz", "_ftz", useFTZ>;
+
+// saturation modifier
+def timm_false : TImmLeaf<i1, [{ return ((uint64_t)Imm) == 0; }]>;
+def timm_true : TImmLeaf<i1, [{ return ((uint64_t)Imm) == (uint64_t)-1; }]>;
+class SaturationModifier<string satName, string satOp, TImmLeaf satNode> {
+ string name = satName;
+ string op = satOp;
+ TImmLeaf node = satNode;
+}
+def NoSatMod : SaturationModifier<"", "", timm_false>;
+def DsatMod : SaturationModifier<".dsat", "_sat", timm_true>;
+
+// NaN-propagation modifier
+class NanPropagateModifier<string nanpName, TImmLeaf nanpNode> {
+ string name = nanpName;
+ TImmLeaf node = nanpNode;
+}
+def NoNanpMod : NanPropagateModifier<"", timm_false>;
+def NanpMod : NanPropagateModifier<".nanp", timm_true>;
+defvar AllNanpMods = [NoNanpMod, NanpMod];
+
+// wrapper of TImmLeaf with instruction qualifier name
+class NamedTImmLeaf<string _name, ValueType vt, code pred> : TImmLeaf<vt, pred> {
+ string name = _name;
+}
+
+// ired operation types (PISAIntrinsicUtils.h)
+def IRedOp_SUM_timm : NamedTImmLeaf<".sum", i8, [{ return Imm == pisa::IRedOp::SUM; }]>;
+def IRedOp_SMIN_timm : NamedTImmLeaf<".smin", i8, [{ return Imm == pisa::IRedOp::SMIN; }]>;
+def IRedOp_SMAX_timm : NamedTImmLeaf<".smax", i8, [{ return Imm == pisa::IRedOp::SMAX; }]>;
+def IRedOp_UMIN_timm : NamedTImmLeaf<".umin", i8, [{ return Imm == pisa::IRedOp::UMIN; }]>;
+def IRedOp_UMAX_timm : NamedTImmLeaf<".umax", i8, [{ return Imm == pisa::IRedOp::UMAX; }]>;
+def IRedOp_AND_timm : NamedTImmLeaf<".and", i8, [{ return Imm == pisa::IRedOp::AND; }]>;
+def IRedOp_OR_timm : NamedTImmLeaf<".or", i8, [{ return Imm == pisa::IRedOp::OR; }]>;
+def IRedOp_XOR_timm : NamedTImmLeaf<".xor", i8, [{ return Imm == pisa::IRedOp::XOR; }]>;
+def IRedOp_ABSMAX_timm : NamedTImmLeaf<".absmax", i8, [{ return Imm == pisa::IRedOp::ABSMAX; }]>;
+defvar IRedOpNodes = [IRedOp_SUM_timm, IRedOp_SMIN_timm, IRedOp_SMAX_timm,
+ IRedOp_UMIN_timm, IRedOp_UMAX_timm, IRedOp_AND_timm,
+ IRedOp_OR_timm, IRedOp_XOR_timm, IRedOp_ABSMAX_timm];
+
+// fred operation types (PISAIntrinsicUtils.h)
+def FRedOp_MIN_timm : NamedTImmLeaf<".min", i8, [{ return Imm == pisa::FRedOp::MIN; }]>;
+def FRedOp_MAX_timm : NamedTImmLeaf<".max", i8, [{ return Imm == pisa::FRedOp::MAX; }]>;
+def FRedOp_ABSMAX_timm : NamedTImmLeaf<".absmax", i8, [{ return Imm == pisa::FRedOp::ABSMAX; }]>;
+defvar FRedOpNodes = [FRedOp_MIN_timm, FRedOp_MAX_timm, FRedOp_ABSMAX_timm];
+
+
+// shfl mode types (PISAIntrinsicUtils.h)
+def SHFLMode_UP_timm : NamedTImmLeaf<".up", i8, [{ return Imm == pisa::SHFLMode::UP; }]>;
+def SHFLMode_DOWN_timm : NamedTImmLeaf<".dn", i8, [{ return Imm == pisa::SHFLMode::DOWN; }]>;
+def SHFLMode_XOR_timm : NamedTImmLeaf<".xor", i8, [{ return Imm == pisa::SHFLMode::XOR; }]>;
+def SHFLMode_IDX_timm : NamedTImmLeaf<".idx", i8, [{ return Imm == pisa::SHFLMode::IDX; }]>;
+defvar SHFLModeNodes = [SHFLMode_UP_timm, SHFLMode_DOWN_timm, SHFLMode_XOR_timm, SHFLMode_IDX_timm];
+
+
+// memory ordering types (AtomicOrdering.h)
+def MemoryOrder_relaxed_timm : NamedTImmLeaf<".relaxed", i8, [{ return Imm == (unsigned)llvm::AtomicOrdering::Monotonic; }]>;
+def MemoryOrder_acquire_timm : NamedTImmLeaf<".acquire", i8, [{ return Imm == (unsigned)llvm::AtomicOrdering::Acquire; }]>;
+def MemoryOrder_release_timm : NamedTImmLeaf<".release", i8, [{ return Imm == (unsigned)llvm::AtomicOrdering::Release; }]>;
+def MemoryOrder_acq_rel_timm : NamedTImmLeaf<".acq_rel", i8, [{ return Imm == (unsigned)llvm::AtomicOrdering::AcquireRelease; }]>;
+defvar MemoryOrderNodes = [MemoryOrder_relaxed_timm, MemoryOrder_acquire_timm, MemoryOrder_release_timm, MemoryOrder_acq_rel_timm];
+
+// rounding mode types (FloatingPointMode.h)
+def RoundingMode_RZ_timm : NamedTImmLeaf<".rz", i8, [{ return Imm == (int)llvm::RoundingMode::TowardZero; }]>;
+def RoundingMode_RE_timm : NamedTImmLeaf<".re", i8, [{ return Imm == (int)llvm::RoundingMode::NearestTiesToEven; }]>;
+def RoundingMode_RU_timm : NamedTImmLeaf<".ru", i8, [{ return Imm == (int)llvm::RoundingMode::TowardPositive; }]>;
+def RoundingMode_RD_timm : NamedTImmLeaf<".rd", i8, [{ return Imm == (int)llvm::RoundingMode::TowardNegative; }]>;
+def RoundingMode_RNA_timm : NamedTImmLeaf<".rna", i8, [{ return Imm == (int)llvm::RoundingMode::NearestTiesToAway; }]>;
+def RoundingMode_INV_timm : NamedTImmLeaf<"", i8, [{ return Imm == (int)llvm::RoundingMode::Invalid; }]>;
+defvar RoundingModeNodes = [RoundingMode_RZ_timm, RoundingMode_RE_timm, RoundingMode_RU_timm,
+ RoundingMode_RD_timm, RoundingMode_RNA_timm];
+defvar RoundingModeNone = [RoundingMode_INV_timm];
+defvar RoundingModeAllNodes = !listconcat(RoundingModeNodes, RoundingModeNone);
+
+
+//////
+// BEGIN: special register definitions
+multiclass SpecialRegPat<Intrinsic intr, Register reg> {
+ defvar RetTy = intr.RetTypes[0];
+ defvar VT = !if(!isa<LLVMQualPointerType>(RetTy),
+ !cast<pisaVT>("_p"#!cast<LLVMQualPointerType>(RetTy).Sig[1]),
+ !cast<pisaVT>("_"#RetTy.VT));
+ def : Pat<(intr),
+ (COPY_TO_REGCLASS (VT.VT reg), VT.RC)>;
+}
+
+defm : SpecialRegPat<int_pisa_local_id_x, SpecialReg_LocalIdX>;
+defm : SpecialRegPat<int_pisa_local_id_y, SpecialReg_LocalIdY>;
+defm : SpecialRegPat<int_pisa_local_id_z, SpecialReg_LocalIdZ>;
+defm : SpecialRegPat<int_pisa_local_size_x, SpecialReg_LocalSizeX>;
+defm : SpecialRegPat<int_pisa_local_size_y, SpecialReg_LocalSizeY>;
+defm : SpecialRegPat<int_pisa_local_size_z, SpecialReg_LocalSizeZ>;
+defm : SpecialRegPat<int_pisa_enqueued_local_size_x, SpecialReg_EnqueuedLocalSizeX>;
+defm : SpecialRegPat<int_pisa_enqueued_local_size_y, SpecialReg_EnqueuedLocalSizeY>;
+defm : SpecialRegPat<int_pisa_enqueued_local_size_z, SpecialReg_EnqueuedLocalSizeZ>;
+defm : SpecialRegPat<int_pisa_global_offset_x, SpecialReg_GlobalOffsetX>;
+defm : SpecialRegPat<int_pisa_global_offset_y, SpecialReg_GlobalOffsetY>;
+defm : SpecialRegPat<int_pisa_global_offset_z, SpecialReg_GlobalOffsetZ>;
+defm : SpecialRegPat<int_pisa_global_size_x, SpecialReg_GlobalSizeX>;
+defm : SpecialRegPat<int_pisa_global_size_y, SpecialReg_GlobalSizeY>;
+defm : SpecialRegPat<int_pisa_global_size_z, SpecialReg_GlobalSizeZ>;
+defm : SpecialRegPat<int_pisa_group_id_x, SpecialReg_GroupIdX>;
+defm : SpecialRegPat<int_pisa_group_id_y, SpecialReg_GroupIdY>;
+defm : SpecialRegPat<int_pisa_group_id_z, SpecialReg_GroupIdZ>;
+defm : SpecialRegPat<int_pisa_group_count_x, SpecialReg_GroupCountX>;
+defm : SpecialRegPat<int_pisa_group_count_y, SpecialReg_GroupCountY>;
+defm : SpecialRegPat<int_pisa_group_count_z, SpecialReg_GroupCountZ>;
+defm : SpecialRegPat<int_pisa_subgroup_size, SpecialReg_SubgroupSize>;
+defm : SpecialRegPat<int_pisa_lane_id, SpecialReg_LaneId>;
+defm : SpecialRegPat<int_pisa_activemask, SpecialReg_ActiveMask>;
+defm : SpecialRegPat<int_pisa_work_dim, SpecialReg_WorkDim>;
+// END: special register definitions
+//////
+
+//////
+// BEGIN: auxiliary class definitions
+// generate a table that maps name to instruction
+// - used in selection of insert/extract instructions
+class Name2InstrEntry<string name, Instruction inst> {
+ string Name = name;
+ Instruction Inst = inst;
+}
+def Name2InstrOpTable : GenericTable {
+ let FilterClass = "Name2InstrEntry";
+ let Fields = ["Name", "Inst"];
+}
+def lookupName2InstrOpEntry : SearchIndex {
+ let Table = Name2InstrOpTable;
+ let Key = ["Name"];
+}
+// Generate a table that maps a register-class name to lifetime.start opcode.
+class LifetimeStartEntry<string rcName, Instruction inst> {
+ string RegClassName = rcName;
+ Instruction Inst = inst;
+}
+def LifetimeStartTable : GenericTable {
+ let FilterClass = "LifetimeStartEntry";
+ let Fields = ["RegClassName", "Inst"];
+}
+def lookupLifetimeStartByRegClass : SearchIndex {
+ let Table = LifetimeStartTable;
+ let Key = ["RegClassName"];
+}
+// END: auxiliary class definitions
+//////
+
+//////
+// BEGIN: unary instruction patterns
+class UnaryInst_r<string name, SDPatternOperator node, pisaVT dstVT, pisaVT srcVT,
+ dag extraArgs = (ops)>
+ : PISAInst_r<name, dstVT, srcVT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (srcVT.VT srcRC:$src)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (srcVT.VT srcRC:$src)))]);
+}
+class UnaryInst_i<string name, SDPatternOperator node, pisaVT dstVT, pisaVT srcVT,
+ dag extraArgs = (ops)>
+ : PISAInst_i<name, dstVT, srcVT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (srcVT.VT immVal:$src)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (srcVT.VT immVal:$src)))]);
+}
+
+// unary integer instruction
+multiclass UnaryInst<string name, SDPatternOperator node, list<pisaVT> types = [_i16, _i32, _i64]> {
+ foreach VT = types in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, VT, VT>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, VT, VT>;
+ }
+}
+
+// unary float instruction
+multiclass UnaryFloatInst<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], bit isFast = 0> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ defvar fastName = !if(isFast, ".fast", "");
+ foreach VT = types in {
+ defvar directive = !interleave([ftz.name, fastName, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ }
+ }
+}
+
+// unary float instruction with rounding mode
+multiclass UnaryFloatInstRnd<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64]> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ foreach rndNode = RoundingModeNodes in {
+ defvar directive = !interleave([rndNode.name, ftz.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, VT, VT, (node rndNode)>, Requires<[ftz.pred]>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, VT, VT, (node rndNode)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+// END: unary instruction patterns
+//////
+
+//////
+// BEGIN: binary instruction patterns
+class BinaryInst_rr<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src1VT, pisaVT dstVT=src0VT,
+ dag extraArgs = (ops)>
+ : PISAInst_rr<name, dstVT, src0VT, src1VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src1VT.VT src1RC:$src1)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src1VT.VT src1RC:$src1)))]);
+}
+class BinaryInst_ri<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src1VT, pisaVT dstVT=src0VT,
+ dag extraArgs = (ops)>
+ : PISAInst_ri<name, dstVT, src0VT, src1VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src1VT.VT immVal1:$src1)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src1VT.VT immVal1:$src1)))]);
+}
+class BinaryInst_ir<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src1VT, pisaVT dstVT=src0VT,
+ bit isCommutative = true, dag extraArgs = (ops)>
+ : PISAInst_ir<name, dstVT, src0VT, src1VT> {
+ // Pattern is suppressed for commutative instructions: the canonicalizer always
+ // places immediates on the RHS, so the _ir form would never match. Pass
+ // isCommutative = false (e.g. for intrinsics) to emit a pattern.
+ let Pattern = !if(isCommutative,
+ [],
+ !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src1VT.VT src1RC:$src1)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src1VT.VT src1RC:$src1)))]));
+}
+class BinaryInst_ii<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src1VT, pisaVT dstVT=src0VT,
+ dag extraArgs = (ops)>
+ : PISAInst_ii<name, dstVT, src0VT, src1VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (dstVT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src1VT.VT immVal1:$src1)), extraArgs))],
+ [(set (dstVT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src1VT.VT immVal1:$src1)))]);
+}
+
+// binary integer instruction with optional saturation modifier
+multiclass BinaryInst<string name, SDPatternOperator node, list<pisaVT> types = [_i16, _i32, _i64], bit isCommutative = true, list<SaturationModifier> satMods = [NoSatMod]> {
+ foreach sat = satMods in {
+ foreach VT = types in {
+ defvar directive = !interleave([sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr": BinaryInst_rr<name#directive, node, VT, VT>;
+ def opcode#"_ri": BinaryInst_ri<name#directive, node, VT, VT>;
+ def opcode#"_ir": BinaryInst_ir<name#directive, node, VT, VT, VT, isCommutative>;
+ def opcode#"_ii": BinaryInst_ii<name#directive, node, VT, VT>;
+ }
+ }
+}
+
+// binary float instruction with optional saturation modifier
+multiclass BinaryFloatInst<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], bit isCommutative = true, list<SaturationModifier> satMods = [NoSatMod], bit isFast = false> {
+ foreach sat = satMods in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ defvar fastName = !if(isFast, ".fast", "");
+ foreach VT = types in {
+ defvar directive = !interleave([ftz.name, fastName, sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : BinaryInst_rr<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_ri" : BinaryInst_ri<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_ir" : BinaryInst_ir<name#directive, node, VT, VT, VT, isCommutative>, Requires<[ftz.pred]>;
+ def opcode#"_ii" : BinaryInst_ii<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+
+// binary float instruction with a trailing i1 nanp immediate
+multiclass BinaryFloatInstNanp<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], bit isCommutative = true, list<SaturationModifier> satMods = [NoSatMod]> {
+ foreach sat = satMods in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach nanp = AllNanpMods in {
+ foreach VT = types in {
+ defvar directive = !interleave([ftz.name, sat.name, nanp.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : BinaryInst_rr<name#directive, node, VT, VT, VT, (node nanp.node)>, Requires<[ftz.pred]>;
+ def opcode#"_ri" : BinaryInst_ri<name#directive, node, VT, VT, VT, (node nanp.node)>, Requires<[ftz.pred]>;
+ def opcode#"_ir" : BinaryInst_ir<name#directive, node, VT, VT, VT, isCommutative, (node nanp.node)>, Requires<[ftz.pred]>;
+ def opcode#"_ii" : BinaryInst_ii<name#directive, node, VT, VT, VT, (node nanp.node)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+ }
+}
+
+// binary float instruction with .nanp modifier only
+multiclass BinaryFloatInstNanpOnly<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], bit isCommutative = true> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ defvar directive = !interleave([ftz.name, ".nanp", VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : BinaryInst_rr<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_ri" : BinaryInst_ri<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_ir" : BinaryInst_ir<name#directive, node, VT, VT, VT, isCommutative>, Requires<[ftz.pred]>;
+ def opcode#"_ii" : BinaryInst_ii<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ }
+ }
+}
+
+// binary float instruction with rounding mode
+multiclass BinaryFloatInstRnd<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], list<NamedTImmLeaf> rndNodes = RoundingModeNodes> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ foreach rndNode = rndNodes in {
+ defvar directive = !interleave([rndNode.name, ftz.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrii": BinaryInst_rr<name#directive, node, VT, VT, VT, (node rndNode)>, Requires<[ftz.pred]>;
+ def opcode#"_riii": BinaryInst_ri<name#directive, node, VT, VT, VT, (node rndNode)>, Requires<[ftz.pred]>;
+ def opcode#"_irii": BinaryInst_ir<name#directive, node, VT, VT, VT, false, (node rndNode)>, Requires<[ftz.pred]>;
+ def opcode#"_iiii": BinaryInst_ii<name#directive, node, VT, VT, VT, (node rndNode)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+
+// binary float instruction with rounding mode and saturation modifier
+multiclass BinaryFloatInstRndSat<string name, SDPatternOperator node, list<pisaVT> types = [_bf16, _f16, _f32, _f64], list<NamedTImmLeaf> rndNodes = RoundingModeAllNodes, list<SaturationModifier> satMods = [NoSatMod, DsatMod], bit isCommutative = true, bit isFullPrecision = false> {
+ defvar fullName = !if(isFullPrecision, ".full", "");
+ foreach sat = satMods in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ foreach rndNode = rndNodes in {
+ defvar directive = !interleave([fullName, rndNode.name, ftz.name, sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ defvar resultVT = !if(isFullPrecision, _f32, VT);
+ def opcode#"_rrii": BinaryInst_rr<name#directive, node, VT, VT, resultVT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_riii": BinaryInst_ri<name#directive, node, VT, VT, resultVT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_irii": BinaryInst_ir<name#directive, node, VT, VT, resultVT, isCommutative, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_iiii": BinaryInst_ii<name#directive, node, VT, VT, resultVT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+ }
+}
+
+// END: binary instruction patterns
+//////
+
+//////
+// BEGIN: ternary instruction patterns
+class TernaryInst_rrr<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_rrr<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src0VT.VT src1RC:$src1), (src0VT.VT src2RC:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src0VT.VT src1RC:$src1), (src0VT.VT src2RC:$src2)))]);
+}
+class TernaryInst_rri<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_rri<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT immVal2:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT immVal2:$src2)))]);
+}
+class TernaryInst_rir<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_rir<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT src2RC:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT src2RC:$src2)))]);
+}
+class TernaryInst_rii<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_rii<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT src0RC:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT immVal2:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT src0RC:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT immVal2:$src2)))]);
+}
+class TernaryInst_irr<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_irr<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT src2RC:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT src2RC:$src2)))]);
+}
+class TernaryInst_iri<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_iri<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT immVal2:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src0VT.VT src1RC:$src1), (src2VT.VT immVal2:$src2)))]);
+}
+class TernaryInst_iir<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_iir<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT src2RC:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT src2RC:$src2)))]);
+}
+class TernaryInst_iii<string name, SDPatternOperator node, pisaVT src0VT, pisaVT src2VT,
+ dag extraArgs = (ops)>
+ : PISAInst_iii<name, src2VT, src0VT, src0VT, src2VT> {
+ let Pattern = !if(!gt(!size(extraArgs), 0),
+ [(set (src0VT.VT dstRC:$dst), !con((node (src0VT.VT immVal0:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT immVal2:$src2)), extraArgs))],
+ [(set (src0VT.VT dstRC:$dst), (node (src0VT.VT immVal0:$src0), (src0VT.VT immVal1:$src1), (src2VT.VT immVal2:$src2)))]);
+}
+
+// ternary integer instruction with optional saturation modifier
+multiclass TernaryInst<string name, SDPatternOperator node, list<pisaVT> types = [_i16, _i32, _i64], list<SaturationModifier> satMods = [NoSatMod]> {
+ foreach sat = satMods in {
+ foreach VT = types in {
+ defvar directive = !interleave([sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr" : TernaryInst_rrr<name#directive, node, VT, VT>;
+ def opcode#"_rri" : TernaryInst_rri<name#directive, node, VT, VT>;
+ def opcode#"_rir" : TernaryInst_rir<name#directive, node, VT, VT>;
+ def opcode#"_rii" : TernaryInst_rii<name#directive, node, VT, VT>;
+ def opcode#"_irr" : TernaryInst_irr<name#directive, node, VT, VT>;
+ def opcode#"_iri" : TernaryInst_iri<name#directive, node, VT, VT>;
+ def opcode#"_iir" : TernaryInst_iir<name#directive, node, VT, VT>;
+ def opcode#"_iii" : TernaryInst_iii<name#directive, node, VT, VT>;
+ }
+ }
+}
+
+// ternary float instruction with optional saturation modifier
+multiclass TernaryFloatInst<string name, SDPatternOperator node> {
+ foreach sat = [NoSatMod] in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = [_bf16, _f16, _f32, _f64] in {
+ defvar directive = !interleave([ftz.name, sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr" : TernaryInst_rrr<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rri" : TernaryInst_rri<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rir" : TernaryInst_rir<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rii" : TernaryInst_rii<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_irr" : TernaryInst_irr<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_iri" : TernaryInst_iri<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_iir" : TernaryInst_iir<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ def opcode#"_iii" : TernaryInst_iii<name#directive, node, VT, VT>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+
+// ternary float instruction with rounding mode and optional saturation modifier
+multiclass TernaryFloatInstRndSat<string name, SDPatternOperator node, list<pisaVT> types> {
+ foreach sat = [NoSatMod, DsatMod] in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ foreach rndNode = RoundingModeAllNodes in {
+ defvar directive = !interleave([rndNode.name, ftz.name, sat.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrrii" : TernaryInst_rrr<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_rriii" : TernaryInst_rri<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_ririi" : TernaryInst_rir<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_riiii" : TernaryInst_rii<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_irrii" : TernaryInst_irr<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_iriii" : TernaryInst_iri<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_iirii" : TernaryInst_iir<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_iiiii" : TernaryInst_iii<name#directive, node, VT, VT, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+ }
+}
+
+// END: ternary instruction patterns
+//////
+
+//////
+// BEGIN: convert instruction patterns
+// convert instructions from multiple types to a single type
+multiclass ConvertMultiInst<string name, SDPatternOperator Node, pisaVT dstTy, list<pisaVT> srcTys> {
+ foreach srcTy = srcTys in {
+ defvar directive = !interleave([dstTy.directive, srcTy.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r" : UnaryInst_r<name#directive, Node, dstTy, srcTy>;
+ def opcode#"_i" : UnaryInst_i<name#directive, Node, dstTy, srcTy>;
+ }
+}
+
+// convert float instructions from multiple types to a single type
+multiclass ConvertMultiFloatInst<string name, SDPatternOperator node, pisaVT dstTy, list<pisaVT> srcTys> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach srcTy = srcTys in {
+ defvar directive = !interleave([dstTy.directive, srcTy.directive, ftz.name], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, dstTy, srcTy>, Requires<[ftz.pred]>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, dstTy, srcTy>, Requires<[ftz.pred]>;
+ }
+ }
+}
+
+multiclass ConvertMultiFloatInstRnd<string name, SDPatternOperator node, pisaVT dstTy, list<pisaVT> srcTys> {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach srcTy = srcTys in {
+ foreach rndNode = RoundingModeNodes in {
+ defvar directive = !interleave([dstTy.directive, srcTy.directive, rndNode.name, ftz.name], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, dstTy, srcTy, (node rndNode)>, Requires<[ftz.pred]>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, dstTy, srcTy, (node rndNode)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+
+// convert float instructions with rounding mode and saturation modifier
+multiclass ConvertMultiFloatInstRndSat<string name, SDPatternOperator node, pisaVT dstTy, list<pisaVT> srcTys> {
+ foreach sat = [NoSatMod, DsatMod] in {
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach srcTy = srcTys in {
+ foreach rndNode = RoundingModeAllNodes in {
+ defvar directive = !interleave([dstTy.directive, srcTy.directive, rndNode.name, ftz.name, sat.name], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, dstTy, srcTy, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, dstTy, srcTy, (node rndNode, sat.node)>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+ }
+}
+
+// convert instructions from single type to multiple types
+multiclass ConvertSingleInst<string name, SDPatternOperator node, list<pisaVT> dstTys, pisaVT srcTy> {
+ foreach dstTy = dstTys in {
+ defvar directive = !interleave([dstTy.directive, srcTy.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r" : UnaryInst_r<name#directive, node, dstTy, srcTy>;
+ def opcode#"_i" : UnaryInst_i<name#directive, node, dstTy, srcTy>;
+ }
+}
+// END: convert instruction patterns
+//////
+
+// function parameters
+let isMeta = 1 in {
+ multiclass FunctionParameter {
+ foreach VT = VTs.CallReturn in {
+ def "_"#VT.name : PISAInst<(outs VT.RC:$var), (ins i32imm:$idx), " ; ">;
+ }
+ }
+ defm functionParameter : FunctionParameter<>;
+}
+
+//////
+// BEGIN: integer arithmetic instruction definitions
+// => dp4a
+multiclass DP4AInst {
+ foreach node = [int_pisa_dp4a_uu, int_pisa_dp4a_us, int_pisa_dp4a_su, int_pisa_dp4a_ss] in {
+ defvar type = !subst("int_pisa_dp4a_", "", !cast<string>(node));
+ foreach sat = [NoSatMod, DsatMod] in {
+ defvar invalid = !and(!eq(type, "uu"), !eq(sat, DsatMod));
+ if !not(invalid) then {
+ defvar directive = type#sat.name#".32b";
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr": TernaryInst_rrr<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_rri": TernaryInst_rri<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_rir": TernaryInst_rir<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_rii": TernaryInst_rii<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_irr": TernaryInst_irr<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_iri": TernaryInst_iri<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_iir": TernaryInst_iir<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ def opcode#"_iii": TernaryInst_iii<NAME#"."#directive, node, _i32, _i32, (node sat.node)>;
+ }
+ }
+ }
+}
+defm dp4a : DP4AInst;
+
+// => iabs
+defm iabs : UnaryInst<"iabs", abs>;
+
+
+// => iadd
+defm iadd: BinaryInst<"iadd", add>;
+defm ptradd: BinaryInst<"iadd", ptradd>;
+
+// .. with .dsat
+defm iadd: BinaryInst<"iadd", saddsat, satMods=[DsatMod]>;
+
+
+// => ineg
+defm ineg: UnaryInst<"ineg", ineg>;
+
+// => isub
+defm isub: BinaryInst<"isub", sub, isCommutative=false>;
+// .. with .dsat
+defm isub: BinaryInst<"isub", ssubsat, isCommutative=false, satMods=[DsatMod]>;
+
+// => sdiv
+defm sdiv_quo: BinaryInst<"sdiv.quo", sdiv, isCommutative=false>;
+defm sdiv: BinaryInst<"sdiv", sdiv, isCommutative=false>;
+defm sdiv_rem: BinaryInst<"sdiv.rem", srem, isCommutative=false>;
+
+// umad/smad
+class MadInst_rrr<string name, pisaVT bitwidth> : PISAInst_rrr<name, bitwidth, bitwidth, bitwidth, bitwidth> {
+ let Pattern = [(set dstRC:$dst, (add (mul src0RC:$src0, src1RC:$src1), src2RC:$src2))];
+}
+class MadInst_rri<string name, pisaVT bitwidth> : PISAInst_rri<name, bitwidth, bitwidth, bitwidth, bitwidth> {
+ let Pattern = [(set dstRC:$dst, (add (mul src0RC:$src0, src1RC:$src1), (bitwidth.VT imm:$src2)))];
+}
+class MadInst_rir<string name, pisaVT bitwidth> : PISAInst_rir<name, bitwidth, bitwidth, bitwidth, bitwidth> {
+ let Pattern = [(set dstRC:$dst, (add (mul src0RC:$src0, (bitwidth.VT imm:$src1)), src2RC:$src2))];
+}
+class MadInst_rii<string name, pisaVT bitwidth> : PISAInst_rii<name, bitwidth, bitwidth, bitwidth, bitwidth> {
+ let Pattern = [(set dstRC:$dst, (add (mul src0RC:$src0, (bitwidth.VT imm:$src1)), (bitwidth.VT imm:$src2)))];
+}
+class MadInst_iii<string name, pisaVT bitwidth> : PISAInst_iii<name, bitwidth, bitwidth, bitwidth, bitwidth> {
+ let Pattern = [(set dstRC:$dst, (add (mul (bitwidth.VT imm:$src0), (bitwidth.VT imm:$src1)), (bitwidth.VT imm:$src2)))];
+}
+class MadFullInst_rrr<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned> :
+ PISAInst_rrr<name, extBitwidth, bitwidth, bitwidth, extBitwidth> {
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (add (mul (extNode src0RC:$src0), (extNode src1RC:$src1)), src2RC:$src2))];
+}
+class MadFullInst_rri<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned> :
+ PISAInst_rri<name, extBitwidth, bitwidth, bitwidth, extBitwidth> {
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (add (mul (extNode src0RC:$src0), (extNode src1RC:$src1)), (extBitwidth.VT imm:$src2)))];
+}
+class MadFullInst_rir<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned> :
+ PISAInst_rir<name, extBitwidth, bitwidth, extBitwidth, extBitwidth> {
+ ImmLeaf immCheck1 = immType<isSigned, bitwidth.VT, extBitwidth.VT>.check;
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (add (mul (extNode src0RC:$src0), (immCheck1:$src1)), src2RC:$src2))];
+}
+class MadFullInst_rii<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned> :
+ PISAInst_rii<name, extBitwidth, bitwidth, extBitwidth, extBitwidth> {
+ ImmLeaf immCheck1 = immType<isSigned, bitwidth.VT, extBitwidth.VT>.check;
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (add (mul (extNode src0RC:$src0), (immCheck1:$src1)), (extBitwidth.VT imm:$src2)))];
+}
+multiclass MadInst<string name, bit isSigned> {
+ foreach VT = [_i16, _i32, _i64] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr" : MadInst_rrr<name#directive, VT>;
+ def opcode#"_rri" : MadInst_rri<name#directive, VT>;
+ def opcode#"_rir" : MadInst_rir<name#directive, VT>;
+ def opcode#"_rii" : MadInst_rii<name#directive, VT>;
+ def opcode#"_irr" : PISAInst_irr<name#directive, VT, VT, VT, VT>;
+ def opcode#"_iri" : PISAInst_iri<name#directive, VT, VT, VT, VT>;
+ def opcode#"_iir" : PISAInst_iir<name#directive, VT, VT, VT, VT>;
+ def opcode#"_iii" : MadInst_iii<name#directive, VT>;
+ }
+ foreach VT = [_i16, _i32] in {
+ defvar extBitwidth = !if(!eq(VT, _i16), _i32, _i64);
+ defvar directive = ".full"#VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr" : MadFullInst_rrr<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_rir" : MadFullInst_rir<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_rri" : MadFullInst_rri<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_rii" : MadFullInst_rii<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_irr" : PISAInst_irr<name#directive, extBitwidth, extBitwidth, VT, extBitwidth>;
+ def opcode#"_iir" : PISAInst_iir<name#directive, extBitwidth, extBitwidth, extBitwidth, extBitwidth>;
+ def opcode#"_iri" : PISAInst_iri<name#directive, extBitwidth, extBitwidth, VT, extBitwidth>;
+ def opcode#"_iii" : PISAInst_iii<name#directive, extBitwidth, extBitwidth, extBitwidth, extBitwidth>;
+ }
+}
+
+// => smad
+defm smad: MadInst<"smad", 1>;
+defm int_smad: TernaryInst<"smad", int_pisa_smad>;
+
+// => umad
+defm umad: MadInst<"umad", 0>;
+
+// => smax
+defm smax: BinaryInst<"smax", smax>;
+
+// => smin
+defm smin: BinaryInst<"smin", smin>;
+
+// => smul/umul
+// umul/smul
+class MulFullInst_rr<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned>
+ : PISAInst2<name, extBitwidth, bitwidth, bitwidth> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1);
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (mul (extNode src0RC:$src0), (extNode src1RC:$src1)))];
+}
+class MulFullInst_ri<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned>
+ : PISAInst2<name, extBitwidth, bitwidth, bitwidth> {
+ ImmLeaf immCheck1 = immType<isSigned, bitwidth.VT, extBitwidth.VT>.check;
+ DAGOperand immOpd1 = extBitwidth.ImmOpnd;
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1);
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [(set dstRC:$dst, (mul (extNode src0RC:$src0), (immCheck1:$src1)))];
+}
+class MulFullInst_ir<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned>
+ : PISAInst2<name, extBitwidth, bitwidth, bitwidth> {
+ DAGOperand immOpd0 = extBitwidth.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1);
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [];
+}
+class MulFullInst_ii<string name, pisaVT bitwidth, pisaVT extBitwidth, bit isSigned>
+ : PISAInst2<name, extBitwidth, bitwidth, bitwidth> {
+ DAGOperand immOpd0 = extBitwidth.ImmOpnd;
+ DAGOperand immOpd1 = extBitwidth.ImmOpnd;
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1);
+ SDNode extNode = !if(isSigned, sext, zext);
+ let Pattern = [];
+}
+multiclass MulInst<string name, bit isSigned> {
+ defm NAME: BinaryInst<name, mul>;
+ foreach VT = [_i16, _i32] in {
+ defvar extBitwidth = !if(!eq(VT, _i16), _i32, _i64);
+ defvar directive = ".full"#VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : MulFullInst_rr<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_ri" : MulFullInst_ri<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_ir" : MulFullInst_ir<name#directive, VT, extBitwidth, isSigned>;
+ def opcode#"_ii" : MulFullInst_ii<name#directive, VT, extBitwidth, isSigned>;
+ }
+}
+defm smul: MulInst<"smul", isSigned=true>;
+defm umul: MulInst<"umul", isSigned=false>;
+
+// uaddc/usubb
+class CarryOutInst_rr<string name, pisaVT VT> : PISAInst_rr<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1);
+}
+class CarryOutInst_ri<string name, pisaVT VT> : PISAInst_ri<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1);
+}
+class CarryOutInst_ir<string name, pisaVT VT> : PISAInst_ir<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1);
+}
+class CarryOutInst_ii<string name, pisaVT VT> : PISAInst_ii<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1);
+}
+multiclass CarryOutInst<string name> {
+ foreach VT = [_i32] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : CarryOutInst_rr<name#directive, VT>;
+ def opcode#"_ri" : CarryOutInst_ri<name#directive, VT>;
+ def opcode#"_ir" : CarryOutInst_ir<name#directive, VT>;
+ def opcode#"_ii" : CarryOutInst_ii<name#directive, VT>;
+ }
+}
+class CarryInInst_rr<string name, pisaVT VT> : PISAInst_rr<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst);
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, PredRC:$cin);
+}
+class CarryInInst_ri<string name, pisaVT VT> : PISAInst_ri<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst);
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1, PredRC:$cin);
+}
+class CarryInInst_ir<string name, pisaVT VT> : PISAInst_ir<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst);
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1, PredRC:$cin);
+}
+class CarryInInst_ii<string name, pisaVT VT> : PISAInst_ii<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst);
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1, PredRC:$cin);
+}
+multiclass CarryInInst<string name> {
+ foreach VT = [_i32] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : CarryInInst_rr<name#directive, VT>;
+ def opcode#"_ri" : CarryInInst_ri<name#directive, VT>;
+ def opcode#"_ir" : CarryInInst_ir<name#directive, VT>;
+ def opcode#"_ii" : CarryInInst_ii<name#directive, VT>;
+ }
+}
+class CarryInOutInst_rr<string name, pisaVT VT> : PISAInst_rr<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, PredRC:$cin);
+}
+class CarryInOutInst_ri<string name, pisaVT VT> : PISAInst_ri<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins src0RC:$src0, immOpd1:$src1, PredRC:$cin);
+}
+class CarryInOutInst_ir<string name, pisaVT VT> : PISAInst_ir<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins immOpd0:$src0, src1RC:$src1, PredRC:$cin);
+}
+class CarryInOutInst_ii<string name, pisaVT VT> : PISAInst_ii<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $cout, $src0, $src1, $cin;";
+ let OutOperandList = (outs dstRC:$dst, PredRC:$cout);
+ let InOperandList = (ins immOpd0:$src0, immOpd1:$src1, PredRC:$cin);
+}
+multiclass CarryInOutInst<string name> {
+ foreach VT = [_i32] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr" : CarryInOutInst_rr<name#directive, VT>;
+ def opcode#"_ri" : CarryInOutInst_ri<name#directive, VT>;
+ def opcode#"_ir" : CarryInOutInst_ir<name#directive, VT>;
+ def opcode#"_ii" : CarryInOutInst_ii<name#directive, VT>;
+ }
+}
+
+// => uaddc
+defm uaddc_co : CarryOutInst<"uaddc.co">;
+defm uaddc_ci : CarryInInst<"uaddc.ci">;
+defm uaddc_ci_co : CarryInOutInst<"uaddc.ci.co">;
+
+// => usubb
+defm usubb_co : CarryOutInst<"usubb.co">;
+defm usubb_ci : CarryInInst<"usubb.ci">;
+defm usubb_ci_co : CarryInOutInst<"usubb.ci.co">;
+
+// => udiv
+defm udiv_quo: BinaryInst<"udiv.quo", udiv, isCommutative=false>;
+defm udiv: BinaryInst<"udiv", udiv, isCommutative=false>;
+defm udiv_rem: BinaryInst<"udiv.rem", urem, isCommutative=false>;
+
+// => umax
+defm umax: BinaryInst<"umax", umax>;
+
+// => umin
+defm umin: BinaryInst<"umin", umin>;
+
+// END: integer arithmetic instruction definitions
+//////
+
+//////
+// BEGIN: floating point instruction definitions
+// => fabs
+defm fabs: UnaryFloatInst<"fabs", int_pisa_fabs>;
+
+
+// => fadd
+defm fadd: BinaryFloatInst<"fadd", fadd>;
+defm fadd: BinaryFloatInstRndSat<"fadd", int_pisa_fadd>;
+
+// => fcos
+defm fcos: UnaryFloatInst<"fcos", fcos, types=[_bf16, _f16, _f32]>;
+
+// => fdiv
+defm fdiv: BinaryFloatInst<"fdiv", fdiv, types=[_f32, _f64], isCommutative=false>;
+defm fdiv: BinaryFloatInst<"fdiv", fastbinop<fdiv>, isCommutative=false, isFast=1>;
+defm fdiv: BinaryFloatInstRnd<"fdiv", int_pisa_fdiv_rnd, [_f32, _f64]>;
+
+// => fexp2
+defm fexp2: UnaryFloatInst<"fexp2", fexp2, types=[_bf16, _f16, _f32]>;
+
+// => flog2
+defm flog2: UnaryFloatInst<"flog2", flog2, types=[_bf16, _f16, _f32]>;
+
+// => fmad
+defm fmad: TernaryFloatInst<"fmad", fma>;
+defm fmad: TernaryFloatInstRndSat<"fmad", int_pisa_fma, [_bf16, _f16, _f32, _f64]>;
+
+
+// => fmax
+defm fmax: BinaryFloatInst<"fmax", fmaxnum>;
+defm fmax: BinaryFloatInstNanp<"fmax", int_pisa_fmax_sat, satMods=[DsatMod]>;
+// fmax.nanp (NaN-propagating, no saturation) <= fmaximum, the
+// IEEE-754 maximum (any NaN input -> NaN). The plain fmax above (<= fmaxnum) is
+// maximumNumber, which suppresses NaN (returns the non-NaN operand).
+defm fmax: BinaryFloatInstNanpOnly<"fmax", fmaximum>;
+
+// => fmin
+defm fmin: BinaryFloatInst<"fmin", fminnum>;
+defm fmin: BinaryFloatInstNanp<"fmin", int_pisa_fmin_sat, satMods=[DsatMod]>;
+// fmin.nanp (NaN-propagating, no saturation) <= fminimum, the
+// IEEE-754 minimum (any NaN input -> NaN). The plain fmin above (<= fminnum) is
+// minimumNumber, which suppresses NaN (returns the non-NaN operand).
+defm fmin: BinaryFloatInstNanpOnly<"fmin", fminimum>;
+
+// => fmul
+defm fmul: BinaryFloatInst<"fmul", fmul>;
+defm fmul: BinaryFloatInstRndSat<"fmul", int_pisa_fmul>;
+
+// => fneg
+defm fneg: UnaryFloatInst<"fneg", fneg>;
+
+// => frc
+defm frc: UnaryFloatInst<"frc", int_pisa_frc, [_f32]>;
+
+// => frcp
+defm frcp: UnaryFloatInst<"frcp", int_pisa_frcp, types=[_bf16, _f16, _f32]>;
+
+// => frsqrt
+defm frsqrt: UnaryFloatInst<"frsqrt", int_pisa_frsqrt, isFast=1>;
+
+
+// => fsin
+defm fsin: UnaryFloatInst<"fsin", fsin, types=[_bf16, _f16, _f32]>;
+
+// => fsqrt
+defm fsqrt: UnaryFloatInst<"fsqrt", fsqrt, types=[_f32, _f64]>;
+defm fsqrt: UnaryFloatInst<"fsqrt", fastunaryop<fsqrt>, types=[_f32, _f64], isFast=1>;
+defm fsqrt: UnaryFloatInst<"fsqrt", fsqrt, types=[_bf16, _f16], isFast=1>;
+defm fsqrt: UnaryFloatInstRnd<"fsqrt", int_pisa_fsqrt_rnd, types=[_f32, _f64]>;
+
+// => fsub
+defm fsub: BinaryFloatInst<"fsub", fsub, isCommutative=false>;
+defm fsub: BinaryFloatInstRndSat<"fsub", int_pisa_fsub, isCommutative=false>;
+
+// => ftanh
+defm ftanh: UnaryFloatInst<"ftanh", ftanh, types=[_bf16, _f16, _f32]>;
+// END: floating point instruction definitions
+//////
+
+//////
+// BEGIN: bitwise instruction definitions
+// => and
+defm and : BinaryInst<"and", and>;
+
+// => bfi
+class Bfi_rrrr<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3))];
+}
+class Bfi_rrri<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, src1RC:$src1, src2RC:$src2, (VT.VT imm:$src3)))];
+}
+class Bfi_rrir<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, src1RC:$src1, (VT.VT imm:$src2), src3RC:$src3))];
+}
+class Bfi_rrii<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, src1RC:$src1, (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+}
+class Bfi_rirr<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, (VT.VT imm:$src1), src2RC:$src2, src3RC:$src3))];
+}
+class Bfi_riri<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, (VT.VT imm:$src1), src2RC:$src2, (VT.VT imm:$src3)))];
+}
+class Bfi_riir<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, (VT.VT imm:$src1), (VT.VT imm:$src2), src3RC:$src3))];
+}
+class Bfi_riii<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node src0RC:$src0, (VT.VT imm:$src1), (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+}
+class Bfi_irrr<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), src1RC:$src1, src2RC:$src2, src3RC:$src3))];
+}
+class Bfi_irri<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, src1RC:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), src1RC:$src1, src2RC:$src2, (VT.VT imm:$src3)))];
+}
+class Bfi_irir<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, src1RC:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), src1RC:$src1, (VT.VT imm:$src2), src3RC:$src3))];
+}
+class Bfi_irii<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, src1RC:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), src1RC:$src1, (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+}
+class Bfi_iirr<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, VT.ImmOpnd:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), (VT.VT imm:$src1), src2RC:$src2, src3RC:$src3))];
+}
+class Bfi_iiri<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, VT.ImmOpnd:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), (VT.VT imm:$src1), src2RC:$src2, (VT.VT imm:$src3)))];
+}
+class Bfi_iiir<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), (VT.VT imm:$src1), (VT.VT imm:$src2), src3RC:$src3))];
+}
+class Bfi_iiii<string name, SDPatternOperator node, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins VT.ImmOpnd:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node (VT.VT imm:$src0), (VT.VT imm:$src1), (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+}
+multiclass BfiInst {
+ defvar directive = "bfi.32b";
+ def _rrrr: Bfi_rrrr<directive, int_pisa_bfi, _i32>;
+ def _rrri: Bfi_rrri<directive, int_pisa_bfi, _i32>;
+ def _rrir: Bfi_rrir<directive, int_pisa_bfi, _i32>;
+ def _rrii: Bfi_rrii<directive, int_pisa_bfi, _i32>;
+ def _rirr: Bfi_rirr<directive, int_pisa_bfi, _i32>;
+ def _riri: Bfi_riri<directive, int_pisa_bfi, _i32>;
+ def _riir: Bfi_riir<directive, int_pisa_bfi, _i32>;
+ def _riii: Bfi_riii<directive, int_pisa_bfi, _i32>;
+ def _irrr: Bfi_irrr<directive, int_pisa_bfi, _i32>;
+ def _irri: Bfi_irri<directive, int_pisa_bfi, _i32>;
+ def _irir: Bfi_irir<directive, int_pisa_bfi, _i32>;
+ def _irii: Bfi_irii<directive, int_pisa_bfi, _i32>;
+ def _iirr: Bfi_iirr<directive, int_pisa_bfi, _i32>;
+ def _iiri: Bfi_iiri<directive, int_pisa_bfi, _i32>;
+ def _iiir: Bfi_iiir<directive, int_pisa_bfi, _i32>;
+ def _iiii: Bfi_iiii<directive, int_pisa_bfi, _i32>;
+}
+defm bfi : BfiInst;
+
+// => bfn
+class Bfn_irrr<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_irri<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, src1RC:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, src1RC:$src1, src2RC:$src2, (VT.VT imm:$src3)))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_irir<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, src1RC:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, src1RC:$src1, (VT.VT imm:$src2), src3RC:$src3))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_irii<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, src1RC:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, src1RC:$src1, (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_iirr<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, VT.ImmOpnd:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, (VT.VT imm:$src1), src2RC:$src2, src3RC:$src3))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_iiri<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, VT.ImmOpnd:$src1, src2RC:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, (VT.VT imm:$src1), src2RC:$src2, (VT.VT imm:$src3)))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_iiir<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, (VT.VT imm:$src1), (VT.VT imm:$src2), src3RC:$src3))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+class Bfn_iiii<string name, SDPatternOperator node, pisaVT opVT, pisaVT VT>
+ : PISAInst4<name, VT, opVT, VT, VT, VT> {
+ DAGOperand BfnOpcodeOpnd = PISABfnOpcodeOperand<opVT.VT, "Imm"#opVT.VT.Size#"Opnd">;
+ let InOperandList = (ins BfnOpcodeOpnd:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, VT.ImmOpnd:$src3);
+ let Pattern = [(set dstRC:$dst, (node timm:$src0, (VT.VT imm:$src1), (VT.VT imm:$src2), (VT.VT imm:$src3)))];
+ let AsmString = name#".${src0}"#VT.directive#" \t$dst, $src1, $src2, $src3;";
+}
+multiclass BfnInst {
+ defvar directive = "bfn";
+ def _irrr: Bfn_irrr<directive, int_pisa_bfn, _i8, _i32>;
+ def _irri: Bfn_irri<directive, int_pisa_bfn, _i8, _i32>;
+ def _irir: Bfn_irir<directive, int_pisa_bfn, _i8, _i32>;
+ def _irii: Bfn_irii<directive, int_pisa_bfn, _i8, _i32>;
+ def _iirr: Bfn_iirr<directive, int_pisa_bfn, _i8, _i32>;
+ def _iiri: Bfn_iiri<directive, int_pisa_bfn, _i8, _i32>;
+ def _iiir: Bfn_iiir<directive, int_pisa_bfn, _i8, _i32>;
+ def _iiii: Bfn_iiii<directive, int_pisa_bfn, _i8, _i32>;
+}
+defm bfn : BfnInst;
+
+// => bfrev
+defm bfrev : UnaryInst<"bfrev", bitreverse, types=[_i32]>;
+
+// => cbit
+defm cbit : UnaryInst<"cbit", ctpop, types=[_i16, _i32]>;
+
+// fbh/fbl
+multiclass FirstBitInst<SDPatternOperator node, list<pisaVT> types> {
+ foreach VT = types in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ defvar uminOp = !cast<Instruction>(umin#opcode#"_ri");
+ defvar bitSize = VT.VT.Size;
+ def opcode#"_r" : PISAInst_r<NAME#directive, VT, VT>;
+ def : Pat<(node VT.VT:$src0),
+ (uminOp (!cast<Instruction>(NAME#opcode#"_r") VT.RC:$src0), bitSize)>;
+ def opcode#"_i" : PISAInst_i<NAME#directive, VT, VT>;
+ def : Pat<(node (VT.VT imm:$src0)),
+ (uminOp (!cast<Instruction>(NAME#opcode#"_i") (VT.VT imm:$src0)), bitSize)>;
+ }
+}
+
+// => fbh
+defm fbh0 : UnaryInst<"fbh", ctlz_zero_poison, types=[_i16, _i32]>;
+defm fbh : FirstBitInst<ctlz, [_i16, _i32]>;
+
+// => fbl
+defm fbl0 : UnaryInst<"fbl", cttz_zero_poison, types=[_i16, _i32]>;
+defm fbl : FirstBitInst<cttz, [_i16, _i32]>;
+
+// => not
+defm not : UnaryInst<"not", not>;
+
+// => or
+defm or : BinaryInst<"or", or>;
+
+// => sbfe
+defm sbfe : TernaryInst<"sbfe", int_pisa_sbfe, types=[_i32]>;
+
+
+// => ubfe
+defm ubfe : TernaryInst<"ubfe", int_pisa_ubfe, types=[_i32]>;
+
+// => xor
+defm xor : BinaryInst<"xor", xor>;
+
+// Temporarily support "bfe.32b" as an alias for "ubfe.32b" until all uses are replaced
+def : InstAlias<TernaryInst_rrr<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_rrr Reg32b:$dst, Reg32b:$src0, Reg32b:$src1, Reg32b:$src2)>;
+def : InstAlias<TernaryInst_rri<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_rri Reg32b:$dst, Reg32b:$src0, Reg32b:$src1, _i32.ImmOpnd:$src2)>;
+def : InstAlias<TernaryInst_rir<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_rir Reg32b:$dst, Reg32b:$src0, _i32.ImmOpnd:$src1, Reg32b:$src2)>;
+def : InstAlias<TernaryInst_rii<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_rii Reg32b:$dst, Reg32b:$src0, _i32.ImmOpnd:$src1, _i32.ImmOpnd:$src2)>;
+def : InstAlias<TernaryInst_irr<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_irr Reg32b:$dst, _i32.ImmOpnd:$src0, Reg32b:$src1, Reg32b:$src2)>;
+def : InstAlias<TernaryInst_iri<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_iri Reg32b:$dst, _i32.ImmOpnd:$src0, Reg32b:$src1, _i32.ImmOpnd:$src2)>;
+def : InstAlias<TernaryInst_iir<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_iir Reg32b:$dst, _i32.ImmOpnd:$src0, _i32.ImmOpnd:$src1, Reg32b:$src2)>;
+def : InstAlias<TernaryInst_iii<"bfe.32b", int_pisa_ubfe, _i32, _i32>.AsmString, (ubfe_32b_iii Reg32b:$dst, _i32.ImmOpnd:$src0, _i32.ImmOpnd:$src1, _i32.ImmOpnd:$src2)>;
+// END: bitwise instruction definitions
+//////
+
+//////
+// BEGIN: shift instruction definitions
+// asr/shr/shl
+multiclass ShiftInst<string name, SDNode node> {
+ foreach VT = [_i16, _i32, _i64] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rr": BinaryInst_rr<name#directive, node, VT, _i32>;
+ def opcode#"_ri": BinaryInst_ri<name#directive, node, VT, _i32>;
+ def opcode#"_ir": BinaryInst_ir<name#directive, node, VT, _i32, VT, false>;
+ def opcode#"_ii": BinaryInst_ii<name#directive, node, VT, _i32>;
+ }
+}
+
+// => asr
+defm asr : ShiftInst<"asr", sra>;
+
+// => shr
+defm shr : ShiftInst<"shr", srl>;
+
+// => shl
+defm shl : ShiftInst<"shl", shl>;
+
+// => shf
+defm shf_r : TernaryInst<"shf.r", fshr, types=[_i32]>;
+defm shf_l : TernaryInst<"shf.l", fshl, types=[_i32]>;
+// END: shift instruction definitions
+//////
+
+//////
+// BEGIN: data movement and type conversion instructions definitions
+// => addrcast
+def : GINodeEquiv<G_ADDRSPACE_CAST, addrspacecast>;
+class AddrcastToGeneric<string name, AddressSpace Out, AddressSpace In> : _PISAInst<name> {
+ let AsmString = name#" \t$dst, $src;";
+ let OutOperandList = (outs Out.RC:$dst);
+ let InOperandList = (ins In.RC:$src);
+ let Pattern = [(set (p0 Out.RC:$dst), (addrspacecast (In.PVT In.RC:$src)))];
+}
+class AddrcastToGenericFromFI<string name> : _PISAInst<name> {
+ let AsmString = name#" \t$dst, $frameindex;";
+ let OutOperandList = (outs generic.RC:$dst);
+ let InOperandList = (ins FrameIndex:$frameindex);
+}
+class AddrcastToGenericFromGVar<string name> : _PISAInst<name> {
+ let AsmString = name#" \t$dst, $gvar;";
+ let OutOperandList = (outs generic.RC:$dst);
+ let InOperandList = (ins PISAGlobalVariableOpnd:$gvar);
+}
+class AddrcastFromGeneric<string name, AddressSpace Out, AddressSpace In>
+ : AddrcastToGeneric<name, Out, In> {
+ let Pattern = [(set (Out.PVT Out.RC:$dst), (addrspacecast (p0 In.RC:$src)))];
+}
+class AddrcastToGenericCustomISel<string name, AddressSpace Out, AddressSpace In> : AddrcastToGeneric<name, Out, In> {
+ let Pattern = [];
+}
+multiclass AddrcastInst<string name> {
+ defvar Gen = generic;
+ // addrspacecast.generic.[private|shared] %dst, @V
+ foreach NonGen = [private, shared] in {
+ def _gen_#NonGen#_FI: AddrcastToGenericFromFI<name#"."#Gen#"."#NonGen>;
+ }
+ // addrspacecast.generic.[global|const] %dst, @G
+ foreach NonGen = [global, const] in {
+ def _gen_#NonGen#_var: AddrcastToGenericFromGVar<name#"."#Gen#"."#NonGen>;
+ }
+ foreach NonGen = [private, shared, global] in {
+ def _gen_#NonGen: AddrcastToGeneric<name#"."#Gen#"."#NonGen, Gen, NonGen>;
+ def _#NonGen#_gen: AddrcastFromGeneric<name#"."#NonGen#"."#Gen, NonGen, Gen>;
+ }
+ foreach NonGen = [const] in {
+ def _gen_#NonGen: AddrcastToGeneric<name#"."#Gen#"."#NonGen, Gen, NonGen>;
+ }
+}
+defm addrcast : AddrcastInst<"addrcast">;
+
+// => addrof
+multiclass AddrOf {
+ def _32b: PISAInst<(outs _i32.RegOpnd:$var), (ins FrameIndex:$frameindex), "addrof.32b $var, $frameindex;">;
+ def _64b: PISAInst<(outs _i64.RegOpnd:$var), (ins PISAGlobalVariableOpnd:$varname), "addrof.64b $var, $varname;">;
+ def _64b_func: PISAInst<(outs _i64.RegOpnd:$var), (ins FunctionOpnd:$funcname), "addrof.64b $var, $funcname;">;
+}
+defm addrof : AddrOf<>; // MIR for G_FRAME_INDEX
+
+// => extract
+class ExtractInst_r<string name, pisaVT dstVT, pisaVT srcVT>
+ : PISAInst_r<name, dstVT, srcVT> {
+}
+multiclass Extract {
+ defvar VTs = [_v2i32, _v3i32, _v4i32, _v5i32, _v6i32, _v7i32, _v8i32, _v16i32, _v32i32, _v64i32];
+ defvar lengths = [2, 3, 4, 5, 6, 7, 8, 16, 32, 64];
+ // For extract, src must be a vector type.
+ // For each src vector type...
+ foreach i = !range(VTs) in {
+ defvar srcVT = VTs[i];
+ defvar srcLength = lengths[i];
+ // The range of valid extract indices is [0, ..., length - 1] (i.e., for v4, [0, 1, 2, 3])...
+ foreach index = !range(srcLength) in {
+ // At each valid index, you can always extract a scalar.
+ def "_"#index#"_i32_"#srcVT.name#"_r" : ExtractInst_r<"extract."#index#".32b", _i32, srcVT>;
+ // For each valid destination vector type, check if it fits in remaining space
+ foreach j = !range(VTs) in {
+ defvar dstVT = VTs[j];
+ defvar dstLength = lengths[j];
+ // Destination vector is valid if dstLength <= (srcLength - index)
+ if !le(dstLength, !sub(srcLength, index)) then {
+ def "_"#index#"_"#dstVT.name#"_"#srcVT.name#"_r" : ExtractInst_r<"extract."#index#dstVT.directive, dstVT, srcVT>;
+ }
+ }
+ }
+ }
+}
+defset list<Instruction> ExtractInsts = {
+ defm extract : Extract;
+}
+foreach EI = ExtractInsts in {
+ def : Name2InstrEntry<!cast<string>(EI), EI>;
+}
+
+// => extract.dynamic
+multiclass ExtractDynamic {
+ defvar srcVTs = [_v2i32, _v3i32, _v4i32, _v5i32, _v6i32, _v7i32, _v8i32, _v16i32, _v32i32, _v64i32];
+ foreach srcVT = srcVTs in {
+ def "_"#srcVT.name : PISAInst_rr<"extract.dynamic"#srcVT.directive, _i32, srcVT, _i32>;
+ }
+}
+defset list<Instruction> ExtractDynamicInsts = {
+ defm extract_dynamic : ExtractDynamic;
+}
+foreach EDI = ExtractDynamicInsts in {
+ def : Name2InstrEntry<!cast<string>(EDI), EDI>;
+}
+
+// => f2i
+multiclass FP2IntInst<string name, SDPatternOperator node, bit isSigned> {
+ foreach srcVT = [_bf16, _f16, _f32, _f64] in {
+ foreach dstVT = [_i8, _i16, _i32, _i64] in {
+ defvar toName = !if(isSigned,".s",".u")#dstVT.VT.Size;
+ defvar directive = !interleave([toName, srcVT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r" : UnaryInst_r<name#directive, node, dstVT, srcVT>;
+ def opcode#"_i" : UnaryInst_i<name#directive, node, dstVT, srcVT>;
+ }
+ }
+}
+multiclass FP2IntSatInst<string name, SDPatternOperator node, bit isSigned> {
+ foreach srcVT = [_bf16, _f16, _f32, _f64] in {
+ foreach dstVT = [_i8, _i16, _i32, _i64] in {
+ defvar toName = !if(isSigned,".s",".u")#dstVT.VT.Size;
+ defvar directive = !interleave([toName, srcVT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r" : UnaryInst_r<name#directive, node, dstVT, srcVT, (node dstVT.VT)>;
+ def opcode#"_i" : UnaryInst_i<name#directive, node, dstVT, srcVT, (node dstVT.VT)>;
+ }
+ }
+}
+multiclass FP2IntRndInst<string name, SDPatternOperator node, bit isSigned> {
+ foreach srcVT = [_bf16, _f16, _f32, _f64] in {
+ foreach dstVT = [_i8, _i16, _i32, _i64] in {
+ defvar toName = !if(isSigned,".s",".u")#dstVT.VT.Size;
+ foreach rndNode = RoundingModeNodes in {
+ defvar directive = !interleave([toName, srcVT.directive, rndNode.name], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r": UnaryInst_r<name#directive, node, dstVT, srcVT, (node rndNode)>;
+ def opcode#"_i": UnaryInst_i<name#directive, node, dstVT, srcVT, (node rndNode)>;
+ }
+ }
+ }
+}
+defm f2i : FP2IntInst<"f2i", fp_to_sint, isSigned=true>;
+defm f2i_sat : FP2IntSatInst<"f2i", fp_to_sint_sat, isSigned=true>;
+defm f2i : FP2IntInst<"f2i", fp_to_uint, isSigned=false>;
+defm f2i_sat : FP2IntSatInst<"f2i", fp_to_uint_sat, isSigned=false>;
+defm f2i : FP2IntRndInst<"f2i", int_pisa_fptosi_rnd, isSigned=true>;
+defm f2i : FP2IntRndInst<"f2i", int_pisa_fptoui_rnd, isSigned=false>;
+
+// => fext
+defm fext : ConvertSingleInst<"fext", fpextend, [_f32, _f64], _bf16>;
+defm fext : ConvertSingleInst<"fext", fpextend, [_f32, _f64], _f16>;
+defm fext : ConvertSingleInst<"fext", fpextend, [_f64], _f32>;
+
+// => frnd
+def pisa_nearestint : PatFrags<(ops node:$src0), [(frint node:$src0),
+ (fnearbyint node:$src0),
+ (froundeven node:$src0)]>;
+defm frnd_re : UnaryFloatInst<"frnd.re", pisa_nearestint>;
+defm frnd_rd : UnaryFloatInst<"frnd.rd", ffloor>;
+defm frnd_ru : UnaryFloatInst<"frnd.ru", fceil>;
+defm frnd_rz : UnaryFloatInst<"frnd.rz", ftrunc>;
+defm frnd_rna: UnaryFloatInst<"frnd.rna", fround>;
+defm frnd0 : UnaryFloatInstRnd<"frnd", int_pisa_frnd_rnd>;
+
+// => ftrunc
+defm ftrunc : ConvertMultiFloatInst<"ftrunc", fpround, _bf16, [_f32, _f64]>;
+defm ftrunc : ConvertMultiFloatInst<"ftrunc", fpround, _f16, [_f32, _f64]>;
+defm ftrunc : ConvertMultiFloatInst<"ftrunc", fpround, _f32, [_f64]>;
+// .. llvm.fptrunc.round
+defm ftrunc : ConvertMultiFloatInstRnd<"ftrunc", fptrunc_round, _bf16, [_f32, _f64]>;
+defm ftrunc : ConvertMultiFloatInstRnd<"ftrunc", fptrunc_round, _f16, [_f32, _f64]>;
+defm ftrunc : ConvertMultiFloatInstRnd<"ftrunc", fptrunc_round, _f32, [_f64]>;
+defm ftrunc0 : ConvertMultiFloatInstRndSat<"ftrunc", int_pisa_ftrunc, _bf16, [_f32, _f64]>;
+defm ftrunc0 : ConvertMultiFloatInstRndSat<"ftrunc", int_pisa_ftrunc, _f16, [_f32, _f64]>;
+defm ftrunc0 : ConvertMultiFloatInstRndSat<"ftrunc", int_pisa_ftrunc, _f32, [_f64]>;
+
+
+// => i2f
+multiclass Int2FPInst<string name, SDPatternOperator node, bit isSigned> {
+ foreach srcVT = [_i8, _i16, _i32, _i64] in {
+ defvar fromName = !if(isSigned,".s",".u")#srcVT.VT.Size;
+ foreach dstVT = [_bf16, _f16, _f32, _f64] in {
+ defvar directive = !interleave([dstVT.directive, fromName], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_r" : UnaryInst_r<name#directive, node, dstVT, srcVT>;
+ def opcode#"_i" : UnaryInst_i<name#directive, node, dstVT, srcVT>;
+ }
+ }
+}
+multiclass Int2FPInstRndSat<string name, SDPatternOperator node, bit isSigned> {
+ foreach sat = [NoSatMod, DsatMod] in {
+ foreach srcVT = [_i8, _i16, _i32, _i64] in {
+ defvar fromName = !if(isSigned,".s",".u")#srcVT.VT.Size;
+ foreach dstVT = [_bf16, _f16, _f32, _f64] in {
+ foreach rndNode = RoundingModeAllNodes in {
+ defvar directive = !interleave([dstVT.directive, fromName, rndNode.name, sat.name], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rii": UnaryInst_r<name#directive, node, dstVT, srcVT, (node rndNode, sat.node)>;
+ def opcode#"_iii": UnaryInst_i<name#directive, node, dstVT, srcVT, (node rndNode, sat.node)>;
+ }
+ }
+ }
+ }
+}
+defm i2f : Int2FPInst<"i2f", sint_to_fp, true>;
+defm i2f : Int2FPInst<"i2f", uint_to_fp, false>;
+defm i2f: Int2FPInstRndSat<"i2f", int_pisa_sitofp, true>;
+defm i2f: Int2FPInstRndSat<"i2f", int_pisa_uitofp, false>;
+
+// => insert
+class InsertInst_rr<string name, pisaVT dstVT, pisaVT srcVT>
+ : PISAInst_rr<name, dstVT, dstVT, srcVT> {
+ let AsmString = name#" \t$dst, $src1;";
+}
+multiclass Insert {
+ defvar VTs = [_v2i32, _v3i32, _v4i32, _v5i32, _v6i32, _v7i32, _v8i32, _v16i32, _v32i32, _v64i32];
+ defvar lengths = [2, 3, 4, 5, 6, 7, 8, 16, 32, 64];
+ // For insert, dst must be a vector type.
+ // For each dst vector type...
+ foreach i = !range(VTs) in {
+ defvar dstVT = VTs[i];
+ defvar dstLength = lengths[i];
+ // The range of valid insert indices is [0, ..., length - 1] (i.e., for v4, [0, 1, 2, 3])...
+ foreach index = !range(dstLength) in {
+ // At each valid index, you can always insert a scalar.
+ let Constraints = "$src0 = $dst" in {
+ def "_"#index#"_"#dstVT.name#"_i32_r" : InsertInst_rr<"insert."#index#".32b", dstVT, _i32>;
+ // For each valid source vector type, check if it fits in remaining space
+ foreach j = !range(VTs) in {
+ defvar srcVT = VTs[j];
+ defvar srcLength = lengths[j];
+ // Source vector is valid if srcLength <= (dstLength - index)
+ if !le(srcLength, !sub(dstLength, index)) then {
+ def "_"#index#"_"#dstVT.name#"_"#srcVT.name#"_r" : InsertInst_rr<"insert."#index#srcVT.directive, dstVT, srcVT>;
+ }
+ }
+ }
+ }
+ }
+}
+defset list<Instruction> InsertInsts = {
+ defm insert : Insert;
+}
+foreach II = InsertInsts in {
+ def : Name2InstrEntry<!cast<string>(II), II>;
+}
+
+// => insert.dynamic
+multiclass InsertDynamic {
+ let Constraints = "$src0 = $dst" in {
+ defvar srcVTs = [_v2i32, _v3i32, _v4i32, _v5i32, _v6i32, _v7i32, _v8i32, _v16i32, _v32i32, _v64i32];
+ foreach srcVT = srcVTs in {
+ let AsmString = "insert.dynamic"#srcVT.directive#" \t$dst, $src1, $src2;" in {
+ def "_"#srcVT.name : PISAInst_rrr<"insert.dynamic"#srcVT.directive, srcVT, srcVT, _i32, _i32>;
+ }
+ }
+ }
+}
+defset list<Instruction> InsertDynamicInsts = {
+ defm insert_dynamic : InsertDynamic;
+}
+foreach IDI = InsertDynamicInsts in {
+ def : Name2InstrEntry<!cast<string>(IDI), IDI>;
+}
+
+// => isaddr
+class IsaddrInst_r<string name, Intrinsic intrinsic, pisaVT dstVT, pisaVT srcVT>
+ : UnaryInst_r<name, intrinsic, dstVT, srcVT>;
+// FIXME: Need special selection as the input pointer may be from any address space.
+multiclass IsaddrInst<string name> {
+ def _private: IsaddrInst_r<name#".private", int_pisa_isaddr_private, _i32, _i64>;
+ def _shared : IsaddrInst_r<name#".shared", int_pisa_isaddr_shared, _i32, _i64>;
+ def _global : IsaddrInst_r<name#".global", int_pisa_isaddr_global, _i32, _i64>;
+}
+defm isaddr : IsaddrInst<"isaddr">;
+
+
+// => mov
+class MovInst_i<string name, pisaVT dstVT, pisaVT srcVT> : PISAInst_i<name, dstVT, srcVT> {
+ let Pattern = [(set dstRC:$dst, (srcVT.VT immVal:$src))];
+ let isMoveImm = 1;
+}
+class MovInst_r<string name, pisaVT dstVT, pisaVT srcVT> : PISAInst_r<name, dstVT, srcVT> {
+ let isMoveReg = 1;
+}
+multiclass VectorScalar<string name, pisaIntVT VT1, pisaVectorVT VT2> {
+ defvar directive = VT1.directive;
+ defvar opcode = !subst(".", "_", directive);
+ def VT1.name#_#VT2.name#_r : MovInst_r<name#directive, VT1, VT2>;
+ def VT2.name#_#VT1.name#_r : MovInst_r<name#directive, VT2, VT1>;
+}
+multiclass VectorVector<string name, string bitwidth, pisaVectorVT VT1, pisaVectorVT VT2> {
+ def VT1.name#_#VT2.name#_r : MovInst_r<name#"."#bitwidth, VT1, VT2>;
+ def VT2.name#_#VT1.name#_r : MovInst_r<name#"."#bitwidth, VT2, VT1>;
+}
+multiclass Mov {
+ foreach VT = VTs.MovImm in {
+ defvar directive = "mov."#VT.VT.Size#"b";
+ def _#VT.name#_i: MovInst_i<directive, VT, VT>;
+ }
+ foreach VT = VTs.MovReg in {
+ defvar directive = "mov."#VT.VT.Size#"b";
+ def _#VT.name#_r : MovInst_r<directive, VT, VT>;
+ }
+
+ // 16bit
+ defm _: VectorScalar<"mov", _i16, _v2i8>;
+
+ // 32bit
+ defm _: VectorScalar<"mov", _i32, _v4i8>;
+ defm _: VectorScalar<"mov", _i32, _v2i16>;
+ defm _: VectorVector<"mov", "32b", _v2i16, _v4i8>;
+
+ // 64bit
+ defm _: VectorScalar<"mov", _i64, _v4i16>;
+ defm _: VectorScalar<"mov", _i64, _v2i32>;
+ defm _: VectorVector<"mov", "64b", _v2i32, _v4i16>;
+
+ // 128bit
+ defm _: VectorScalar<"mov", _i128, _v4i32>;
+ defm _: VectorScalar<"mov", _i128, _v2i64>;
+ defm _: VectorVector<"mov", "128b", _v2i64, _v4i32>;
+}
+defm mov : Mov;
+
+// .. ptr G_CONSTANT imm
+def : Pat<(_p0.VT imm:$imm), (mov_i64_i imm:$imm)>;
+def : Pat<(_p1.VT imm:$imm), (mov_i64_i imm:$imm)>;
+def : Pat<(_p2.VT imm:$imm), (mov_i64_i imm:$imm)>;
+def : Pat<(_p3.VT imm:$imm), (mov_i32_i imm:$imm)>;
+def : Pat<(_p4.VT imm:$imm), (mov_i32_i imm:$imm)>;
+
+
+// => sext
+defm sext : ConvertMultiInst<"sext", sext, _i64, [_i8, _i16, _i32]>;
+defm sext : ConvertMultiInst<"sext", sext, _i32, [_i8, _i16]>;
+defm sext : ConvertMultiInst<"sext", sext, _i16, [_i8]>;
+
+// => trunc
+defm trunc : ConvertMultiInst<"trunc", trunc, _i8, [_i16, _i32, _i64]>;
+defm trunc : ConvertMultiInst<"trunc", trunc, _i16, [_i32, _i64]>;
+defm trunc : ConvertMultiInst<"trunc", trunc, _i32, [_i64]>;
+
+// => zext
+defm zext : ConvertMultiInst<"zext", zanyext, _i64, [_i8, _i16, _i32]>;
+defm zext : ConvertMultiInst<"zext", zanyext, _i32, [_i8, _i16]>;
+defm zext : ConvertMultiInst<"zext", zanyext, _i16, [_i8]>;
+// END: data movement and type conversion instructions definitions
+//////
+
+//////
+// BEGIN: comparison and selection instruction definitions
+// ucmp/scmp
+class CmpBinaryInstPred_rr<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_rr<name, _i1, srcVT, srcVT> {
+ let Pattern = [(set i1:$dst, (setcc srcVT.VT:$src0, srcVT.VT:$src1, (OtherVT setcnd)))];
+}
+class CmpBinaryInstPred_ri<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ri<name, _i1, srcVT, srcVT> {
+ let Pattern = [(set i1:$dst, (setcc srcVT.VT:$src0, immVal1:$src1, setcnd))];
+}
+class CmpBinaryInstPred_ir<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ir<name, _i1, srcVT, srcVT> {
+ let Pattern = [(set i1:$dst, (setcc (srcVT.VT immVal0:$src0), srcVT.VT:$src1, setcnd))];
+}
+class CmpBinaryInstPred_ii<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ii<name, _i1, srcVT, srcVT> {
+ let Pattern = [(set i1:$dst, (setcc (srcVT.VT immVal0:$src0), immVal1:$src1, setcnd))];
+}
+
+def selextend : PatFrag<(ops node:$in), (select node:$in, -1, 0)>;
+class CmpBinaryInstReg_rr<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_rr<name, _i32, srcVT, srcVT> {
+ let Pattern = [(set i32:$dst, (selextend (i1 (setcc srcVT.VT:$src0, srcVT.VT:$src1, setcnd))))];
+}
+class CmpBinaryInstReg_ri<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ri<name, _i32, srcVT, srcVT> {
+ let Pattern = [(set i32:$dst, (selextend (i1 (setcc srcVT.VT:$src0, immVal1:$src1, setcnd))))];
+}
+class CmpBinaryInstReg_ir<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ir<name, _i32, srcVT, srcVT> {
+ let Pattern = [(set i32:$dst, (selextend (i1 (setcc (srcVT.VT immVal0:$src0), srcVT.VT:$src1, setcnd))))];
+}
+class CmpBinaryInstReg_ii<string name, CondCode setcnd, pisaVT srcVT>: PISAInst_ii<name, _i32, srcVT, srcVT> {
+ let Pattern = [(set i32:$dst, (selextend (i1 (setcc (srcVT.VT immVal0:$src0), immVal1:$src1, setcnd))))];
+}
+multiclass CmpInst<string name, list<pisaVT> types, bit isSigned> {
+ defvar prefix = !if(isSigned, "", "u");
+ foreach cc = ["ne", "eq", "gt", "ge", "lt", "le"] in {
+ defvar ccName = !toupper(!if(!or(!eq(cc, "ne"), !eq(cc, "eq")), cc, prefix#cc));
+ defvar cond = !cast<CondCode>("SET"#ccName);
+ foreach VT = types in {
+ defvar directive = "."#cc#VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+
+ // Defs for cmp opcodes where dst is a predicate
+ // cmp is directly matched, e.g.
+ // %2 = icmp slt i32 %0, %1
+ def opcode#"_prr" : CmpBinaryInstPred_rr<name#directive, cond, VT>;
+ def opcode#"_pri" : CmpBinaryInstPred_ri<name#directive, cond, VT>;
+ def opcode#"_pir" : CmpBinaryInstPred_ir<name#directive, cond, VT>;
+ def opcode#"_pii" : CmpBinaryInstPred_ii<name#directive, cond, VT>;
+
+ // Defs for cmp opcodes where dst is a 32b register
+ // cmp+sext is matched, e.g.
+ // %2 = icmp slt i32 %0, %1
+ // %3 = sext i1 %2 to i32
+ def opcode#"_rrr" : CmpBinaryInstReg_rr<name#directive, cond, VT>;
+ def opcode#"_rri" : CmpBinaryInstReg_ri<name#directive, cond, VT>;
+ def opcode#"_rir" : CmpBinaryInstReg_ir<name#directive, cond, VT>;
+ def opcode#"_rii" : CmpBinaryInstReg_ii<name#directive, cond, VT>;
+ }
+ }
+}
+
+// => ucmp
+defm ucmp : CmpInst<"ucmp", [_i16, _i32, _i64], 0>;
+
+// => scmp
+defm scmp : CmpInst<"scmp", [_i16, _i32, _i64], 1>;
+
+// => fcmp
+multiclass FCmpInst<string name, list<pisaVT> types> {
+ foreach cc = ["oge", "ogt", "ole", "olt", "oeq", "une"] in {
+ defvar cond = !substr(cc, 1);
+ defvar node = !cast<CondCode>("SET"#!toupper(cc));
+ foreach ftz = [omitFTZMod, usesFTZMod] in {
+ foreach VT = types in {
+ defvar directive = !interleave(["."#cond, ftz.name, VT.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+
+ // Defs for fcmp opcodes where dst is a predicate
+ // fcmp is directly matched, e.g.
+ // %2 = fcmp olt f32 %0, %1
+ def opcode#"_prr" : CmpBinaryInstPred_rr<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_pri" : CmpBinaryInstPred_ri<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_pir" : CmpBinaryInstPred_ir<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_pii" : CmpBinaryInstPred_ii<name#directive, node, VT>, Requires<[ftz.pred]>;
+
+ // Defs for fcmp opcodes where dst is a 32b register
+ // fcmp+sext is matched, e.g.
+ // %2 = fcmp olt i32 %0, %1
+ // %3 = sext i1 %2 to i32
+ def opcode#"_rrr" : CmpBinaryInstReg_rr<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rri" : CmpBinaryInstReg_ri<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rir" : CmpBinaryInstReg_ir<name#directive, node, VT>, Requires<[ftz.pred]>;
+ def opcode#"_rii" : CmpBinaryInstReg_ii<name#directive, node, VT>, Requires<[ftz.pred]>;
+ }
+ }
+ }
+}
+defm fcmp : FCmpInst<"fcmp", [_bf16, _f16, _f32, _f64]>;
+
+
+// => sel
+class SelInst_rrp<string name, pisaVT VT> : PISAInst_rr<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $p;";
+ let InOperandList = (ins PredRC:$p, src0RC:$src0, src1RC:$src1);
+ let Pattern = [(set VT.VT:$dst, (select i1:$p, VT.VT:$src0, src1RC:$src1))];
+}
+class SelInst_rip<string name, pisaVT VT> : PISAInst_ri<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $p;";
+ let InOperandList = (ins PredRC:$p, src0RC:$src0, immOpd1:$src1);
+ let Pattern = [(set VT.VT:$dst, (select i1:$p, VT.VT:$src0, immVal1:$src1))];
+}
+class SelInst_irp<string name, pisaVT VT> : PISAInst_ir<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $p;";
+ let InOperandList = (ins PredRC:$p, immOpd0:$src0, src1RC:$src1);
+ let Pattern = [(set VT.VT:$dst, (select i1:$p, immVal0:$src0, VT.VT:$src1))];
+}
+class SelInst_iip<string name, pisaVT VT> : PISAInst_ii<name, VT, VT, VT> {
+ DAGOperand PredRC = _i1.RegOpnd;
+ let AsmString = name#" \t$dst, $src0, $src1, $p;";
+ let InOperandList = (ins PredRC:$p, immOpd0:$src0, immOpd1:$src1);
+ let Pattern = [(set VT.VT:$dst, (select i1:$p, immVal0:$src0, immVal1:$src1))];
+}
+class SelInst_rrr<string name, pisaVT VT, pisaVT predVT> : PISAInst_rrr<name, VT, VT, VT, predVT> {
+ let Pattern = [(set VT.VT:$dst, (pisa_select predVT.VT:$src2, VT.VT:$src0, VT.VT:$src1))];
+}
+class SelInst_rir<string name, pisaVT VT, pisaVT predVT> : PISAInst_rir<name, VT, VT, VT, predVT> {
+ let Pattern = [(set VT.VT:$dst, (pisa_select predVT.VT:$src2, VT.VT:$src0, immVal1:$src1))];
+}
+class SelInst_irr<string name, pisaVT VT, pisaVT predVT> : PISAInst_irr<name, VT, VT, VT, predVT> {
+ let Pattern = [(set VT.VT:$dst, (pisa_select predVT.VT:$src2, immVal0:$src0, VT.VT:$src1))];
+}
+class SelInst_iir<string name, pisaVT VT, pisaVT predVT> : PISAInst_iir<name, VT, VT, VT, predVT> {
+ let Pattern = [(set VT.VT:$dst, (pisa_select predVT.VT:$src2, immVal0:$src0, immVal1:$src1))];
+}
+multiclass SelInst<string name> {
+ foreach VT = [_i16, _i32, _i64, _bf16, _f16, _f32, _f64] in {
+ defvar directive = "."#VT.bitwidth;
+ defvar opcode = VT.opcode;
+
+ // select on predicate
+ def opcode#"_rrp" : SelInst_rrp<name#directive, VT>;
+ def opcode#"_irp" : SelInst_irp<name#directive, VT>;
+ def opcode#"_rip" : SelInst_rip<name#directive, VT>;
+ def opcode#"_iip" : SelInst_iip<name#directive, VT>;
+ // select on register
+ defvar predVT = !if(!eq(VT.bitwidth,"16b"), _i16, !if(!eq(VT.bitwidth,"32b"), _i32, _i64));
+ def opcode#"_rrr"#predVT.bitwidth : SelInst_rrr<name#directive, VT, predVT>;
+ def opcode#"_irr"#predVT.bitwidth : SelInst_irr<name#directive, VT, predVT>;
+ def opcode#"_rir"#predVT.bitwidth : SelInst_rir<name#directive, VT, predVT>;
+ def opcode#"_iir"#predVT.bitwidth : SelInst_iir<name#directive, VT, predVT>;
+ }
+}
+class SelInst_rrnp<string name, pisaVT VT> : PISAInst_rr<name, VT, VT, VT> {
+ let AsmString = name#" \t$dst, $src0, $src1, $negate$p;";
+ let InOperandList = (ins Negate:$negate, _i1.RegOpnd:$p, src0RC:$src0, src1RC:$src1);
+}
+class SelInst_rinp<string name, pisaVT VT> : PISAInst_ri<name, VT, VT, VT> {
+ let AsmString = name#" \t$dst, $src0, $src1, $negate$p;";
+ let InOperandList = (ins Negate:$negate, _i1.RegOpnd:$p, src0RC:$src0, immOpd1:$src1);
+}
+class SelInst_irnp<string name, pisaVT VT> : PISAInst_ir<name, VT, VT, VT> {
+ let AsmString = name#" \t$dst, $src0, $src1, $negate$p;";
+ let InOperandList = (ins Negate:$negate, _i1.RegOpnd:$p, immOpd0:$src0, src1RC:$src1);
+}
+class SelInst_iinp<string name, pisaVT VT> : PISAInst_ii<name, VT, VT, VT> {
+ let AsmString = name#" \t$dst, $src0, $src1, $negate$p;";
+ let InOperandList = (ins Negate:$negate, _i1.RegOpnd:$p, immOpd0:$src0, immOpd1:$src1);
+}
+multiclass InvSelInst<string name> {
+ foreach VT = [_i16, _i32, _i64, _bf16, _f16, _f32, _f64] in {
+ defvar directive = VT.directive;
+ defvar opcode = !subst(".", "_", directive);
+
+ // select on negative predicate
+ def opcode#"_rrnp" : SelInst_rrnp<name#directive, VT>;
+ def opcode#"_irnp" : SelInst_irnp<name#directive, VT>;
+ def opcode#"_rinp" : SelInst_rinp<name#directive, VT>;
+ def opcode#"_iinp" : SelInst_iinp<name#directive, VT>;
+ }
+}
+defm sel : SelInst<"sel">;
+defm sel : InvSelInst<"sel">;
+
+// emit patterns to match the pointer variants for sel and cmp
+// e.g.,
+// %2:registers(s1) = G_ICMP intpred(eq), %0(p2), %1
+// %3:reg64b(p2) = G_SELECT %2(s1), %0, %1
+
+foreach VT = VTs.Ptr in {
+ defvar selop = !cast<Instruction>("sel_"#VT.VT.Size#"_rrp");
+ def : Pat<(select i1:$p, VT.VT:$src0, VT.VT:$src1),
+ (selop Pred:$p, VT.RC:$src0, VT.RC:$src1)>;
+ foreach isSigned = [0, 1] in {
+ defvar opcode = !if(isSigned, "scmp", "ucmp");
+ defvar prefix = !if(isSigned, "", "u");
+ foreach cc = ["ne", "eq", "gt", "ge", "lt", "le"] in {
+ defvar ccName = !toupper(!if(!or(!eq(cc, "ne"), !eq(cc, "eq")), cc, prefix#cc));
+ defvar cond = !cast<CondCode>("SET"#ccName);
+ defvar cmpop = !cast<Instruction>(opcode#"_"#cc#VT.opcode#"b_prr");
+ def : Pat<(setcc VT.VT:$src0, VT.VT:$src1, cond),
+ (cmpop VT.RC:$src0, VT.RC:$src1)>;
+ }
+ }
+}
+// END: comparison and selection instruction definitions
+//////
+
+//////
+// BEGIN: control flow instruction definitions
+// => call
+let isCall = 1 in {
+ // Define function call instructions with a FunctionCallTargetOpnd:
+ def functionCall_void: PISAInst<(outs),
+ (ins FunctionCallTargetOpnd:$func, variable_ops),
+ "call void, $func">;
+ foreach VT = VTs.CallReturn in {
+ def "functionCall_"#VT.name#"_r": PISAInst<(outs VT.RegOpnd:$ret),
+ (ins FunctionCallTargetOpnd:$func, variable_ops),
+ "call $ret, $func">;
+ }
+ // Define function call instructions with a register call target:
+ def "indirectFunctionCall_void_r": PISAInst<(outs),
+ (ins IndirectFunctionCallTargetOpnd:$func, variable_ops),
+ "call void, $func">;
+ foreach VT = VTs.CallReturn in {
+ def "indirectFunctionCall_"#VT.name#"_r_i64_r": PISAInst<(outs VT.RegOpnd:$ret),
+ (ins IndirectFunctionCallTargetOpnd:$func, variable_ops),
+ "call $ret, $func">;
+ }
+}
+
+// => call.cond
+let isCall=1, isTerminator=1 in {
+ // Define function call instructions with a FunctionCallTargetOpnd:
+ def predCall_void: PISAInst<(outs),
+ (ins Negate:$negate, _i1.RegOpnd:$p, FunctionCallTargetOpnd:$func, variable_ops),
+ "call.cond $negate$p, void, $func">;
+ foreach VT = VTs.CallReturn in {
+ def "predCall_"#VT.name#"_r": PISAInst<(outs VT.RegOpnd:$ret),
+ (ins Negate:$negate, _i1.RegOpnd:$p, FunctionCallTargetOpnd:$func, variable_ops),
+ "call.cond $negate$p, $ret, $func">;
+ }
+ // Define function call instructions with a register call target:
+ def "indirectPredCall_void_r": PISAInst<(outs),
+ (ins Negate:$negate, _i1.RegOpnd:$p, IndirectFunctionCallTargetOpnd:$func, variable_ops),
+ "call.cond $negate$p, void, $func">;
+ foreach VT = VTs.CallReturn in {
+ def "indirectPredCall_"#VT.name#"_r_i64_r": PISAInst<(outs VT.RegOpnd:$ret),
+ (ins Negate:$negate, _i1.RegOpnd:$p, IndirectFunctionCallTargetOpnd:$func, variable_ops),
+ "call.cond $negate$p, $ret, $func">;
+ }
+}
+
+
+// => goto
+def BrTargetOpnd : PISASymbolOperand<OtherVT, "BrTargetOpnd">;
+def : GINodeEquiv<G_BRCOND, brcond>;
+
+let isTerminator=1, isBranch=1 in {
+ def gotolabel : PISAInst<(outs), (ins BrTargetOpnd:$label),
+ "goto $label;",
+ [(br bb:$label)]> {
+ let isBarrier = 1;
+ let Defs = [SpecialReg_ActiveMask];
+ }
+ // FIXME: Need refactoring and adding a complex predicate operand, which is
+ // composed of a negate operand and a predicate register.
+ def predgoto : PISAInst<(outs), (ins Negate:$negate, _i1.RegOpnd:$cond, BrTargetOpnd:$label),
+ "goto.cond $negate$cond, $label;",
+ [(brcond i1:$cond, bb:$label)]> {
+ let Defs = [SpecialReg_ActiveMask];
+ }
+}
+
+// => return
+class SimpleOp<string name, list<dag> pattern = []> : PISAInst<(outs), (ins), name, pattern>;
+multiclass RetVal<string name> {
+ foreach VT = VTs.CallReturn in {
+ def "_"#VT.name#"_r": PISAInst<(outs), (ins VT.RegOpnd:$value), name#" $value;">;
+ }
+ // support return of immediates
+ foreach VT = VTs.Integer in {
+ def "_"#VT.name#"_i": PISAInst<(outs), (ins VT.ImmOpnd:$value), name#" $value;">;
+ }
+ foreach VT = VTs.Float in {
+ def "_"#VT.name#"_i": PISAInst<(outs), (ins VT.ImmOpnd:$value), name#" $value;">;
+ }
+}
+let isReturn = 1, isBarrier = 1, isTerminator = 1, isNotDuplicable = 1 in {
+ def ret: SimpleOp<"return;">;
+ defm retValue: RetVal<"return">;
+}
+
+// => return.cond
+multiclass PredRet<string name> {
+ foreach VT = VTs.CallReturn in {
+ def "_"#VT.name#"_r": PISAInst<(outs), (ins Negate:$negate, _i1.RegOpnd:$cond, VT.RegOpnd:$value), name#" $negate$cond, $value;">;
+ }
+}
+let isReturn = 1, isBarrier = 1, isTerminator=1, isBranch=1 in {
+ defm predRet: PredRet<"return.cond">;
+}
+// END: control flow instruction definitions
+//////
+
+//////
+// BEGIN: subgroup communication instruction definitions
+// ired/fred
+class Red_rrr<string name, SDPatternOperator node, TImmLeaf opNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rrr<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT src1RC:$src1), (src0VT.VT src2RC:$src2)))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+class Red_rir<string name, SDPatternOperator node, TImmLeaf opNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rir<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT imm:$src1), (src0VT.VT src2RC:$src2)))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+class Red_rir_bf<string name, SDPatternOperator node, TImmLeaf opNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rir<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT imm:$src1), (src0VT.VT src2RC:$src2)))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+class Red_rir_bf_nanp<string name, SDPatternOperator node, TImmLeaf opNode, TImmLeaf nanpNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rir<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT imm:$src1), (src0VT.VT src2RC:$src2), nanpNode))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+// fred 5-operand variants - match int_pisa_fred with nanp ImmArg.
+class Red_rrr_nanp<string name, SDPatternOperator node, TImmLeaf opNode, TImmLeaf nanpNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rrr<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT src1RC:$src1), (src0VT.VT src2RC:$src2), nanpNode))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+class Red_rir_nanp<string name, SDPatternOperator node, TImmLeaf opNode, TImmLeaf nanpNode, pisaVT src0VT, pisaVT src1VT>
+ : PISAInst_rir<name, src0VT, src0VT, src1VT, src0VT> {
+ let Pattern = [(set (src0VT.VT dstRC:$dst), (node opNode, (src0VT.VT src0RC:$src0), (src1VT.VT imm:$src1), (src0VT.VT src2RC:$src2), nanpNode))];
+ let AsmString = name#" \t$dst, $src0, $src1;";
+ let Constraints = "$src2 = $dst";
+}
+
+// => fred
+multiclass FRedInst<string name> {
+ foreach opNode = FRedOpNodes in {
+ // scalar bf16/f16/f32: loop over AllNanpMods to emit nanp=0 and nanp=1 variants.
+ // Directive: fred.<op>[.nanp].<type>
+ foreach nanp = AllNanpMods in {
+ foreach type = [_bf16, _f16, _f32] in {
+ defvar directive = !interleave([opNode.name, nanp.name, type.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr": Red_rrr_nanp<name#directive, int_pisa_fred, opNode, nanp.node, type, _i32>;
+ def opcode#"_rir": Red_rir_nanp<name#directive, int_pisa_fred, opNode, nanp.node, type, _i32>;
+ }
+ }
+ }
+}
+let isConvergent = 1 in {
+ defm fred: FRedInst<"fred">;
+}
+
+// => ired
+multiclass IRedInst<string name> {
+ foreach opNode = IRedOpNodes in {
+ // defvar red = IRedOpMode<opNode>.name;
+ foreach type = [_i16, _i32] in {
+ defvar directive = !interleave([opNode.name, type.directive], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrr": Red_rrr<name#directive, int_pisa_ired, opNode, type, _i32>;
+ def opcode#"_rir": Red_rir<name#directive, int_pisa_ired, opNode, type, _i32>;
+ }
+ }
+}
+let isConvergent = 1 in {
+ defm ired: IRedInst<"ired">;
+}
+
+// => redfirstidx
+let isConvergent = 1 in {
+ defm redfirstidx: UnaryInst<"redfirstidx", int_pisa_redfirstidx, types=[_i32]>;
+}
+
+// => shfl
+class Shuffle_rrrr<string name, TImmLeaf modeNode, TImmLeaf sgNode, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (int_pisa_shfl modeNode, src0RC:$src0, src1RC:$src1, src2RC:$src2, src3RC:$src3, sgNode))];
+ let AsmString = name#" \t$dst, $src0, $src1, $src2;";
+ let Constraints = "$src3 = $dst";
+}
+class Shuffle_rrir<string name, TImmLeaf modeNode, TImmLeaf sgNode, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, src1RC:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (int_pisa_shfl modeNode, src0RC:$src0, src1RC:$src1, (VT.VT imm:$src2), src2RC:$src3, sgNode))];
+ let AsmString = name#" \t$dst, $src0, $src1, $src2;";
+ let Constraints = "$src3 = $dst";
+}
+class Shuffle_rirr<string name, TImmLeaf modeNode, TImmLeaf sgNode, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, src2RC:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (int_pisa_shfl modeNode, src0RC:$src0, (VT.VT imm:$src1), src2RC:$src2, src3RC:$src3, sgNode))];
+ let AsmString = name#" \t$dst, $src0, $src1, $src2;";
+ let Constraints = "$src3 = $dst";
+}
+class Shuffle_riir<string name, TImmLeaf modeNode, TImmLeaf sgNode, pisaVT VT>
+ : PISAInst4<name, VT, VT, VT, VT, VT> {
+ let InOperandList = (ins src0RC:$src0, VT.ImmOpnd:$src1, VT.ImmOpnd:$src2, src3RC:$src3);
+ let Pattern = [(set dstRC:$dst, (int_pisa_shfl modeNode, src0RC:$src0, (VT.VT imm:$src1), (VT.VT imm:$src2), src3RC:$src3, sgNode))];
+ let AsmString = name#" \t$dst, $src0, $src1, $src2;";
+ let Constraints = "$src3 = $dst";
+}
+multiclass ShflInst<string name> {
+ foreach modeNode = SHFLModeNodes in {
+ foreach sgNode = [timm_false, timm_true] in {
+ defvar sg = !if(!eq(!cast<string>(sgNode), !cast<string>(timm_true)), ".sg", "");
+ defvar directive = !interleave([modeNode.name, sg, ".32b"], "");
+ defvar opcode = !subst(".", "_", directive);
+ def opcode#"_rrrr" : Shuffle_rrrr<name#directive, modeNode, sgNode, _i32>;
+ def opcode#"_rrir" : Shuffle_rrir<name#directive, modeNode, sgNode, _i32>;
+ def opcode#"_rirr" : Shuffle_rirr<name#directive, modeNode, sgNode, _i32>;
+ def opcode#"_riir" : Shuffle_riir<name#directive, modeNode, sgNode, _i32>;
+ }
+ }
+}
+let isConvergent = 1 in {
+ defm shfl: ShflInst<"shfl">;
+}
+// END: subgroup communication instruction definitions
+//////
+
+//////
+// BEGIN: memory instruction definitions
+
+// => fatom
+defm fatomic_load_add : binary_atomic_op_fp<atomic_load_fadd>;
+defm fatomic_load_sub : binary_atomic_op_fp<atomic_load_fsub>;
+defm fatomic_load_min : binary_atomic_op_fp<atomic_load_fmin>;
+defm fatomic_load_max : binary_atomic_op_fp<atomic_load_fmax>;
+
+// 2 source operands
+multiclass BinaryFloatAtomicInst<string Op> {
+ foreach AS = [global, shared, generic] in {
+ foreach VT = [_bf16, _f16, _f32, _f64] in {
+ // fatom.[shared|generic]* are not allowed for f64
+ defvar invalid0 = !and(!or(!eq(AS, shared), !eq(AS, generic)), !eq(VT, _f64));
+ // fatom.*.[min|max] are not allowed for f64
+ defvar invalid1 = !and(!or(!eq(Op, "min"), !eq(Op, "max")), !eq(VT, _f64));
+ defvar invalidOperations = !or(invalid0, invalid1);
+ // GlobalISel only checks memory size, not type, when matching MemoryVT.
+ // bf16 and f16 are both 16-bit, so we need a GISelPredicateCode to
+ // disambiguate them by checking the register type of the result operand.
+ defvar needsMemVTCheck = !or(!eq(VT, _bf16), !eq(VT, _f16));
+ defvar memVTCheck = !if(!eq(VT, _bf16),
+ "MRI.getType(MI.getOperand(0).getReg()).isBFloat16()",
+ "MRI.getType(MI.getOperand(0).getReg()).isFloat16()");
+ if !not(invalidOperations) then {
+ foreach MO = ["monotonic", "release", "acquire", "acq_rel"] in {
+ defvar fullName = !strconcat("fatomic_load_", Op, "_"#VT.VT, "_", MO);
+ defvar node = !cast<PatFrag>(fullName);
+ defvar MOStr = AtomicMemOrder<MO>.name;
+
+ defvar opcode = "_"#VT.name#"_"#AS.name#"_"#MO#"_"#Op;
+ defvar directive = "fatom."#AS.name#"."#MOStr;
+ if needsMemVTCheck then {
+ let Pattern = [(set VT.VT:$dst,
+ (BinaryAtomAddrspaceMemVTPat<AS, node, memVTCheck>
+ PISAMem_ri<AS>.Pattern:$addr, VT.VT:$src0))] in {
+ def opcode#"_ri" : PISABinaryAtom_ri<directive, Op, AS, VT, node>;
+ }
+ let Pattern = [(set VT.VT:$dst,
+ (BinaryAtomAddrspaceMemVTPat<AS, node, memVTCheck>
+ PISAMem_rr<AS>.Pattern:$addr, VT.VT:$src0))] in {
+ def opcode#"_rr" : PISABinaryAtom_rr<directive, Op, AS, VT, node>;
+ }
+ } else {
+ def opcode#"_ri" : PISABinaryAtom_ri<directive, Op, AS, VT, node>;
+ def opcode#"_rr" : PISABinaryAtom_rr<directive, Op, AS, VT, node>;
+ }
+ if !eq(MO, "monotonic") then {
+ // .memorder is optional
+ defvar opcodeA = "_"#VT.name#"_"#AS.name#"_"#Op;
+ defvar directiveA = "fatom."#AS.name;
+ if needsMemVTCheck then {
+ let Pattern = [(set VT.VT:$dst,
+ (BinaryAtomAddrspaceMemVTPat<AS, node, memVTCheck>
+ PISAMem_ri<AS>.Pattern:$addr, VT.VT:$src0))] in {
+ def opcodeA#"_ri" : PISABinaryAtom_ri<directiveA, Op, AS, VT, node>;
+ }
+ let Pattern = [(set VT.VT:$dst,
+ (BinaryAtomAddrspaceMemVTPat<AS, node, memVTCheck>
+ PISAMem_rr<AS>.Pattern:$addr, VT.VT:$src0))] in {
+ def opcodeA#"_rr" : PISABinaryAtom_rr<directiveA, Op, AS, VT, node>;
+ }
+ } else {
+ def opcodeA#"_ri" : PISABinaryAtom_ri<directiveA, Op, AS, VT, node>;
+ def opcodeA#"_rr" : PISABinaryAtom_rr<directiveA, Op, AS, VT, node>;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+defm atomic_load_fadd_impl : BinaryFloatAtomicInst<"add">;
+defm atomic_load_fsub_impl : BinaryFloatAtomicInst<"sub">;
+defm atomic_load_fmin_impl : BinaryFloatAtomicInst<"min">;
+defm atomic_load_fmax_impl : BinaryFloatAtomicInst<"max">;
+
+// fatom.*.cas.* intrinsic
+class FAtomCasAddrspacePat<AddressSpace Addrspace, SDPatternOperator pat> :
+ PatFrag<(ops node:$ptr, node:$opnd1, node:$opnd2, node:$opnd3), (pat node:$ptr, node:$opnd1, node:$opnd2, node:$opnd3)> {
+ let AddressSpaces = [ Addrspace.num ];
+ let IsLoad = true;
+ let IsStore = true;
+}
+class FAtomCasInst_rri<string directive, string op, AddressSpace AS, pisaVT VT, SDPatternOperator node, SDPatternOperator moNode>
+ : PISATernaryAtom<directive, op, AS, VT, PISAMem_rr<AS>, node> {
+ let Pattern = [(set VT.VT:$dst, (FAtomCasAddrspacePat<AS, node> ADDR_rr:$addr, VT.VT:$src0, VT.VT:$src1, moNode))];
+}
+class FAtomCasInst_rii<string directive, string op, AddressSpace AS, pisaVT VT, SDPatternOperator node, SDPatternOperator moNode>
+ : PISATernaryAtom<directive, op, AS, VT, PISAMem_ri<AS>, node> {
+ let Pattern = [(set VT.VT:$dst, (FAtomCasAddrspacePat<AS, node> ADDR_ri:$addr, VT.VT:$src0, VT.VT:$src1, moNode))];
+}
+multiclass FAtomCasInst<string name> {
+ defvar Op = "cas";
+ foreach AS = [global, shared, generic] in {
+ foreach VT = [_bf16, _f16, _f32] in {
+ foreach moNode = MemoryOrderNodes in {
+ defvar directive = !interleave([AS.directive, moNode.name], "");
+ defvar opcode = !subst(".", "_", !interleave([VT.opcode, AS.directive, moNode.name, "_", Op], ""));
+ def opcode#"_rri" : FAtomCasInst_rri<name#directive, Op, AS, VT, int_pisa_cas_fatom, moNode>;
+ def opcode#"_rii" : FAtomCasInst_rii<name#directive, Op, AS, VT, int_pisa_cas_fatom, moNode>;
+ if !eq(moNode.name, ".relaxed") then {
+ // .memorder is optional
+ defvar directiveA = AS.directive;
+ defvar opcodeA = !subst(".", "_", !interleave([VT.opcode, AS.directive, "_", Op], ""));
+ def opcodeA#"_rri" : FAtomCasInst_rri<name#directiveA, Op, AS, VT, int_pisa_cas_fatom, moNode>;
+ def opcodeA#"_rii" : FAtomCasInst_rii<name#directiveA, Op, AS, VT, int_pisa_cas_fatom, moNode>;
+ }
+ }
+ }
+ }
+}
+defm fatom : FAtomCasInst<"fatom">;
+
+// => iatom
+multiclass load_atomic_op_ord {
+ def NAME#_monotonic : PatFrag<(ops node:$ptr), (!cast<SDPatternOperator>(NAME) node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingMonotonic = true;
+ }
+ def NAME#_acquire : PatFrag<(ops node:$ptr), (!cast<SDPatternOperator>(NAME) node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingAcquire = true;
+ }
+ def NAME#_release : PatFrag<(ops node:$ptr), (!cast<SDPatternOperator>(NAME) node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingRelease = true;
+ }
+ def NAME#_acq_rel : PatFrag<(ops node:$ptr), (!cast<SDPatternOperator>(NAME) node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingAcquireRelease = true;
+ }
+ def NAME#_seq_cst : PatFrag<(ops node:$ptr), (!cast<SDPatternOperator>(NAME) node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingSequentiallyConsistent = true;
+ }
+}
+class IAtomOpName<string Op> {
+ string name = !cond(!eq(Op, "min"): "smin",
+ !eq(Op, "max"): "smax",
+ !eq(Op, "swap"): "xchg",
+ true : Op);
+}
+
+def atomic_swap_i128 : PatFrag<(ops node:$ptr, node:$val), (atomic_swap node:$ptr, node:$val)> {
+ let IsAtomic = true;
+ let MemoryVT = i128;
+ }
+defm atomic_swap_i128 : binary_atomic_op_ord;
+def atomic_cmp_swap_i128 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+ (atomic_cmp_swap node:$ptr, node:$cmp, node:$val)> {
+ let IsAtomic = true;
+ let MemoryVT = i128;
+ }
+defm atomic_cmp_swap_i128 : ternary_atomic_op_ord;
+
+// Define atomic inc/dec as atomic add/sub with imm 1
+multiclass atomic_load_incdec<SDNode atomic_op> {
+ foreach vt = [i16, i32, i64] in {
+ def _#vt : PatFrag<(ops node:$ptr), (atomic_op node:$ptr, 1)> {
+ let IsAtomic = true;
+ let MemoryVT = vt;
+ }
+ defm NAME#_#vt : load_atomic_op_ord;
+ }
+}
+defm atomic_load_inc : atomic_load_incdec<atomic_load_add>;
+defm atomic_load_dec : atomic_load_incdec<atomic_load_sub>;
+
+// 1 source operand
+multiclass UnaryAtomicInst<string Op> {
+ foreach AS = [global, shared, generic] in {
+ defvar VTs = !if(!eq(AS, global), [_i16, _i32, _i64], [_i16, _i32]);
+ foreach VT = VTs in {
+ foreach MO = ["monotonic", "release", "acquire", "acq_rel"] in {
+ defvar fullName = NAME#"_"#VT.VT#"_"#MO;
+ defvar node = !cast<PatFrag>(fullName);
+ defvar MOStr = AtomicMemOrder<MO>.name;
+ defvar OpStr = IAtomOpName<Op>.name;
+
+ defvar opcode = "_"#VT.name#"_"#AS.name#"_"#MO#"_"#Op;
+ defvar directive = "iatom."#AS.name#"."#MOStr;
+ def opcode#"_ri" : PISAUnaryAtom_ri<directive, OpStr, AS, VT, node>;
+ def opcode#"_rr" : PISAUnaryAtom_rr<directive, OpStr, AS, VT, node>;
+ if !eq(MO, "monotonic") then {
+ // .memorder is optional
+ defvar opcodeA = "_"#VT.name#"_"#AS.name#"_"#Op;
+ defvar directiveA = "iatom."#AS.name;
+ def opcodeA#"_ri" : PISAUnaryAtom_ri<directiveA, OpStr, AS, VT, node>;
+ def opcodeA#"_rr" : PISAUnaryAtom_rr<directiveA, OpStr, AS, VT, node>;
+ }
+ }
+ }
+ }
+}
+
+// 2 source operands
+defm atomic_load_incwrap : binary_atomic_op<atomic_load_uinc_wrap>;
+defm atomic_load_decwrap : binary_atomic_op<atomic_load_udec_wrap>;
+
+multiclass BinaryAtomicInst<string Op> {
+ foreach AS = [global, shared, generic] in {
+ foreach VT = [_i16, _i32, _i64, _i128] in {
+ // i128 is only allowed for "swap" op.
+ defvar invalid0 = !and(!eq(VT, _i128), !ne(Op, "swap"));
+ // iatom.[shared|generic].<non-cas-op> is only allowed for i16, i32
+ defvar invalid1 = !and(!or(!eq(AS, shared), !eq(AS, generic)), !eq(VT, _i64));
+ // iatom*.[inc|dec]wrap is only allowed for i32, i64
+ defvar invalid2 = !and(!or(!eq(Op, "incwrap"), !eq(Op, "decwrap")), !eq(VT, _i16));
+ defvar invalidOperations = !or(invalid0, invalid1, invalid2);
+ if !not(invalidOperations) then {
+ foreach MO = ["monotonic", "release", "acquire", "acq_rel"] in {
+ defvar opPrefix = !if(!eq(Op,"swap"), "atomic_", "atomic_load_");
+ defvar fullName = !strconcat(opPrefix , Op, "_"#VT.VT , "_", MO);
+ defvar node = !cast<PatFrag>(fullName);
+ defvar MOStr = AtomicMemOrder<MO>.name;
+ defvar OpStr = IAtomOpName<Op>.name;
+
+ defvar opcode = "_"#VT.name#"_"#AS.name#"_"#MO#"_"#Op;
+ defvar directive = "iatom."#AS.name#"."#MOStr;
+ def opcode#"_ri" : PISABinaryAtom_ri<directive, OpStr, AS, VT, node>;
+ def opcode#"_rr" : PISABinaryAtom_rr<directive, OpStr, AS, VT, node>;
+ if !eq(MO, "monotonic") then {
+ // .memorder is optional
+ defvar opcodeA = "_"#VT.name#"_"#AS.name#"_"#Op;
+ defvar directiveA = "iatom."#AS.name;
+ def opcodeA#"_ri" : PISABinaryAtom_ri<directiveA, OpStr, AS, VT, node>;
+ def opcodeA#"_rr" : PISABinaryAtom_rr<directiveA, OpStr, AS, VT, node>;
+ }
+ }
+ }
+ }
+ }
+}
+
+// 3 source operands
+multiclass TernaryAtomicInst<string Op> {
+ foreach AS = [global, shared, generic] in {
+ foreach VT = [_i16, _i32, _i64, _i128] in {
+ foreach MO = ["monotonic", "release", "acquire", "acq_rel"] in {
+ defvar fullName = !strconcat("atomic_", Op, "_"#VT.VT , "_", MO);
+ defvar node = !cast<SDPatternOperator>(fullName);
+ defvar MOStr = AtomicMemOrder<MO>.name;
+
+ defvar opcode = "_"#VT.name#"_"#AS.name#"_"#MO#"_"#Op;
+ defvar directive = "iatom."#AS.name#"."#MOStr;
+ def opcode#"_ri" : PISATernaryAtom_ri<directive, "cas", AS, VT, node>;
+ def opcode#"_rr" : PISATernaryAtom_rr<directive, "cas", AS, VT, node>;
+ if !eq(MO, "monotonic") then {
+ // .memorder is optional
+ defvar opcodeA = "_"#VT.name#"_"#AS.name#"_"#Op;
+ defvar directiveA = "iatom."#AS.name;
+ def opcodeA#"_ri" : PISATernaryAtom_ri<directiveA, "cas", AS, VT, node>;
+ def opcodeA#"_rr" : PISATernaryAtom_rr<directiveA, "cas", AS, VT, node>;
+ }
+ }
+ }
+ }
+}
+
+defm atomic_load_inc : UnaryAtomicInst<"inc">;
+defm atomic_load_dec : UnaryAtomicInst<"dec">;
+
+defm atomic_load_incwrap_impl : BinaryAtomicInst<"incwrap">;
+defm atomic_load_decwrap_impl : BinaryAtomicInst<"decwrap">;
+defm atomic_load_add_impl : BinaryAtomicInst<"add">;
+defm atomic_load_sub_impl : BinaryAtomicInst<"sub">;
+defm atomic_load_and_impl : BinaryAtomicInst<"and">;
+defm atomic_load_or_impl : BinaryAtomicInst<"or">;
+defm atomic_load_xor_impl : BinaryAtomicInst<"xor">;
+defm atomic_load_min_impl : BinaryAtomicInst<"min">;
+defm atomic_load_max_impl : BinaryAtomicInst<"max">;
+defm atomic_load_umin_impl : BinaryAtomicInst<"umin">;
+defm atomic_load_umax_impl : BinaryAtomicInst<"umax">;
+defm atomic_swap_impl : BinaryAtomicInst<"swap">;
+
+// TODO: llvm's cmpxchg instruction takes 2 memory orderings. Why?
+defm atomic_cmp_swap : TernaryAtomicInst<"cmp_swap">;
+
+// => ld
+multiclass load_atomic_op<SDNode atomic_op> {
+ foreach vt = VTs.AtomicLoadStore in {
+ def _#vt : PatFrag<(ops node:$ptr), (atomic_op node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = vt.VT;
+ }
+ defm NAME#_#vt : load_atomic_op_ord;
+ }
+}
+defm load_atomic : load_atomic_op<atomic_load>;
+// atomic loads
+multiclass AtomicLoadInst<string name> {
+ foreach pVT = VTs.AtomicLoadStore in {
+ foreach MO = ["monotonic", "acquire", "seq_cst"] in {
+ defvar MOStr = AtomicMemOrder<MO>.name;
+ defvar fullName = "load_atomic_"#pVT#"_"#MO;
+ defvar node = !cast<PatFrag>(fullName);
+ foreach AS = [global, generic, shared] in {
+ defvar directive = name#"."#AS.name#"."#MOStr;
+ def "_"#AS#"_"#MO#"_"#pVT.name#"_rr" : PISAAtomicLoad_rr<directive, AS, pVT, node>;
+ def "_"#AS#"_"#MO#"_"#pVT.name#"_ri" : PISAAtomicLoad_ri<directive, AS, pVT, node>;
+ }
+ // .addrspace defaults to .generic when not specified
+ defvar directive = name#"."#MOStr;
+ def "_generic_"#MO#"_"#pVT.name#"_alias_rr" : PISAAtomicLoad_rr<directive, generic, pVT, node>;
+ def "_generic_"#MO#"_"#pVT.name#"_alias_ri" : PISAAtomicLoad_ri<directive, generic, pVT, node>;
+ }
+ }
+}
+// non-atomic loads
+multiclass LoadInst<string name> {
+ foreach pVT = VTs.LoadStore in {
+ foreach AS = [global, const, generic, private, shared] in {
+ defvar directive = name#"."#AS;
+ def "_"#AS#"_"#pVT.name#"_ri" : PISALoad_ri<directive, AS, pVT, load>;
+ def "_"#AS#"_"#pVT.name#"_rr" : PISALoad_rr<directive, AS, pVT, load>;
+ // 'ld.weak' is an alias for 'ld'
+ defvar directiveA = name#"."#AS#".weak";
+ def "_"#AS#"_weak_"#pVT.name#"_ri" : PISALoad_ri<directiveA, AS, pVT, load>;
+ def "_"#AS#"_weak_"#pVT.name#"_rr" : PISALoad_rr<directiveA, AS, pVT, load>;
+ }
+ // .addrspace defaults to .generic when not specified
+ def "_generic_"#pVT.name#"_alias_ri" : PISALoad_ri<name, generic, pVT, load>;
+ def "_generic_"#pVT.name#"_alias_rr" : PISALoad_rr<name, generic, pVT, load>;
+ // 'ld.weak' is an alias for 'ld'
+ def "_generic_weak_"#pVT.name#"_alias_ri" : PISALoad_ri<name#".weak", generic, pVT, load>;
+ def "_generic_weak_"#pVT.name#"_alias_rr" : PISALoad_rr<name#".weak", generic, pVT, load>;
+ }
+}
+// kernel param loads (ld.param)
+multiclass LoadParamInst {
+ foreach VT = VTs.LoadParam in {
+ def _#VT.name: PISAInst<(outs VT.RegOpnd:$res),
+ (ins MEM_ii:$addr, variable_ops),
+ "ld.param"#VT.directive#" $res, $addr;">;
+ def : Pat<(VT.VT (load MEM_ii:$addr)), (!cast<Instruction>("loadParam_"#VT.name) MEM_ii:$addr)>;
+ def _#VT.name#"_ir": PISAInst<(outs VT.RegOpnd:$res),
+ (ins MEM_ir:$addr, variable_ops),
+ "ld.param"#VT.directive#" $res, $addr;">;
+ def : Pat<(VT.VT (load MEM_ir:$addr)), (!cast<Instruction>("loadParam_"#VT.name#"_ir") MEM_ir:$addr)>;
+ }
+}
+defm loadParam: LoadParamInst;
+defm ld : AtomicLoadInst<"ld">;
+defm ld : LoadInst<"ld">;
+
+
+// => st
+multiclass store_atomic_op_ord {
+ def NAME#_monotonic : PatFrag<(ops node:$val, node:$ptr), (!cast<SDPatternOperator>(NAME) node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingMonotonic = true;
+ }
+ def NAME#_acquire : PatFrag<(ops node:$val, node:$ptr), (!cast<SDPatternOperator>(NAME) node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingAcquire = true;
+ }
+ def NAME#_release : PatFrag<(ops node:$val, node:$ptr), (!cast<SDPatternOperator>(NAME) node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingRelease = true;
+ }
+ def NAME#_acq_rel : PatFrag<(ops node:$val, node:$ptr), (!cast<SDPatternOperator>(NAME) node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingAcquireRelease = true;
+ }
+ def NAME#_seq_cst : PatFrag<(ops node:$val, node:$ptr), (!cast<SDPatternOperator>(NAME) node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let IsAtomicOrderingSequentiallyConsistent = true;
+ }
+}
+multiclass store_atomic_op<SDNode atomic_op> {
+ foreach vt = VTs.AtomicLoadStore in {
+ def _#vt : PatFrag<(ops node:$val, node:$ptr), (atomic_op node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = vt.VT;
+ }
+ defm NAME#_#vt : store_atomic_op_ord;
+ }
+}
+defm store_atomic : store_atomic_op<atomic_store>;
+// atomic stores
+multiclass AtomicStoreInst<string name> {
+ foreach pVT = VTs.AtomicLoadStore in {
+ foreach MO = ["monotonic", "release", "seq_cst"] in {
+ defvar MOStr = AtomicMemOrder<MO>.name;
+ defvar fullName = "store_atomic_"#pVT#"_"#MO;
+ defvar node = !cast<PatFrag>(fullName);
+ foreach AS = [global, generic, shared] in {
+ defvar directive = name#"."#AS.name#"."#MOStr;
+ def "_"#AS#"_"#MO#"_"#pVT.name#"_rr" : PISAAtomicStore_rr<directive, AS, pVT, node>;
+ def "_"#AS#"_"#MO#"_"#pVT.name#"_ri" : PISAAtomicStore_ri<directive, AS, pVT, node>;
+ }
+ // .addrspace defaults to .generic when not specified
+ defvar directive = name#"."#MOStr;
+ def "_generic_"#MO#"_"#pVT.name#"_alias_rr" : PISAAtomicStore_rr<directive, generic, pVT, node>;
+ def "_generic_"#MO#"_"#pVT.name#"_alias_ri" : PISAAtomicStore_ri<directive, generic, pVT, node>;
+ }
+ }
+}
+// non-atomic stores
+multiclass StoreInst<string name> {
+ foreach pVT = VTs.LoadStore in {
+ foreach AS = [global, generic, private, shared] in {
+ defvar directive = name#"."#AS;
+ def "_"#AS#"_"#pVT.name#"_ri" : PISAStore_ri<directive, AS, pVT, store>;
+ def "_"#AS#"_"#pVT.name#"_rr" : PISAStore_rr<directive, AS, pVT, store>;
+ // 'st.weak' is an alias for 'st'
+ defvar directiveA = name#"."#AS#".weak";
+ def "_"#AS#"_weak_"#pVT.name#"_ri" : PISAStore_ri<directiveA, AS, pVT, store>;
+ def "_"#AS#"_weak_"#pVT.name#"_rr" : PISAStore_rr<directiveA, AS, pVT, store>;
+ }
+ // .addrspace defaults to .generic when not specified
+ def "_generic_"#pVT.name#"_alias_ri" : PISAStore_ri<name, generic, pVT, store>;
+ def "_generic_"#pVT.name#"_alias_rr" : PISAStore_rr<name, generic, pVT, store>;
+ // 'st.weak' is an alias for 'st'
+ def "_generic_weak_"#pVT.name#"_alias_ri" : PISAStore_ri<name#".weak", generic, pVT, store>;
+ def "_generic_weak_"#pVT.name#"_alias_rr" : PISAStore_rr<name#".weak", generic, pVT, store>;
+ }
+}
+defm st : AtomicStoreInst<"st">;
+defm st : StoreInst<"st">;
+
+// END: memory instruction definitions
+//////
+
+//////
+// BEGIN: synchronization instruction definitions
+
+// => barrier.workgroup
+let isConvergent = 1, hasSideEffects = 1 in {
+ def barrier_wg : PISAInst<(outs), (ins), "barrier.workgroup;", [(int_pisa_workgroup_barrier)]>;
+}
+
+// => fence
+multiclass FenceInst<string name> {
+ foreach as = [generic, global, shared, default] in {
+ foreach scope = [
+ workgroup,
+ gpu,
+ system] in {
+ // system scope is only supported for global fence
+ if !or(!ne(scope, system), !eq(as, global)) then {
+ defvar asName = !if(!empty(as.name), "", "."#as.name);
+ defvar opcode = !if(!empty(as.name), "_"#scope, "_"#as.name#"_"#scope);
+ defvar asmstr = name#asName#"${order}."#scope#";";
+ def opcode: PISAInst<(outs), (ins FenceMemOrderOpnd:$order), asmstr>;
+ }
+ }
+ }
+ defvar directive = ".subgroup";
+ defvar opcode = !subst(".", "_", directive);
+ def opcode: PISAInst<(outs), (ins), name#directive#";">;
+}
+defm fence : FenceInst<"fence">;
+
+// END: synchronization instruction definitions
+//////
+
+
+
+// => lifetime.start
+let hasSideEffects = 1, mayLoad = 0, mayStore = 0, isReMaterializable = 0 in {
+ multiclass LifetimeStart {
+ foreach VT = VTs.LifetimeTypes in {
+ def "_"#VT.name#"_r" : PISAInst<(outs VT.RegOpnd:$dst), (ins),
+ "lifetime.start \t$dst;", []>;
+ }
+ }
+ defm lifetime_start : LifetimeStart;
+}
+// Register-class to lifetime.start opcode.
+foreach VT = VTs.LifetimeTypes in {
+ def : LifetimeStartEntry<!cast<string>(VT.RC),
+ !cast<Instruction>("lifetime_start_"#VT.name#"_r")>;
}
-// Placeholder instruction. Real PISA instructions are added in later changes.
-def NOP : PISAInst<(outs), (ins), "nop", []>;
diff --git a/llvm/lib/Target/PISA/PISAMCInstLower.cpp b/llvm/lib/Target/PISA/PISAMCInstLower.cpp
new file mode 100644
index 0000000000000..9517dd490472f
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAMCInstLower.cpp
@@ -0,0 +1,134 @@
+//===-- PISAMCInstLower.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAMCInstLower.h"
+#include "MCTargetDesc/PISAMCExpr.h"
+#include "PISA.h"
+#include "PISARegManager.h"
+#include "PISASubtarget.h"
+#include "PISAUtils.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+MCOperand PISAMCInstLower::lowerSymbolOperand(const MachineOperand &MO,
+ MCSymbol &Sym) const {
+ const MCExpr *Expr = PISAGlobalAddressMCExpr::create(Sym, OutContext);
+ return MCOperand::createExpr(Expr);
+}
+
+MCSymbol &
+PISAMCInstLower::getGlobalAddressSymbol(const MachineOperand &MO) const {
+ assert(MO.isGlobal());
+ return *AP.getSymbol(MO.getGlobal());
+}
+
+void PISAMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
+ OutMI.setOpcode(MI->getOpcode());
+ for (unsigned OpNo = 0, E = MI->getNumOperands(); OpNo != E; ++OpNo) {
+ const MachineOperand &MO = MI->getOperand(OpNo);
+ MCOperand MCOp;
+ switch (MO.getType()) {
+ default:
+ llvm_unreachable("unknown operand type");
+ case MachineOperand::MO_Metadata: {
+ // Skip call to addOperand() outside the switch as MCInst doesn't
+ // support metadata operand types.
+ continue;
+ }
+ case MachineOperand::MO_ExternalSymbol: {
+ // Convert external symbol operand (kernel arg name on loadParam
+ // instructions) to an MCExpr for PISAInstPrinter::printParamMemOperand.
+ MCSymbol *Sym = OutContext.getOrCreateSymbol(MO.getSymbolName());
+ MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(Sym, OutContext));
+ break;
+ }
+ case MachineOperand::MO_FrameIndex: {
+ // MachineOperand of type MO_FrameIndex is lowered to immediate operand
+ // with value equal to frame index. Whether an immediate operand is
+ // frame index is indicated by flags that are stored in MCInst instance.
+ // During operand printing, we inspect flag value first and decide if
+ // the operand has to be printed as private variable or a generic
+ // immediate value.
+ unsigned int FrameIndex = MO.getIndex();
+ MCOp = MCOperand::createImm(FrameIndex);
+ setVariableRef(OutMI, OpNo);
+ break;
+ }
+ case MachineOperand::MO_GlobalAddress: {
+ MCOp = lowerSymbolOperand(MO, getGlobalAddressSymbol(MO));
+ break;
+ }
+ case MachineOperand::MO_MachineBasicBlock:
+ MCOp = MCOperand::createExpr(
+ MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), OutContext));
+ break;
+ case MachineOperand::MO_Register: {
+ Register CurReg = MO.getReg();
+ unsigned EncodedVal = CurReg;
+ if (CurReg.isVirtual()) {
+ auto &MRI = MI->getParent()->getParent()->getRegInfo();
+ auto *RC = MRI.getRegClass(CurReg);
+ auto NumElts = TRI.getNumEltsFromRegClass(RC);
+ auto EltSize = TRI.getBitSizeFromRegClass(RC);
+ auto Bank = RegMgr.getRegBank(NumElts, EltSize);
+ EncodedVal = RegMgr.encodeVirtualRegister(Bank, CurReg);
+ }
+ MCOp = MCOperand::createReg(EncodedVal);
+ auto SubReg = MO.getSubReg();
+ auto IsMov = MI->isMoveImmediate() || MI->isMoveReg();
+ auto Swizzle = TRI.getSwizzle(SubReg);
+ if (IsMov && !SubReg && CurReg.isVirtual()) {
+ // vector args in mov instructions always print swizzle
+ auto &MRI = MI->getParent()->getParent()->getRegInfo();
+ auto *RC = MRI.getRegClass(CurReg);
+ auto NumElts = TRI.getNumEltsFromRegClass(RC);
+ if (NumElts == 2)
+ Swizzle = PISA::Swizzle::XY;
+ else if (NumElts == 4)
+ Swizzle = PISA::Swizzle::XYZW;
+ else
+ assert((NumElts == 1) && "unknown swizzle value");
+ }
+ setSwizzle(OutMI, OpNo, Swizzle);
+ break;
+ }
+ case MachineOperand::MO_Immediate:
+ MCOp = MCOperand::createImm(MO.getImm());
+ break;
+ case MachineOperand::MO_FPImmediate:
+ // All floating point values are bitcasted
+ // into integer ones. Double- and single-precison ones could be specially
+ // taged in MC as SFP or DFP immediate. But, in general, as each type of
+ // immediate operand has its own methods, general immediate operands
+ // won't loss any semantics.
+ uint64_t ImmVal =
+ MO.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue();
+ // TODO: bfloat16 has a different exponent/mantissa layout than
+ // IEEE half; a separate kBFPImmediate kind may be needed to
+ // distinguish them (see also getFPImmInReg TODO).
+ if (MO.getFPImm()->getType()->isHalfTy() ||
+ MO.getFPImm()->getType()->isBFloatTy())
+ MCOp = MCOperand::createHFPImm(ImmVal);
+ else if (MO.getFPImm()->getType()->isFloatTy())
+ MCOp = MCOperand::createSFPImm(ImmVal);
+ else if (MO.getFPImm()->getType()->isDoubleTy())
+ MCOp = MCOperand::createDFPImm(ImmVal);
+ else
+ MCOp = MCOperand::createImm(ImmVal);
+ break;
+ }
+
+ OutMI.addOperand(MCOp);
+ }
+}
diff --git a/llvm/lib/Target/PISA/PISAMCInstLower.h b/llvm/lib/Target/PISA/PISAMCInstLower.h
new file mode 100644
index 0000000000000..9575cbb5293b4
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAMCInstLower.h
@@ -0,0 +1,103 @@
+//===-- PISAMCInstLower.h -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAMCINSTLOWER_H
+#define LLVM_LIB_TARGET_PISA_PISAMCINSTLOWER_H
+
+#include "PISADefines.h"
+#include "PISARegisterInfo.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/Support/Compiler.h"
+#include <cassert>
+
+namespace llvm {
+class MCContext;
+class MCOperand;
+class MCSymbol;
+class MachineInstr;
+class MachineOperand;
+
+namespace PISA {
+class RegManager;
+} // namespace PISA
+
+struct PISAMCOpnd {
+ // given immediate value represents the index of a variable (.e.g, @R0).
+ bool IsVariable;
+ // swizzle of given operand
+ unsigned Swizzle;
+};
+
+class PISAMCInst : public MCInst {
+public:
+ SmallDenseMap<unsigned, PISAMCOpnd> Opnds;
+
+ // Source line number in the original assembly file, used for debug info and
+ // error reporting
+ unsigned SourceLine = 0;
+ unsigned getSourceLine() const { return SourceLine; }
+ void setSourceLine(unsigned L) { SourceLine = L; }
+};
+
+// This class is used to lower a MachineInstr into an MCInst.
+class LLVM_LIBRARY_VISIBILITY PISAMCInstLower {
+public:
+ static void setVariableRef(MCInst &MI, unsigned OpNo) {
+ auto *MC = static_cast<PISAMCInst *>(&MI);
+ auto It = MC->Opnds.find(OpNo);
+ if (It == MC->Opnds.end())
+ MC->Opnds[OpNo] = {true, 0};
+ else
+ It->second.IsVariable = true;
+ }
+
+ static bool isVariableRef(const MCInst &MI, unsigned int OpNo) {
+ const auto *MC = static_cast<const PISAMCInst *>(&MI);
+ auto It = MC->Opnds.find(OpNo);
+ if (It == MC->Opnds.end())
+ return false;
+ return It->second.IsVariable;
+ }
+
+ static void setSwizzle(MCInst &MI, unsigned OpNo, PISA::Swizzle Swizzle) {
+ static_assert(static_cast<unsigned>(PISA::Swizzle::NONE) <= 7);
+ auto *MC = static_cast<PISAMCInst *>(&MI);
+ auto It = MC->Opnds.find(OpNo);
+ if (It == MC->Opnds.end())
+ MC->Opnds[OpNo] = {false, static_cast<unsigned>(Swizzle)};
+ else
+ It->second.Swizzle = static_cast<unsigned>(Swizzle);
+ }
+
+ static PISA::Swizzle getSwizzle(const MCInst &MI, unsigned OpNo) {
+ const auto *MC = static_cast<const PISAMCInst *>(&MI);
+ auto It = MC->Opnds.find(OpNo);
+ if (It == MC->Opnds.end())
+ return static_cast<PISA::Swizzle>(PISA::Swizzle::NONE);
+ return static_cast<PISA::Swizzle>(It->second.Swizzle);
+ }
+
+ PISAMCInstLower(MCContext &Ctx, const PISARegisterInfo &TRI,
+ const PISA::RegManager &RegMgr, const AsmPrinter &AP)
+ : OutContext(Ctx), TRI(TRI), RegMgr(RegMgr), AP(AP) {}
+ void lower(const MachineInstr *MI, MCInst &OutMI) const;
+
+private:
+ MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol &Sym) const;
+ MCSymbol &getGlobalAddressSymbol(const MachineOperand &MO) const;
+
+private:
+ MCContext &OutContext;
+ const PISARegisterInfo &TRI;
+ const PISA::RegManager &RegMgr;
+ const AsmPrinter &AP;
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISAMCINSTLOWER_H
diff --git a/llvm/lib/Target/PISA/PISARegManager.cpp b/llvm/lib/Target/PISA/PISARegManager.cpp
new file mode 100644
index 0000000000000..11171220b1272
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISARegManager.cpp
@@ -0,0 +1,62 @@
+//===-- PISARegManager.cpp - Manage PISA virtual registers ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISARegManager.h"
+#include "PISAInstrInfo.h"
+#include "PISASubtarget.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+using namespace PISA;
+
+void RegManager::computeMapping() {
+ std::array<unsigned, RegType::NUM_TYPE> Count{};
+ auto *TII = MF.getSubtarget<PISASubtarget>().getInstrInfo();
+ for (auto &MBB : MF) {
+ for (auto &MI : MBB) {
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg() || (!MO.isDef() && !MO.isUndef()))
+ continue;
+
+ Register CurReg = MO.getReg();
+
+ if (CurReg.isPhysical())
+ continue;
+
+ if (Mapping.count(CurReg) != 0)
+ continue;
+
+ unsigned Flags = Usage::None;
+ if (TII->isNoEmissionInstr(MI))
+ Flags |= Usage::NoEmissionDef;
+
+ auto Type = getRegType(MRI.getRegClass(MO.getReg()));
+ RegInfo Info{Type, Count[Type]++, static_cast<Usage>(Flags)};
+ Mapping[CurReg] = Info;
+ }
+ }
+ }
+}
+
+unsigned RegManager::getRegIdx(Register Reg) const {
+ const auto *I = Mapping.find(Reg);
+ assert(I != Mapping.end() && "missing?");
+ return I->second.Idx;
+}
+
+unsigned RegManager::encodeVirtualRegister(RegBank Bank, Register Reg) const {
+ auto &MRI = MF.getRegInfo();
+ unsigned Idx = getRegIdx(Reg);
+ auto Type = getRegType(MRI.getRegClass(Reg));
+ return RegEncoder::encodeVirtualRegister(Idx, Bank, Type);
+}
+
+RegManager::RegManager(const MachineFunction &MF)
+ : MF(MF), MRI(MF.getRegInfo()) {
+ computeMapping();
+}
diff --git a/llvm/lib/Target/PISA/PISARegManager.h b/llvm/lib/Target/PISA/PISARegManager.h
new file mode 100644
index 0000000000000..3feeab6a7af8e
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISARegManager.h
@@ -0,0 +1,47 @@
+//===-- PISARegManager.h - Manage PISA virtual registers ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAREGMANAGER_H
+#define LLVM_LIB_TARGET_PISA_PISAREGMANAGER_H
+
+#include "MCTargetDesc/PISARegEncoder.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+
+namespace llvm {
+namespace PISA {
+
+class RegManager : public RegEncoder {
+public:
+ enum Usage { None = 0, NoEmissionDef = (1 << 0) };
+ struct RegInfo {
+ RegType Type;
+ unsigned Idx;
+ Usage Flags;
+ };
+
+public:
+ RegManager(const MachineFunction &MF);
+ unsigned getRegIdx(Register Reg) const;
+ void setRegIdx(Register Reg, unsigned Idx) { Mapping[Reg].Idx = Idx; }
+ unsigned encodeVirtualRegister(RegBank Bank, Register Reg) const;
+ auto mapping() const { return make_range(Mapping.begin(), Mapping.end()); }
+ bool exists(Register Reg) const { return Mapping.count(Reg) > 0; }
+
+private:
+ MapVector<Register, RegInfo> Mapping;
+ const MachineFunction &MF;
+ const MachineRegisterInfo &MRI;
+ void computeMapping();
+};
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISAREGMANAGER_H
diff --git a/llvm/lib/Target/PISA/PISARegisterBanks.td b/llvm/lib/Target/PISA/PISARegisterBanks.td
new file mode 100644
index 0000000000000..ee28b20fb07e1
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISARegisterBanks.td
@@ -0,0 +1,15 @@
+//===-- PISARegisterBanks.td - Describe PISA RegBanks ------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+def RegistersRegBank : RegisterBank<"Registers",
+ [Pred, Reg8b, Reg16b, Reg32b, Reg64b, Reg128b,
+ RegV2_8b, RegV3_8b, RegV4_8b,
+ RegV2_16b, RegV3_16b, RegV4_16b,
+ RegV2_32b, RegV3_32b, RegV4_32b, RegV5_32b, RegV6_32b, RegV7_32b, RegV8_32b, RegV16_32b, RegV32_32b, RegV64_32b,
+ RegV2_64b, RegV3_64b, RegV4_64b]
+>;
diff --git a/llvm/lib/Target/PISA/PISARegisterInfo.cpp b/llvm/lib/Target/PISA/PISARegisterInfo.cpp
index f66db9c9e0283..fdf4c7c68e7af 100644
--- a/llvm/lib/Target/PISA/PISARegisterInfo.cpp
+++ b/llvm/lib/Target/PISA/PISARegisterInfo.cpp
@@ -9,6 +9,9 @@
#include "PISARegisterInfo.h"
#include "PISA.h"
#include "PISASubtarget.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringTable.h"
#include "llvm/CodeGen/MachineFunction.h"
#define GET_REGINFO_TARGET_DESC
@@ -16,14 +19,273 @@
using namespace llvm;
-PISARegisterInfo::PISARegisterInfo() : PISAGenRegisterInfo(PISA::DummyReg) {}
+namespace {
+struct SpecialRegEntry {
+ StringTable::Offset Name;
+};
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#define GET_SpecialRegNames_IMPL
+#include "PISAGenSearchableTables.inc"
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+} // anonymous namespace
+
+static_assert(PISA::NUM_TARGET_SUBREGS == 63, "updated needed!");
+DenseMap<unsigned, PISARegisterInfo::SwizzleDesc> PISARegisterInfo::SwizzleMap =
+ {
+ {PISA::NoSubRegister, {PISA::Swizzle::NONE, nullptr}},
+ {PISA::sub8_0, {PISA::Swizzle::X, ".x"}},
+ {PISA::sub8_1, {PISA::Swizzle::Y, ".y"}},
+ {PISA::sub8_2, {PISA::Swizzle::Z, ".z"}},
+ {PISA::sub8_3, {PISA::Swizzle::W, ".w"}},
+ {PISA::sub16_0, {PISA::Swizzle::X, ".x"}},
+ {PISA::sub16_1, {PISA::Swizzle::Y, ".y"}},
+ {PISA::sub16_2, {PISA::Swizzle::Z, ".z"}},
+ {PISA::sub16_3, {PISA::Swizzle::W, ".w"}},
+ {PISA::sub32_0, {PISA::Swizzle::X, ".x"}},
+ {PISA::sub32_1, {PISA::Swizzle::Y, ".y"}},
+ {PISA::sub32_2, {PISA::Swizzle::Z, ".z"}},
+ {PISA::sub32_3, {PISA::Swizzle::W, ".w"}},
+ {PISA::sub64_0, {PISA::Swizzle::X, ".x"}},
+ {PISA::sub64_1, {PISA::Swizzle::Y, ".y"}},
+ {PISA::sub64_2, {PISA::Swizzle::Z, ".z"}},
+ {PISA::sub64_3, {PISA::Swizzle::W, ".w"}},
+ {PISA::sub8_xy, {PISA::Swizzle::XY, ".xy"}},
+ {PISA::sub8_zw, {PISA::Swizzle::ZW, ".zw"}},
+ {PISA::sub16_xy, {PISA::Swizzle::XY, ".xy"}},
+ {PISA::sub16_zw, {PISA::Swizzle::ZW, ".zw"}},
+ {PISA::sub32_xy, {PISA::Swizzle::XY, ".xy"}},
+ {PISA::sub32_zw, {PISA::Swizzle::ZW, ".zw"}},
+ {PISA::sub64_xy, {PISA::Swizzle::XY, ".xy"}},
+ {PISA::sub64_zw, {PISA::Swizzle::ZW, ".zw"}},
+};
+
+bool PISARegisterInfo::shouldCoalesce(
+ MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
+ const TargetRegisterClass *DstRC, unsigned DstSubReg,
+ const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
+
+ if (!MI->isCopy())
+ return false;
+
+ auto IsLegalSwizzle = [&](unsigned Subreg) {
+ auto Swizzle = SwizzleMap.find(Subreg);
+ return Swizzle != SwizzleMap.end();
+ };
+
+ return IsLegalSwizzle(SubReg) && IsLegalSwizzle(DstSubReg);
+}
+
+PISARegisterInfo::PISARegisterInfo() : PISAGenRegisterInfo(PISA::DummyReg) {
+ // Tablegen can sometimes synthesize register classes if you don't set
+ // subregs explicitly on some regs. For now at least, we probably want to
+ // be explicit about what regclasses exist. If you added a register class
+ // explicitly, go ahead and update this number. If not, you might want to
+ // figure out what happened.
+ // New reg classes must also be reflected in PISATargetLowering constructor
+ static_assert(std::size(PISAMCRegisterClassStorage.Classes) == 30);
+ unsigned NumRCs = getNumRegClasses();
+ for (unsigned I = 0; I < NumRCs; I++) {
+ auto *RC = getRegClass(I);
+ auto RCD = std::make_unique<RegClassDescription>();
+ if (RC == &PISA::RegV64_32bRegClass) {
+ // RegV64_32b has no per-element sub-register structure (LLVM's
+ // LaneBitmask cannot represent 64 lanes), so derive the metadata
+ // from the known type rather than from lane masks.
+ RCD->NumElements = 64;
+ RCD->ScalarBitSize = 32;
+ } else {
+ RCD->NumElements = RC->LaneMask.getNumLanes();
+ RCD->ScalarBitSize = getScalarBitSize(RC, RCD->NumElements);
+ }
+ auto Key = std::make_pair(RCD->NumElements, RCD->ScalarBitSize);
+ if (!VecRegClassMap[Key])
+ VecRegClassMap[std::make_pair(RCD->NumElements, RCD->ScalarBitSize)] = RC;
+ RegClassMap[RC] = std::move(RCD);
+ }
+
+ // Precompute special register BitVector for O(1) lookup.
+ SpecialRegs.resize(getNumRegs());
+ for (unsigned I = 1, E = getNumRegs(); I < E; ++I) {
+ MCRegister Reg(I);
+ if (lookupSpecialRegByName(getName(Reg)))
+ SpecialRegs.set(I);
+ }
+}
+
+BitVector PISARegisterInfo::getReservedRegs(const MachineFunction &MF) const {
+ // Reserve DummyReg (used as a sentinel in machine instructions) and all
+ // special registers (hardware-defined values like %localid, %groupid, etc.
+ // that appear in machine instructions without explicit definitions).
+ //
+ // General-purpose physical registers are left unreserved so that
+ // RegisterClassInfo::getNumAllocatableRegs returns non-zero values for
+ // every register class. This is required because the register coalescer
+ // refuses to coalesce into register classes with zero allocatable
+ // registers. PISA works entirely with virtual registers and disables
+ // physical register allocation (createTargetRegisterAllocator returns
+ // nullptr), so unreserved physical registers are never actually allocated.
+ BitVector Reserved(getNumRegs());
+ Reserved.set(PISA::DummyReg);
+ Reserved |= SpecialRegs;
+ return Reserved;
+}
const MCPhysReg *
PISARegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
- static const MCPhysReg CalleeSavedRegs[] = {0};
- return CalleeSavedRegs;
+ static const MCPhysReg CalleeSavedReg = {0};
+ return &CalleeSavedReg;
}
-BitVector PISARegisterInfo::getReservedRegs(const MachineFunction &MF) const {
- return BitVector(getNumRegs());
+unsigned PISARegisterInfo::getScalarBitSize(const TargetRegisterClass *RC,
+ unsigned NumElts) const {
+ unsigned RegSizeInBits = getRegSizeInBits(*RC);
+ assert((RegSizeInBits % NumElts == 0) && "not divisible?");
+ return RegSizeInBits / NumElts;
+}
+
+unsigned PISARegisterInfo::getSubRegIdx(unsigned Size, unsigned Idx) const {
+ assert(Idx < 4 && "Only 4 sub-registers supported!");
+
+ static unsigned Subregs[4][4] = {
+ {PISA::sub8_0, PISA::sub8_1, PISA::sub8_2, PISA::sub8_3},
+ {PISA::sub16_0, PISA::sub16_1, PISA::sub16_2, PISA::sub16_3},
+ {PISA::sub32_0, PISA::sub32_1, PISA::sub32_2, PISA::sub32_3},
+ {PISA::sub64_0, PISA::sub64_1, PISA::sub64_2, PISA::sub64_3}};
+
+ switch (Size) {
+ case 8:
+ return Subregs[0][Idx];
+ case 16:
+ return Subregs[1][Idx];
+ case 32:
+ return Subregs[2][Idx];
+ case 64:
+ return Subregs[3][Idx];
+ default:
+ assert(0 && "unknown type size!");
+ break;
+ }
+ return 0;
+}
+
+unsigned PISARegisterInfo::getCompositeSubRegIdx(unsigned Size, unsigned Base,
+ unsigned Count) const {
+ // Only nameable 2-element pairs: .xy (base 0) and .zw (base 2).
+ if (Count != 2 || (Base != 0 && Base != 2))
+ return 0;
+ bool Low = (Base == 0);
+ switch (Size) {
+ case 8:
+ return Low ? PISA::sub8_xy : PISA::sub8_zw;
+ case 16:
+ return Low ? PISA::sub16_xy : PISA::sub16_zw;
+ case 32:
+ return Low ? PISA::sub32_xy : PISA::sub32_zw;
+ case 64:
+ return Low ? PISA::sub64_xy : PISA::sub64_zw;
+ default:
+ return 0;
+ }
+}
+
+PISA::Swizzle PISARegisterInfo::getSwizzle(unsigned SubReg) const {
+ auto Swizzle = SwizzleMap.find(SubReg);
+ assert((Swizzle != SwizzleMap.end()) && "invalid swizzle!");
+ return Swizzle->second.Swizzle;
+}
+
+const char *PISARegisterInfo::getSwizzleName(unsigned SubReg) const {
+ auto Swizzle = SwizzleMap.find(SubReg);
+ assert((Swizzle != SwizzleMap.end()) && "invalid swizzle!");
+ return Swizzle->second.SwizzleName;
+}
+
+bool PISARegisterInfo::isSelectorSwizzle(PISA::Swizzle Swizzle) {
+ switch (Swizzle) {
+ case PISA::Swizzle::X:
+ case PISA::Swizzle::Y:
+ case PISA::Swizzle::Z:
+ case PISA::Swizzle::W:
+ return true;
+ default:
+ return false;
+ }
+}
+
+const TargetRegisterClass *PISARegisterInfo::getRegClassFromLLT(LLT Ty) const {
+ if (Ty.isScalar() || Ty.isPointer()) {
+ switch (Ty.getSizeInBits()) {
+ case 1:
+ return &PISA::PredRegClass;
+ case 8:
+ return &PISA::Reg8bRegClass;
+ case 16:
+ return &PISA::Reg16bRegClass;
+ case 32:
+ return &PISA::Reg32bRegClass;
+ case 64:
+ return &PISA::Reg64bRegClass;
+ case 128:
+ return &PISA::Reg128bRegClass;
+ default:
+ break;
+ }
+ } else if (Ty.isVector()) {
+ unsigned NumElts = Ty.getNumElements();
+ unsigned BitSize = Ty.getScalarSizeInBits();
+ return getVectorRegClass(NumElts, BitSize);
+ }
+
+ llvm_unreachable("unhandled LLT!");
+}
+
+unsigned
+PISARegisterInfo::getNumEltsFromRegClass(const TargetRegisterClass *RC) const {
+ auto I = RegClassMap.find(RC);
+ assert(I != RegClassMap.end());
+ auto &RCD = I->second;
+ return RCD->NumElements;
+}
+
+unsigned
+PISARegisterInfo::getBitSizeFromRegClass(const TargetRegisterClass *RC) const {
+ auto I = RegClassMap.find(RC);
+ assert(I != RegClassMap.end());
+ auto &RCD = I->second;
+ return RCD->ScalarBitSize;
+}
+
+const TargetRegisterClass *
+PISARegisterInfo::getVectorRegClass(unsigned NumElts, unsigned BitSize) const {
+ auto P = std::make_pair(NumElts, BitSize);
+ auto I = VecRegClassMap.find(P);
+ assert(I != VecRegClassMap.end());
+ return I->second;
+}
+
+bool PISARegisterInfo::isSpecialReg(Register Reg) const {
+ return Reg.isPhysical() && SpecialRegs.test(Reg.asMCReg());
+}
+
+const TargetRegisterClass *
+PISARegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A,
+ const TargetRegisterClass *B,
+ unsigned SubIdx) const {
+ switch (SubIdx) {
+ default:
+ return PISAGenRegisterInfo::getMatchingSuperRegClass(A, B, SubIdx);
+ // generic code (above) is unable to find 'hi' subreg
+ // index within 2-element vector ('.zw' within v2b?).
+ case PISA::sub8_zw:
+ case PISA::sub16_zw:
+ case PISA::sub32_zw:
+ case PISA::sub64_zw:
+ return PISAGenRegisterInfo::getSubClassWithSubReg(A, SubIdx);
+ }
}
diff --git a/llvm/lib/Target/PISA/PISARegisterInfo.h b/llvm/lib/Target/PISA/PISARegisterInfo.h
index f10780d825eda..b49ba8a98de02 100644
--- a/llvm/lib/Target/PISA/PISARegisterInfo.h
+++ b/llvm/lib/Target/PISA/PISARegisterInfo.h
@@ -9,6 +9,7 @@
#ifndef LLVM_LIB_TARGET_PISA_PISAREGISTERINFO_H
#define LLVM_LIB_TARGET_PISA_PISAREGISTERINFO_H
+#include "PISADefines.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -30,6 +31,60 @@ class PISARegisterInfo : public PISAGenRegisterInfo {
Register getFrameRegister(const MachineFunction &MF) const override {
return 0;
}
+ const TargetRegisterClass *getRegClassFromLLT(LLT Ty) const;
+ unsigned getNumEltsFromRegClass(const TargetRegisterClass *RC) const;
+ unsigned getBitSizeFromRegClass(const TargetRegisterClass *RC) const;
+ const TargetRegisterClass *getVectorRegClass(unsigned NumElts,
+ unsigned BitSize) const;
+ unsigned getSubRegIdx(unsigned Size, unsigned Elt) const;
+
+ // Return the composite sub-register index (.xy / .zw) covering `Count`
+ // consecutive elements of `Size` bits starting at element `Base`, or 0 if
+ // there is no nameable composite for that slice. Only 2-element pairs at
+ // base 0 (.xy) or base 2 (.zw) are supported.
+ unsigned getCompositeSubRegIdx(unsigned Size, unsigned Base,
+ unsigned Count) const;
+
+ PISA::Swizzle getSwizzle(unsigned SubReg) const;
+ const char *getSwizzleName(unsigned SubReg) const;
+ static bool isSelectorSwizzle(PISA::Swizzle Swizzle);
+
+ bool isSpecialReg(Register Reg) const;
+
+ bool shouldCoalesce(MachineInstr *MI, const TargetRegisterClass *SrcRC,
+ unsigned SubReg, const TargetRegisterClass *DstRC,
+ unsigned DstSubReg, const TargetRegisterClass *NewRC,
+ LiveIntervals &LIS) const override;
+
+ const TargetRegisterClass *
+ getMatchingSuperRegClass(const TargetRegisterClass *A,
+ const TargetRegisterClass *B,
+ unsigned SubIdx) const override;
+
+private:
+ unsigned getScalarBitSize(const TargetRegisterClass *RC,
+ unsigned NumElts) const;
+
+ struct RegClassDescription {
+ unsigned NumElements;
+ unsigned ScalarBitSize;
+ };
+
+ // Maps TargetRegisterClass -> RegClassDescription
+ DenseMap<const TargetRegisterClass *, std::unique_ptr<RegClassDescription>>
+ RegClassMap;
+ // Maps <NumElts, BitSize> -> TargetRegisterClass (vector reg class)
+ DenseMap<std::pair<unsigned, unsigned>, const TargetRegisterClass *>
+ VecRegClassMap;
+
+ // map SubReg info to Swizzle
+ struct SwizzleDesc {
+ PISA::Swizzle Swizzle;
+ const char *SwizzleName;
+ };
+ static DenseMap<unsigned, SwizzleDesc> SwizzleMap;
+
+ BitVector SpecialRegs;
};
} // namespace llvm
diff --git a/llvm/lib/Target/PISA/PISARegisterInfo.td b/llvm/lib/Target/PISA/PISARegisterInfo.td
index b2c06a57c27a2..ae407b269a2b3 100644
--- a/llvm/lib/Target/PISA/PISARegisterInfo.td
+++ b/llvm/lib/Target/PISA/PISARegisterInfo.td
@@ -1,4 +1,4 @@
-//===-- PISARegisterInfo.td - PISA Register defs ----------*- tablegen -*-===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -8,32 +8,262 @@
let Namespace = "PISA" in {
+ class SpecialRegister <string name> : Register<name> {
+ string Name = NAME;
+ }
+
class PISARegisterClass<list<ValueType> regTypes, int alignment, dag regList>
- : RegisterClass<"PISA", regTypes, alignment, regList> {}
+ : RegisterClass<"PISA", regTypes, alignment, regList> {}
- // .reg style registers.
- class GeneralRegisterClass<list<ValueType> regTypes, int alignment,
- dag regList>
- : PISARegisterClass<regTypes, alignment, regList> {}
+ // .reg style registers
+ class GeneralRegisterClass<list<ValueType> regTypes, int alignment, dag regList> :
+ PISARegisterClass<regTypes, alignment, regList> {
+ // Keep flags in sync with RegType in RegEncoder
+ let TSFlags{1-0} = 1;
+ }
- // .pred style registers.
- class PredRegisterClass<list<ValueType> regTypes, int alignment, dag regList>
- : PISARegisterClass<regTypes, alignment, regList> {}
+ // .pred style registers
+ class PredRegisterClass<list<ValueType> regTypes, int alignment, dag regList> :
+ PISARegisterClass<regTypes, alignment, regList> {
+ // Keep flags in sync with RegType in RegEncoder
+ let TSFlags{1-0} = 2;
+ }
def DummyReg : Register<"%null">;
- foreach Index = 0-63 in {
- def Reg32b_#Index : Register<"">;
+ multiclass VectorSpecialRegSubRegs<int n, string name> {
+ def X : SpecialRegister<name#".x">;
+ if !ge(n, 2) then {
+ def Y : SpecialRegister<name#".y">;
+ }
+ if !ge(n, 3) then {
+ def Z : SpecialRegister<name#".z">;
+ }
+ if !ge(n, 4) then {
+ def W : SpecialRegister<name#".w">;
+ }
+ }
+
+ // Registers to describe Vector-subregister relationships
+ class VecReg<list<Register> subregs> : RegisterWithSubRegs<"", subregs>;
+
+ foreach Index = 0-3 in { // .x, .y, .z, .w
+ def sub8_#Index : SubRegIndex<8, !shl(Index, 3)>;
+ def sub16_#Index : SubRegIndex<16, !shl(Index, 4)>;
+ def sub32_#Index : SubRegIndex<32, !shl(Index, 5)>;
+ def sub64_#Index : SubRegIndex<64, !shl(Index, 6)>;
+ }
+ foreach Index = 0-1 in { // .xy, .zw
+ defvar Part = !if(!eq(Index, 0), "xy", "zw");
+ def sub8_#Part : SubRegIndex<!shl(8, 1), !shl(Index, 4)>;
+ def sub16_#Part : SubRegIndex<!shl(16, 1), !shl(Index, 5)>;
+ def sub32_#Part : SubRegIndex<!shl(32, 1), !shl(Index, 6)>;
+ def sub64_#Part : SubRegIndex<!shl(64, 1), !shl(Index, 7)>;
+ }
+ foreach Index = 4-31 in { // large vector support
+ def sub32_#Index : SubRegIndex<32, !shl(Index, 5)>;
+ }
+
+ // Special registers
+ defset list<SpecialRegister> LocalIdSubRegs = {
+ defm SpecialReg_LocalId : VectorSpecialRegSubRegs<3, "%localid">;
+ }
+ defset list<SpecialRegister> LocalSizeSubRegs = {
+ defm SpecialReg_LocalSize : VectorSpecialRegSubRegs<3, "%localsize">;
+ }
+ defset list<SpecialRegister> EnqueuedLocalSizeSubRegs = {
+ defm SpecialReg_EnqueuedLocalSize : VectorSpecialRegSubRegs<3, "%enqueuedlocalsize">;
+ }
+ defset list<SpecialRegister> GlobalOffsetSubRegs = {
+ defm SpecialReg_GlobalOffset : VectorSpecialRegSubRegs<3, "%globaloffset">;
}
+ defset list<SpecialRegister> GlobalSizeSubRegs = {
+ defm SpecialReg_GlobalSize : VectorSpecialRegSubRegs<3, "%globalsize">;
+ }
+ defset list<SpecialRegister> GroupIdSubRegs = {
+ defm SpecialReg_GroupId : VectorSpecialRegSubRegs<3, "%groupid">;
+ }
+ defset list<SpecialRegister> GroupCountSubRegs = {
+ defm SpecialReg_GroupCount : VectorSpecialRegSubRegs<3, "%groupcount">;
+ }
+
+ def SpecialReg_ActiveMask : SpecialRegister<"%activemask">;
+ def SpecialReg_LaneId : SpecialRegister<"%laneid">;
+ def SpecialReg_SubgroupSize : SpecialRegister<"%subgroupsize">;
+ def SpecialReg_WorkDim : SpecialRegister<"%workdim">;
+
+
+
+ //
+ // Scalar registers
+ //
foreach Index = 0-3 in {
+ def Reg8b_#Index : Register<"">;
+ def Reg16b_#Index : Register<"">;
+ def Reg32b_#Index : Register<"">;
def Reg64b_#Index : Register<"">;
+ def Reg128b_#Index : Register<"">;
+ }
+ foreach Index = 4-63 in {
+ def Reg32b_#Index : Register<"">;
+ }
+
+ // Predicate physical registers (used only so the Pred register class has
+ // non-zero allocatable registers, PISA never allocates physical registers).
+ foreach Index = 0-3 in {
def Pred_#Index : Register<"">;
}
- def Pred : PredRegisterClass<[i1], 8,
- (add DummyReg, (sequence "Pred_%u", 0, 3))>;
+ def Pred : PredRegisterClass<[i1], 8, (add DummyReg, (sequence "Pred_%u", 0, 3))>;
+ def Reg8b : GeneralRegisterClass<[i8], 8, (add (sequence "Reg8b_%u", 0, 3))>;
+ def Reg16b : GeneralRegisterClass<[i16, f16, bf16], 16, (add (sequence "Reg16b_%u", 0, 3))>;
def Reg32b : GeneralRegisterClass<[i32, f32], 32,
- (add DummyReg, (sequence "Reg32b_%u", 0, 63))>;
+ (add GroupIdSubRegs, GroupCountSubRegs, SpecialReg_SubgroupSize,
+ LocalIdSubRegs, LocalSizeSubRegs, SpecialReg_ActiveMask,
+ EnqueuedLocalSizeSubRegs, SpecialReg_LaneId, SpecialReg_WorkDim,
+ DummyReg,
+ (sequence "Reg32b_%u", 0, 63))>;
def Reg64b : GeneralRegisterClass<[i64, f64], 64,
- (add (sequence "Reg64b_%u", 0, 3))>;
+ (add
+ GlobalOffsetSubRegs, GlobalSizeSubRegs,
+ (sequence "Reg64b_%u", 0, 3))>;
+ def Reg128b : GeneralRegisterClass<[i128], 128, (add (sequence "Reg128b_%u", 0, 3))>;
+
+ //
+ // Vector registers
+ //
+ // 8b subregisters
+ def Reg8bx2 : VecReg<[Reg8b_0, Reg8b_1]> {
+ let SubRegIndices = [sub8_0, sub8_1];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg8bx3 : VecReg<[Reg8b_0, Reg8b_1, Reg8b_2]> {
+ let SubRegIndices = [sub8_0, sub8_1, sub8_2];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg8bx2H : VecReg<[Reg8b_2, Reg8b_3]> {
+ let SubRegIndices = [sub8_2, sub8_3];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg8bx4 : VecReg<[Reg8bx2, Reg8bx2H]> {
+ let SubRegIndices = [sub8_xy, sub8_zw];
+ let CoveredBySubRegs = 1;
+ }
+
+ // 8b vectors
+ def RegV2_8b : GeneralRegisterClass<[v2i8], 8, (add Reg8bx2)>;
+ def RegV3_8b : GeneralRegisterClass<[v3i8], 8, (add Reg8bx3)>;
+ def RegV4_8b : GeneralRegisterClass<[v4i8], 8, (add Reg8bx4)>;
+ let isAllocatable = 0 in {
+ def RegV2_8bH : GeneralRegisterClass<[v2i8], 8, (add Reg8bx2H)>;
+ }
+
+ // 16b subregisters
+ def Reg16bx2 : VecReg<[Reg16b_0, Reg16b_1]> {
+ let SubRegIndices = [sub16_0, sub16_1];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg16bx3 : VecReg<[Reg16b_0, Reg16b_1, Reg16b_2]> {
+ let SubRegIndices = [sub16_0, sub16_1, sub16_2];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg16bx2H : VecReg<[Reg16b_2, Reg16b_3]> {
+ let SubRegIndices = [sub16_2, sub16_3];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg16bx4 : VecReg<[Reg16bx2, Reg16bx2H]> {
+ let SubRegIndices = [sub16_xy, sub16_zw];
+ let CoveredBySubRegs = 1;
+ }
+
+ // 16b vectors
+ def RegV2_16b : GeneralRegisterClass<[v2i16, v2f16, v2bf16], 16, (add Reg16bx2)>;
+ def RegV3_16b : GeneralRegisterClass<[v3i16, v3f16], 16, (add Reg16bx3)>;
+ def RegV4_16b : GeneralRegisterClass<[v4i16, v4f16], 16, (add Reg16bx4)>;
+ let isAllocatable = 0 in {
+ def RegV2_16bH : GeneralRegisterClass<[v2i16, v2f16, v2bf16], 16, (add Reg16bx2H)>;
+ }
+
+ // 32b subregisters
+ def Reg32bx2 : VecReg<[Reg32b_0, Reg32b_1]> {
+ let SubRegIndices = [sub32_0, sub32_1];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg32bx3 : VecReg<[Reg32b_0, Reg32b_1, Reg32b_2]> {
+ let SubRegIndices = [sub32_0, sub32_1, sub32_2];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg32bx2H : VecReg<[Reg32b_2, Reg32b_3]> {
+ let SubRegIndices = [sub32_2, sub32_3];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg32bx4 : VecReg<[Reg32bx2, Reg32bx2H]> {
+ let SubRegIndices = [sub32_xy, sub32_zw];
+ let CoveredBySubRegs = 1;
+ }
+
+ // 32b vectors
+ def RegV2_32b : GeneralRegisterClass<[v2i32, v2f32], 32, (add Reg32bx2)>;
+ def RegV3_32b : GeneralRegisterClass<[v3i32, v3f32], 32, (add Reg32bx3)>;
+ def RegV4_32b : GeneralRegisterClass<[v4i32, v4f32], 32, (add Reg32bx4)>;
+ let isAllocatable = 0 in {
+ def RegV2_32bH : GeneralRegisterClass<[v2i32, v2f32], 32, (add Reg32bx2H)>;
+ }
+
+ // large vector support
+ foreach length = {5-8,16,32} in {
+ // 32b vector registers
+ // Define one VecReg for each 32b vector length
+ def "Reg32bx"#length : VecReg<!foreach(i, !range(length), !cast<Register>("Reg32b_"#i))> {
+ let SubRegIndices = !foreach(i, !range(length), !cast<SubRegIndex>("sub32_"#i));
+ let CoveredBySubRegs = 1;
+ }
+ // 32b vector register classes
+ // Define one GeneralRegisterClass for each 32b vector length
+ def "RegV"#length#"_32b" : GeneralRegisterClass<[!cast<ValueType>("v"#length#"i32"), !cast<ValueType>("v"#length#"f32")], 32, (add !cast<VecReg>("Reg32bx"#length))>;
+ }
+
+ // v64 register: defined without per-element sub-register indices because
+ // LLVM's LaneBitmask (64 bits) cannot represent 64 individual sub-register
+ // lanes plus the existing sub-register indices. All element access uses
+ // extract/insert instructions rather than sub-register operations.
+ def Reg32bx64 : Register<"">;
+ def RegV64_32b : GeneralRegisterClass<[v64i32, v64f32], 32, (add Reg32bx64)>;
+
+ // 64b subregisters
+ def Reg64bx2 : VecReg<[Reg64b_0, Reg64b_1]> {
+ let SubRegIndices = [sub64_0, sub64_1];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg64bx3 : VecReg<[Reg64b_0, Reg64b_1, Reg64b_2]> {
+ let SubRegIndices = [sub64_0, sub64_1, sub64_2];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg64bx2H : VecReg<[Reg64b_2, Reg64b_3]> {
+ let SubRegIndices = [sub64_2, sub64_3];
+ let CoveredBySubRegs = 1;
+ }
+ def Reg64bx4 : VecReg<[Reg64bx2, Reg64bx2H]> {
+ let SubRegIndices = [sub64_xy, sub64_zw];
+ let CoveredBySubRegs = 1;
+ }
+
+ // 64b vectors
+ def RegV2_64b : GeneralRegisterClass<[v2i64, v2f64], 64, (add Reg64bx2)>;
+ def RegV3_64b : GeneralRegisterClass<[v3i64, v3f64], 64, (add Reg64bx3)>;
+ def RegV4_64b : GeneralRegisterClass<[v4i64, v4f64], 64, (add Reg64bx4)>;
+ let isAllocatable = 0 in {
+ def RegV2_64bH : GeneralRegisterClass<[v2i64, v2f64], 64, (add Reg64bx2H)>;
+ }
+}
+
+def SpecialRegNames : GenericTable {
+ let FilterClass = "SpecialRegister";
+ let CppTypeName = "SpecialRegEntry";
+ let Fields = ["Name"];
+}
+
+def lookupSpecialRegByName : SearchIndex {
+ let Table = SpecialRegNames;
+ let Key = ["Name"];
}
diff --git a/llvm/lib/Target/PISA/PISAUtils.cpp b/llvm/lib/Target/PISA/PISAUtils.cpp
new file mode 100644
index 0000000000000..8ecd2c0065ecb
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAUtils.cpp
@@ -0,0 +1,52 @@
+//===-- PISAUtils.cpp ---- PISA Utility Functions -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAUtils.h"
+#include "MCTargetDesc/PISABaseInfo.h"
+#include "PISA.h"
+#include "PISAInstrInfo.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/Demangle/Demangle.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+
+namespace llvm::PISA {
+std::tuple<MachineInstr *, unsigned>
+getDefIgnoringBitcasts(Register Reg, const MachineRegisterInfo &MRI,
+ bool NoVectors) {
+ auto *DefMI = MRI.getVRegDef(Reg);
+ unsigned Opc = DefMI->getOpcode();
+ Register SrcReg = 0;
+ while (Opc == TargetOpcode::G_BITCAST ||
+ isPreISelGenericOptimizationHint(Opc)) {
+ SrcReg = DefMI->getOperand(1).getReg();
+ auto SrcTy = MRI.getType(SrcReg);
+ if (!SrcTy.isValid())
+ break;
+ auto DstTy = MRI.getType(DefMI->getOperand(0).getReg());
+ if (NoVectors && (DstTy.isVector() || SrcTy.isVector()))
+ break;
+ DefMI = MRI.getVRegDef(SrcReg);
+ Opc = DefMI->getOpcode();
+ }
+ unsigned RegIdx = 0;
+ if (SrcReg != 0) {
+ for (unsigned I = 0; I < DefMI->getNumOperands(); I++) {
+ if (DefMI->getOperand(I).isReg() &&
+ DefMI->getOperand(I).getReg() == SrcReg) {
+ RegIdx = I;
+ break;
+ }
+ }
+ }
+ return std::make_tuple(DefMI, RegIdx);
+}
+
+} // namespace llvm::PISA
diff --git a/llvm/lib/Target/PISA/PISAUtils.h b/llvm/lib/Target/PISA/PISAUtils.h
new file mode 100644
index 0000000000000..7975037a75337
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAUtils.h
@@ -0,0 +1,36 @@
+//===-- PISAUtils.h ---- PISA Utility Functions ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAUTILS_H
+#define LLVM_LIB_TARGET_PISA_PISAUTILS_H
+
+#include "MCTargetDesc/PISABaseInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/IR/IRBuilder.h"
+#include <string>
+
+namespace llvm {
+class MachineInstr;
+class MCInst;
+class MachineInstrBuilder;
+class StringRef;
+
+namespace PISA {
+// Similar to getDefIgnoringCopies, but skips any bitcast instructions.
+// Return defining instruction (or nil) and index of the (bitcasted) Reg
+// within defining instruction.
+//
+// - when NoVectors is true, stop search if G_BITCAST src/dst is a vector type
+std::tuple<MachineInstr *, unsigned>
+getDefIgnoringBitcasts(Register Reg, const MachineRegisterInfo &MRI,
+ bool NoVectors = false);
+
+} // namespace PISA
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_PISA_PISAUTILS_H
>From 7fe085861f61e0267efadc6b365f5f03614c4c5c Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 4 Aug 2026 17:47:13 -0700
Subject: [PATCH 2/3] Remove unnecessary edits
---
llvm/lib/Target/PISA/CMakeLists.txt | 1 +
llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt | 1 -
llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h | 2 +-
llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp | 2 +-
llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h | 2 +-
llvm/lib/Target/PISA/PISA.td | 8 ++++----
llvm/lib/Target/PISA/PISAInstrInfo.h | 1 +
llvm/lib/Target/PISA/PISAInstrInfo.td | 2 +-
llvm/lib/Target/PISA/PISARegisterInfo.td | 6 +++---
9 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/PISA/CMakeLists.txt b/llvm/lib/Target/PISA/CMakeLists.txt
index 5c3bfe6ec02bf..321a75d7f88b6 100644
--- a/llvm/lib/Target/PISA/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/CMakeLists.txt
@@ -1,6 +1,7 @@
add_llvm_component_group(PISA)
set(LLVM_TARGET_DEFINITIONS PISA.td)
+
tablegen(LLVM PISAGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM PISAGenInstrInfo.inc -gen-instr-info)
tablegen(LLVM PISAGenMCCodeEmitter.inc -gen-emitter)
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
index 80a21ad6c6aa2..45ea6b7a1d9d9 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/MCTargetDesc/CMakeLists.txt
@@ -18,4 +18,3 @@ add_llvm_component_library(LLVMPISADesc
ADD_TO_COMPONENT
PISA
)
-
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
index 9094807ab82ca..cd96106880fa1 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAInstPrinter.h
@@ -15,8 +15,8 @@
#include "llvm/MC/MCInstPrinter.h"
namespace llvm {
-class PISAInstPrinter : public MCInstPrinter {
+class PISAInstPrinter : public MCInstPrinter {
public:
using MCInstPrinter::MCInstPrinter;
typedef std::function<void(const MCInst *, unsigned, raw_ostream &)>
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
index 3ffd668f3d05a..06cfc4c3748a9 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.cpp
@@ -46,7 +46,7 @@ static MCRegisterInfo *createPISAMCRegisterInfo(const Triple &TT) {
static MCSubtargetInfo *createPISAMCSubtargetInfo(const Triple &TT,
StringRef CPU, StringRef FS) {
- return createPISAMCSubtargetInfoImpl(TT, CPU, /*TuneCPU*/ CPU, FS);
+ return createPISAMCSubtargetInfoImpl(TT, CPU, /*TuneCPU=*/CPU, FS);
}
static MCInstPrinter *createPISAMCInstPrinter(const Triple &T,
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
index 625d0f16077cf..1d1c4a6e0bde1 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISAMCTargetDesc.h
@@ -44,7 +44,7 @@ MCTargetStreamer *createPISANullTargetStreamer(MCStreamer &S);
} // namespace llvm
-// Defines symbolic names for PISA registers. This defines a mapping from
+// Defines symbolic names for PISA registers. This defines a mapping from
// register name to register number.
#define GET_REGINFO_ENUM
#include "PISAGenRegisterInfo.inc"
diff --git a/llvm/lib/Target/PISA/PISA.td b/llvm/lib/Target/PISA/PISA.td
index 0a88485ca6420..011b1cc101faa 100644
--- a/llvm/lib/Target/PISA/PISA.td
+++ b/llvm/lib/Target/PISA/PISA.td
@@ -23,20 +23,20 @@ include "PISAInstrInfo.td"
include "PISACombine.td"
defm : RemapAllTargetPseudoPointerOperands<Reg64b>;
+
def PISAInstrInfo : InstrInfo;
class Proc<string Name, list<SubtargetFeature> Features>
: Processor<Name, NoItineraries, Features>;
-// valid -mcpu values
-def : Proc<"100", [Feature100]>;
+// Valid -mcpu values.
+def : Proc<"100", [Feature100]>;
def PISAInstPrinter : AsmWriter {
- string AsmWriterClassName = "InstPrinter";
+ string AsmWriterClassName = "InstPrinter";
bit isMCAsmWriter = 1;
}
-
def PISA : Target {
let InstructionSet = PISAInstrInfo;
let AssemblyWriters = [PISAInstPrinter];
diff --git a/llvm/lib/Target/PISA/PISAInstrInfo.h b/llvm/lib/Target/PISA/PISAInstrInfo.h
index 4e73ee6e19977..34ec289f7a4c5 100644
--- a/llvm/lib/Target/PISA/PISAInstrInfo.h
+++ b/llvm/lib/Target/PISA/PISAInstrInfo.h
@@ -8,6 +8,7 @@
#ifndef LLVM_LIB_TARGET_PISA_PISAINSTRINFO_H
#define LLVM_LIB_TARGET_PISA_PISAINSTRINFO_H
+
#include "PISARegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
diff --git a/llvm/lib/Target/PISA/PISAInstrInfo.td b/llvm/lib/Target/PISA/PISAInstrInfo.td
index 154e1d8c9c6ea..05a8bfe1263ff 100644
--- a/llvm/lib/Target/PISA/PISAInstrInfo.td
+++ b/llvm/lib/Target/PISA/PISAInstrInfo.td
@@ -1,4 +1,4 @@
-//===-- PISAInstrInfo.td - PISA Instruction defs -----------*- tablegen -*-===//
+//===-- PISAInstrInfo.td - PISA Instruction defs ----------*- tablegen -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/Target/PISA/PISARegisterInfo.td b/llvm/lib/Target/PISA/PISARegisterInfo.td
index ae407b269a2b3..2d757baba2e00 100644
--- a/llvm/lib/Target/PISA/PISARegisterInfo.td
+++ b/llvm/lib/Target/PISA/PISARegisterInfo.td
@@ -1,4 +1,4 @@
-//===----------------------------------------------------------------------===//
+//===-- PISARegisterInfo.td - PISA Register defs ----------*- tablegen -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -13,9 +13,9 @@ let Namespace = "PISA" in {
}
class PISARegisterClass<list<ValueType> regTypes, int alignment, dag regList>
- : RegisterClass<"PISA", regTypes, alignment, regList> {}
+ : RegisterClass<"PISA", regTypes, alignment, regList> {}
- // .reg style registers
+ // .reg style registers.
class GeneralRegisterClass<list<ValueType> regTypes, int alignment, dag regList> :
PISARegisterClass<regTypes, alignment, regList> {
// Keep flags in sync with RegType in RegEncoder
>From b4d1d3ee5dd56b473e741a33addc2be391a874f1 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 4 Aug 2026 17:56:09 -0700
Subject: [PATCH 3/3] Fix formatting
---
llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
index 1443af35549b9..2539286bf4cc5 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISATargetStreamer.cpp
@@ -290,8 +290,8 @@ class PISATargetAsmStreamer final : public PISATargetStreamer {
OS << ".align(" << Param.Align << ") ";
if (Param.hasAS())
OS << ".addrspace("
- << getParamASRepr(static_cast<PISAAS::AddressSpace>(Param.AS))
- << ") ";
+ << getParamASRepr(static_cast<PISAAS::AddressSpace>(Param.AS))
+ << ") ";
if (Param.hasPtrAlign())
OS << ".ptr_align(" << Param.PtrAlign << ") ";
if (!Param.ArgName.empty())
More information about the llvm-branch-commits
mailing list