[llvm] 445a483 - [AMDGPU] Add new GFX11 intrinsic llvm.amdgcn.exp.row
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 16 10:32:40 PDT 2022
Author: Jay Foad
Date: 2022-06-16T18:23:14+01:00
New Revision: 445a483b41a8197a76d6c5044e6290a6cab82ac6
URL: https://github.com/llvm/llvm-project/commit/445a483b41a8197a76d6c5044e6290a6cab82ac6
DIFF: https://github.com/llvm/llvm-project/commit/445a483b41a8197a76d6c5044e6290a6cab82ac6.diff
LOG: [AMDGPU] Add new GFX11 intrinsic llvm.amdgcn.exp.row
Differential Revision: https://reviews.llvm.org/D127671
Added:
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll
Modified:
llvm/include/llvm/IR/IntrinsicsAMDGPU.td
llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
llvm/lib/Target/AMDGPU/EXPInstructions.td
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 24a64302e5e46..54c2892a0f026 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1337,7 +1337,21 @@ def int_amdgcn_exp : Intrinsic <[], [
IntrWillReturn]
>;
-// exp with compr bit set.
+// exp with row_en bit set. Only supported on GFX11+.
+def int_amdgcn_exp_row : Intrinsic <[], [
+ llvm_i32_ty, // tgt,
+ llvm_i32_ty, // en
+ llvm_any_ty, // src0 (f32 or i32)
+ LLVMMatchType<0>, // src1
+ LLVMMatchType<0>, // src2
+ LLVMMatchType<0>, // src3
+ llvm_i1_ty, // done
+ llvm_i32_ty], // row number
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<6>>,
+ IntrWriteMem, IntrInaccessibleMemOnly, IntrWillReturn]
+>;
+
+// exp with compr bit set. Not supported on GFX11+.
def int_amdgcn_exp_compr : Intrinsic <[], [
llvm_i32_ty, // tgt,
llvm_i32_ty, // en
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 99fc2cd136c43..695093322a012 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -621,6 +621,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceInstUsesWith(II, RightShift);
}
case Intrinsic::amdgcn_exp:
+ case Intrinsic::amdgcn_exp_row:
case Intrinsic::amdgcn_exp_compr: {
ConstantInt *En = cast<ConstantInt>(II.getArgOperand(1));
unsigned EnBits = En->getZExtValue();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index cf2881ddd5ee3..01ce1d573fb51 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3116,6 +3116,10 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
constrainOpWithReadfirstlane(MI, MRI, 2);
return;
}
+ case Intrinsic::amdgcn_exp_row:
+ applyDefaultMapping(OpdMapper);
+ constrainOpWithReadfirstlane(MI, MRI, 8); // M0
+ return;
default: {
if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
AMDGPU::lookupRsrcIntrinsic(IntrID)) {
@@ -4542,6 +4546,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
break;
+ case Intrinsic::amdgcn_exp_row:
+ OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+ OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+ OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+ OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+ OpdsMapping[8] = getSGPROpMapping(MI.getOperand(8).getReg(), MRI, *TRI);
+ break;
case Intrinsic::amdgcn_s_sendmsg:
case Intrinsic::amdgcn_s_sendmsghalt: {
// This must be an SGPR, but accept a VGPR.
diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td
index f8742a9ce2ff5..14ba01f0d67c6 100644
--- a/llvm/lib/Target/AMDGPU/EXPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td
@@ -134,6 +134,15 @@ class ExpPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
ExpSrc2:$src2, ExpSrc3:$src3, timm:$vm, 0, timm:$en)
>;
+class ExpRowPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
+ (int_amdgcn_exp_row timm:$tgt, timm:$en,
+ (vt ExpSrc0:$src0), (vt ExpSrc1:$src1),
+ (vt ExpSrc2:$src2), (vt ExpSrc3:$src3),
+ done_val, M0),
+ (Inst timm:$tgt, ExpSrc0:$src0, ExpSrc1:$src1,
+ ExpSrc2:$src2, ExpSrc3:$src3, 0, 0, timm:$en)
+>;
+
class ExpComprPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
(int_amdgcn_exp_compr timm:$tgt, timm:$en,
(vt ExpSrc0:$src0), (vt ExpSrc1:$src1),
@@ -150,6 +159,11 @@ def : ExpPattern<i32, EXP_DONE, -1>;
def : ExpPattern<f32, EXP, 0>;
def : ExpPattern<f32, EXP_DONE, -1>;
+def : ExpRowPattern<i32, EXP_ROW, 0>;
+def : ExpRowPattern<i32, EXP_ROW_DONE, -1>;
+def : ExpRowPattern<f32, EXP_ROW, 0>;
+def : ExpRowPattern<f32, EXP_ROW_DONE, -1>;
+
def : ExpComprPattern<v2i16, EXP, 0>;
def : ExpComprPattern<v2i16, EXP_DONE, -1>;
def : ExpComprPattern<v2f16, EXP, 0>;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll
new file mode 100644
index 0000000000000..a10ea31f2cbcc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+
+declare void @llvm.amdgcn.exp.row.i32(i32, i32, i32, i32, i32, i32, i1, i32)
+declare void @llvm.amdgcn.exp.row.f32(i32, i32, float, float, float, float, i1, i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+define amdgpu_kernel void @undef_i32() #0 {
+; GFX11-LABEL: undef_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: exp pos0 off, off, off, off row_en
+; GFX11-NEXT: exp pos1 off, off, off, off done row_en
+; GFX11-NEXT: s_endpgm
+ call void @llvm.amdgcn.exp.row.i32(i32 12, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i1 false, i32 0)
+ call void @llvm.amdgcn.exp.row.i32(i32 13, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i1 true, i32 0)
+ ret void
+}
+
+define amdgpu_kernel void @undef_f32() #0 {
+; GFX11-LABEL: undef_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: exp pos0 off, off, off, off row_en
+; GFX11-NEXT: exp pos1 off, off, off, off done row_en
+; GFX11-NEXT: s_endpgm
+ call void @llvm.amdgcn.exp.row.f32(i32 12, i32 0, float undef, float undef, float undef, float undef, i1 false, i32 0)
+ call void @llvm.amdgcn.exp.row.f32(i32 13, i32 0, float undef, float undef, float undef, float undef, i1 true, i32 0)
+ ret void
+}
+
+define amdgpu_kernel void @zero_i32() #0 {
+; GFX11-LABEL: zero_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: exp pos0 v0, v0, v0, off row_en
+; GFX11-NEXT: exp pos1 v0, v0, v0, off done row_en
+; GFX11-NEXT: s_endpgm
+ call void @llvm.amdgcn.exp.row.i32(i32 12, i32 7, i32 0, i32 0, i32 0, i32 undef, i1 false, i32 0)
+ call void @llvm.amdgcn.exp.row.i32(i32 13, i32 7, i32 0, i32 0, i32 0, i32 undef, i1 true, i32 0)
+ ret void
+}
+
+define amdgpu_kernel void @one_f32() #0 {
+; GFX11-LABEL: one_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: exp pos0 v0, v0, v0, off row_en
+; GFX11-NEXT: exp pos1 v0, v0, v0, off done row_en
+; GFX11-NEXT: s_endpgm
+ call void @llvm.amdgcn.exp.row.f32(i32 12, i32 7, float 1.0, float 1.0, float 1.0, float undef, i1 false, i32 0)
+ call void @llvm.amdgcn.exp.row.f32(i32 13, i32 7, float 1.0, float 1.0, float 1.0, float undef, i1 true, i32 0)
+ ret void
+}
+
+define amdgpu_kernel void @id_i32() #0 {
+; GFX11-LABEL: id_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: exp pos0 v0, off, off, off done row_en
+; GFX11-NEXT: s_endpgm
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 %id, i32 undef, i32 undef, i32 undef, i1 true, i32 0)
+ ret void
+}
+
+define amdgpu_kernel void @id_arg_i32(i32 %row) #0 {
+; GFX11-LABEL: id_arg_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, s0
+; GFX11-NEXT: exp pos0 v0, off, off, off done row_en
+; GFX11-NEXT: s_endpgm
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 %id, i32 undef, i32 undef, i32 undef, i1 true, i32 %row)
+ ret void
+}
+
+; Divergent row number just causes a readfirstlane for now.
+define amdgpu_kernel void @id_row_i32() #0 {
+; GFX11-SDAG-LABEL: id_row_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0x63
+; GFX11-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX11-SDAG-NEXT: exp pos0 v0, off, off, off done row_en
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: id_row_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0x63
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 m0, v0
+; GFX11-GISEL-NEXT: exp pos0 v1, off, off, off done row_en
+; GFX11-GISEL-NEXT: s_endpgm
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 99, i32 undef, i32 undef, i32 undef, i1 true, i32 %id)
+ ret void
+}
More information about the llvm-commits
mailing list