[llvm] [AMDGPU] Add no return image_sample intrinsics and instructions (PR #97542)
Carl Ritson via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 3 01:52:47 PDT 2024
https://github.com/perlfu created https://github.com/llvm/llvm-project/pull/97542
An appropriately configured image resource descriptor can trigger image_sample instructions to store outputs directly to a linked memory location instead of returning to VGPRs.
This is opaque to the backend as instruction encoding is unchanged; however, a mechanism is require to allow frontends to communicate that these instructions do not require destination VGPRs and store to memory.
Flagging these as stores means they will not be optimized away.
>From 10d15a92164cdbcdeb7015e8b50fad9104727e45 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 3 Jul 2024 16:19:23 +0900
Subject: [PATCH] [AMDGPU] Add no return image_sample intrinsics and
instructions
An appropriately configured image resource descriptor can trigger
image_sample instructions to store outputs directly to a linked
memory location instead of returning to VGPRs.
This is opaque to the backend as instruction encoding is unchanged;
however, a mechanism is require to allow frontends to communicate
that these instructions do not require destination VGPRs and store
to memory.
Flagging these as stores means they will not be optimized away.
---
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 29 ++
.../AMDGPU/AMDGPUInstructionSelector.cpp | 3 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 17 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 2 +
.../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 3 +-
llvm/lib/Target/AMDGPU/MIMGInstructions.td | 223 ++++++--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 31 +-
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 8 +
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 1 +
.../AMDGPU/llvm.amdgcn.image.sample.noret.ll | 479 ++++++++++++++++++
12 files changed, 751 insertions(+), 50 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 71b1e832bde3c..b2d91fd0360c8 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -780,6 +780,20 @@ class AMDGPUDimSampleProfile<string opmod,
let LodClampMip = sample.LodOrClamp;
}
+class AMDGPUDimSampleNoReturnProfile<string opmod,
+ AMDGPUDimProps dim,
+ AMDGPUSampleVariant sample> : AMDGPUDimProfile<opmod, dim> {
+ let IsSample = true;
+ let RetTypes = [];
+ let DataArgs = [];
+ let ExtraAddrArgs = sample.ExtraAddrArgs;
+ let Offset = sample.Offset;
+ let Bias = sample.Bias;
+ let ZCompare = sample.ZCompare;
+ let Gradients = sample.Gradients;
+ let LodClampMip = sample.LodOrClamp;
+}
+
class AMDGPUDimNoSampleProfile<string opmod,
AMDGPUDimProps dim,
list<LLVMType> retty,
@@ -970,6 +984,21 @@ defset list<AMDGPUImageDimIntrinsic> AMDGPUImageDimIntrinsics = {
AMDGPUImageDMaskIntrinsic;
}
+ multiclass AMDGPUImageDimSampleNoReturnDims<string opmod,
+ AMDGPUSampleVariant sample> {
+ foreach dim = AMDGPUDims.NoMsaa in {
+ def !strconcat(NAME, "_", dim.Name) : AMDGPUImageDimIntrinsic<
+ AMDGPUDimSampleNoReturnProfile<opmod, dim, sample>,
+ [IntrWillReturn], [SDNPMemOperand]>;
+ }
+ }
+ foreach sample = AMDGPUSampleVariants in {
+ defm int_amdgcn_image_sample_noret # sample.LowerCaseMod
+ : AMDGPUImageDimSampleNoReturnDims<
+ "SAMPLE" # sample.UpperCaseMod # "_noret", sample>,
+ AMDGPUImageDMaskIntrinsic;
+ }
+
defm int_amdgcn_image_getlod
: AMDGPUImageDimSampleDims<"GET_LOD", AMDGPUSample, 1>,
AMDGPUImageDMaskIntrinsic;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index dcb0f47973c4a..fd00527d73c42 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1883,6 +1883,8 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic(
VDataIn = MI.getOperand(1).getReg();
VDataTy = MRI->getType(VDataIn);
NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
+ } else if (BaseOpcode->NoReturn) {
+ NumVDataDwords = 0;
} else {
VDataOut = MI.getOperand(0).getReg();
VDataTy = MRI->getType(VDataOut);
@@ -3629,6 +3631,7 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
return selectG_INSERT_VECTOR_ELT(I);
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+ case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
const AMDGPU::ImageDimIntrinsicInfo *Intr =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a219d01518458..c99ec79cd5a45 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6322,8 +6322,13 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
const LLT V2S16 = LLT::fixed_vector(2, 16);
unsigned DMask = 0;
- Register VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
- LLT Ty = MRI->getType(VData);
+ Register VData;
+ LLT Ty;
+
+ if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
+ VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
+ Ty = MRI->getType(VData);
+ }
const bool IsAtomicPacked16Bit =
(BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
@@ -6361,7 +6366,11 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
: AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
: AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
- unsigned NewOpcode = NumDefs == 0 ? StoreOpcode : LoadOpcode;
+ unsigned NewOpcode = LoadOpcode;
+ if (BaseOpcode->Store)
+ NewOpcode = StoreOpcode;
+ else if (BaseOpcode->NoReturn)
+ NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
// Track that we legalized this
MI.setDesc(B.getTII().get(NewOpcode));
@@ -6491,7 +6500,7 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
Flags |= 2;
MI.addOperand(MachineOperand::CreateImm(Flags));
- if (BaseOpcode->Store) { // No TFE for stores?
+ if (BaseOpcode->NoReturn) { // No TFE for stores?
// TODO: Handle dmask trim
if (!Ty.isVector() || !IsD16)
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 9e7694f41d6b8..dd30efa826b1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3173,6 +3173,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
}
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+ case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
const AMDGPU::RsrcIntrinsic *RSrcIntrin =
@@ -4843,6 +4844,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
}
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
+ case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
auto IntrID = AMDGPU::getIntrinsicID(MI);
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index b08957d22ee74..0c5a4ef39ae03 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -3864,7 +3864,8 @@ bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst,
int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
int TFEIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
- assert(VDataIdx != -1);
+ if (VDataIdx == -1 && isGFX10Plus()) // no return image_sample
+ return true;
if ((DMaskIdx == -1 || TFEIdx == -1) && isGFX10_AEncoding()) // intersect_ray
return true;
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index e1468bf850cd7..4826969e4c3ae 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -50,6 +50,7 @@ class MIMGBaseOpcode : PredicateControl {
bit MSAA = 0;
bit BVH = 0;
bit A16 = 0;
+ bit NoReturn = 0;
}
def MIMGBaseOpcode : GenericEnum {
@@ -61,7 +62,7 @@ def MIMGBaseOpcodesTable : GenericTable {
let CppTypeName = "MIMGBaseOpcodeInfo";
let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler",
"Gather4", "NumExtraArgs", "Gradients", "G16", "Coordinates",
- "LodOrClampOrMip", "HasD16", "MSAA", "BVH", "A16"];
+ "LodOrClampOrMip", "HasD16", "MSAA", "BVH", "A16", "NoReturn"];
string TypeOf_BaseOpcode = "MIMGBaseOpcode";
let PrimaryKey = ["BaseOpcode"];
@@ -520,6 +521,25 @@ class VSAMPLE_Sampler_gfx12<mimgopc op, string opcode, RegisterClass DataRC,
#!if(BaseOpcode.HasD16, "$d16", "");
}
+class VSAMPLE_Sampler_noret_gfx12<mimgopc op, string opcode,
+ int num_addrs, RegisterClass Addr3RC = VGPR_32,
+ string dns="">
+ : VSAMPLE_gfx12<op.GFX12, (outs), num_addrs, dns, Addr3RC> {
+ let InOperandList = !con(AddrIns,
+ (ins SReg_256:$rsrc),
+ !if(BaseOpcode.Sampler, (ins SReg_128:$samp), (ins)),
+ (ins DMask:$dmask, Dim:$dim, UNorm:$unorm,
+ CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe,
+ LWE:$lwe),
+ !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
+ let AsmString = opcode#" off, "#AddrAsm#", $rsrc"
+ #!if(BaseOpcode.Sampler, ", $samp", "")
+ #"$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"
+ #!if(BaseOpcode.HasD16, "$d16", "");
+ // Force vdata to VGPR0 as no result will be returned.
+ let vdata = 0;
+}
+
multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm,
RegisterClass dst_rc, bit enableDisasm,
bit ExtendedImageInst = 1,
@@ -834,6 +854,7 @@ multiclass MIMG_Store <mimgopc op, string asm, bit has_d16, bit mip = 0> {
let Store = 1;
let LodOrClampOrMip = mip;
let HasD16 = has_d16;
+ let NoReturn = 1;
}
let BaseOpcode = !cast<MIMGBaseOpcode>(NAME) in {
@@ -1135,44 +1156,62 @@ class MIMG_Sampler_gfx90a<mimgopc op, string asm, RegisterClass dst_rc,
#!if(BaseOpcode.HasD16, "$d16", "");
}
+class MIMG_Sampler_OpList_gfx10p<dag OpPrefix, bit HasD16> {
+ dag ret = !con(OpPrefix,
+ (ins SReg_256:$srsrc, SReg_128:$ssamp,
+ DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
+ R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
+ !if(HasD16, (ins D16:$d16), (ins)));
+}
+
+class MIMG_Sampler_Asm_gfx10p<string opcode, string AsmPrefix, bit HasD16> {
+ string ret = opcode#" "#AsmPrefix#", $srsrc, $ssamp$dmask$dim$unorm"
+ #"$cpol$r128$a16$tfe$lwe"
+ #!if(HasD16, "$d16", "");
+}
+
class MIMG_Sampler_gfx10<mimgopc op, string opcode,
RegisterClass DataRC, RegisterClass AddrRC,
string dns="">
: MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> {
- let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256:$srsrc, SReg_128:$ssamp,
- DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
- R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
- !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
- let AsmString = opcode#" $vdata, $vaddr0, $srsrc, $ssamp$dmask$dim$unorm"
- #"$cpol$r128$a16$tfe$lwe"
- #!if(BaseOpcode.HasD16, "$d16", "");
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;
}
class MIMG_Sampler_nsa_gfx10<mimgopc op, string opcode,
RegisterClass DataRC, int num_addrs,
string dns="">
: MIMG_nsa_gfx10<op.GFX10M, (outs DataRC:$vdata), num_addrs, dns> {
- let InOperandList = !con(AddrIns,
- (ins SReg_256:$srsrc, SReg_128:$ssamp, DMask:$dmask,
- Dim:$dim, UNorm:$unorm, CPol:$cpol,
- R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
- !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
- let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc, $ssamp$dmask$dim$unorm"
- #"$cpol$r128$a16$tfe$lwe"
- #!if(BaseOpcode.HasD16, "$d16", "");
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;
+}
+
+class MIMG_Sampler_noret_gfx10<mimgopc op, string opcode,
+ RegisterClass AddrRC,
+ string dns="">
+ : MIMG_gfx10<op.GFX10M, (outs), dns> {
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;
+ // Force vdata to VGPR0 as no result will be returned.
+ let vdata = 0;
+}
+
+class MIMG_Sampler_noret_nsa_gfx10<mimgopc op, string opcode,
+ int num_addrs,
+ string dns="">
+ : MIMG_nsa_gfx10<op.GFX10M, (outs), num_addrs, dns> {
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " off, "#AddrAsm, BaseOpcode.HasD16>.ret;
+ // Force vdata to VGPR0 as no result will be returned.
+ let vdata = 0;
}
class MIMG_Sampler_gfx11<mimgopc op, string opcode,
RegisterClass DataRC, RegisterClass AddrRC,
string dns="">
: MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> {
- let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256:$srsrc, SReg_128:$ssamp,
- DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,
- R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
- !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
- let AsmString = opcode#" $vdata, $vaddr0, $srsrc, $ssamp$dmask$dim$unorm"
- #"$cpol$r128$a16$tfe$lwe"
- #!if(BaseOpcode.HasD16, "$d16", "");
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;
}
class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode,
@@ -1180,14 +1219,26 @@ class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode,
RegisterClass LastVAddrSize, string dns="">
: MIMG_nsa_gfx11<op.GFX11, (outs DataRC:$vdata), num_addrs, dns, [],
LastVAddrSize> {
- let InOperandList = !con(AddrIns,
- (ins SReg_256:$srsrc, SReg_128:$ssamp, DMask:$dmask,
- Dim:$dim, UNorm:$unorm, CPol:$cpol,
- R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),
- !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));
- let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc, $ssamp$dmask$dim$unorm"
- #"$cpol$r128$a16$tfe$lwe"
- #!if(BaseOpcode.HasD16, "$d16", "");
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;
+}
+
+class MIMG_Sampler_noret_gfx11<mimgopc op, string opcode,
+ RegisterClass AddrRC,
+ string dns="">
+ : MIMG_gfx11<op.GFX11, (outs), dns> {
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;
+ let vdata = 0;
+}
+
+class MIMG_Sampler_noret_nsa_gfx11<mimgopc op, string opcode,
+ int num_addrs,
+ RegisterClass LastVAddrSize, string dns="">
+ : MIMG_nsa_gfx11<op.GFX11, (outs), num_addrs, dns, [], LastVAddrSize> {
+ let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;
+ let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, "#AddrAsm, BaseOpcode.HasD16>.ret;
+ let vdata = 0;
}
class MIMGAddrSize<int dw, bit enable_disasm, int AddrDW = dw> {
@@ -1365,6 +1416,57 @@ class MIMG_Sampler_BaseOpcode<AMDGPUSampleVariant sample>
let LodOrClampOrMip = !ne(sample.LodOrClamp, "");
}
+multiclass MIMG_Sampler_NoReturn <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0, bit isG16, string asm> {
+ def "" : MIMG_Sampler_BaseOpcode<sample> {
+ let HasD16 = 1;
+ let G16 = isG16;
+ let NoReturn = 1;
+ }
+
+ let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,
+ mayLoad = 1, mayStore = 1, VDataDwords = 0 in {
+ foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.MachineInstrs in {
+ let VAddrDwords = addr.NumWords in {
+ if op.HAS_GFX10M then {
+ def _V # addr.NumWords # _gfx10
+ : MIMG_Sampler_noret_gfx10 <op, asm, addr.RegClass>;
+ }
+ if op.HAS_GFX11 then {
+ def _V # addr.NumWords # _gfx11
+ : MIMG_Sampler_noret_gfx11 <op, asm, addr.RegClass>;
+ }
+ }
+ }
+
+ foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.NSAInstrs in {
+ let VAddrDwords = addr.NumWords in {
+ if op.HAS_GFX10M then {
+ def _V # addr.NumWords # _nsa_gfx10
+ : MIMG_Sampler_noret_nsa_gfx10<op, asm, addr.NumWords>;
+ }
+ }
+ }
+
+ foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 5/*MaxNSASize*/>.PartialNSAInstrs in {
+ let VAddrDwords = addr.NumWords in {
+ if op.HAS_GFX11 then {
+ def _V # addr.NumWords # _nsa_gfx11
+ : MIMG_Sampler_noret_nsa_gfx11<op, asm, addr.NumWords, addr.RegClass>;
+ }
+ }
+ }
+
+ foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 4/*MaxNSASize*/, 1>.PartialNSAInstrs in {
+ let VAddrDwords = addr.NumWords in {
+ if op.HAS_GFX12 then {
+ def _V # addr.NumWords # _gfx12
+ : VSAMPLE_Sampler_noret_gfx12<op, asm, addr.NumWords, addr.RegClass>;
+ }
+ }
+ }
+ }
+}
+
multiclass MIMG_Sampler <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0,
bit isG16 = 0, bit isGetLod = 0,
string asm = "image_sample"#sample.LowerCaseMod#!if(isG16, "_g16", ""),
@@ -1387,6 +1489,9 @@ multiclass MIMG_Sampler <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0,
let VDataDwords = 5 in
defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_160, 0, ExtendedImageInst, isG16>;
}
+
+ if !not(isGetLod) then
+ defm "_noret" : MIMG_Sampler_NoReturn <op, sample, wqm, isG16, asm>;
}
multiclass MIMG_Sampler_WQM <mimgopc op, AMDGPUSampleVariant sample>
@@ -1754,6 +1859,10 @@ def : MIMGLZMapping<IMAGE_GATHER4_L, IMAGE_GATHER4_LZ>;
def : MIMGLZMapping<IMAGE_GATHER4_C_L, IMAGE_GATHER4_C_LZ>;
def : MIMGLZMapping<IMAGE_GATHER4_L_O, IMAGE_GATHER4_LZ_O>;
def : MIMGLZMapping<IMAGE_GATHER4_C_L_O, IMAGE_GATHER4_C_LZ_O>;
+def : MIMGLZMapping<IMAGE_SAMPLE_L_noret, IMAGE_SAMPLE_LZ_noret>;
+def : MIMGLZMapping<IMAGE_SAMPLE_C_L_noret, IMAGE_SAMPLE_C_LZ_noret>;
+def : MIMGLZMapping<IMAGE_SAMPLE_L_O_noret, IMAGE_SAMPLE_LZ_O_noret>;
+def : MIMGLZMapping<IMAGE_SAMPLE_C_L_O_noret, IMAGE_SAMPLE_C_LZ_O_noret>;
// MIP to NONMIP Optimization Mapping
def : MIMGMIPMapping<IMAGE_LOAD_MIP, IMAGE_LOAD>;
@@ -1776,6 +1885,14 @@ def : MIMGBiasMapping<IMAGE_GATHER4_B_O, IMAGE_GATHER4_O>;
def : MIMGBiasMapping<IMAGE_GATHER4_B_CL_O, IMAGE_GATHER4_CL_O>;
def : MIMGBiasMapping<IMAGE_GATHER4_C_B_O, IMAGE_GATHER4_C_O>;
def : MIMGBiasMapping<IMAGE_GATHER4_C_B_CL_O, IMAGE_GATHER4_C_CL_O>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_noret, IMAGE_SAMPLE_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_noret, IMAGE_SAMPLE_CL_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_noret, IMAGE_SAMPLE_C_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_noret, IMAGE_SAMPLE_C_CL_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_O_noret, IMAGE_SAMPLE_O_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_O_noret, IMAGE_SAMPLE_CL_O_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_O_noret, IMAGE_SAMPLE_C_O_noret>;
+def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_O_noret, IMAGE_SAMPLE_C_CL_O_noret>;
// Offset to NoOffset Optimization Mapping
def : MIMGOffsetMapping<IMAGE_SAMPLE_O, IMAGE_SAMPLE>;
@@ -1818,6 +1935,34 @@ def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16, IMAGE_SAMPLE_CD_G16>;
def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16, IMAGE_SAMPLE_CD_CL_G16>;
def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16, IMAGE_SAMPLE_C_CD_G16>;
def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16, IMAGE_SAMPLE_C_CD_CL_G16>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_O_noret, IMAGE_SAMPLE_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CL_O_noret, IMAGE_SAMPLE_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_noret, IMAGE_SAMPLE_D_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_noret, IMAGE_SAMPLE_D_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_G16_noret, IMAGE_SAMPLE_D_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_G16_noret, IMAGE_SAMPLE_D_CL_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_L_O_noret, IMAGE_SAMPLE_L_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_B_O_noret, IMAGE_SAMPLE_B_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_B_CL_O_noret, IMAGE_SAMPLE_B_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_LZ_O_noret, IMAGE_SAMPLE_LZ_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_O_noret, IMAGE_SAMPLE_C_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CL_O_noret, IMAGE_SAMPLE_C_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_noret, IMAGE_SAMPLE_C_D_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_noret, IMAGE_SAMPLE_C_D_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_G16_noret, IMAGE_SAMPLE_C_D_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_G16_noret, IMAGE_SAMPLE_C_D_CL_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_L_O_noret, IMAGE_SAMPLE_C_L_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_CL_O_noret, IMAGE_SAMPLE_C_B_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_O_noret, IMAGE_SAMPLE_C_B_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_LZ_O_noret, IMAGE_SAMPLE_C_LZ_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_noret, IMAGE_SAMPLE_CD>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_noret, IMAGE_SAMPLE_CD_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_noret, IMAGE_SAMPLE_C_CD_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_noret, IMAGE_SAMPLE_C_CD_CL_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16_noret, IMAGE_SAMPLE_CD_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16_noret, IMAGE_SAMPLE_CD_CL_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16_noret, IMAGE_SAMPLE_C_CD_G16_noret>;
+def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16_noret, IMAGE_SAMPLE_C_CD_CL_G16_noret>;
// G to G16 Optimization Mapping
def : MIMGG16Mapping<IMAGE_SAMPLE_D, IMAGE_SAMPLE_D_G16>;
@@ -1836,3 +1981,19 @@ def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O, IMAGE_SAMPLE_CD_O_G16>;
def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O, IMAGE_SAMPLE_CD_CL_O_G16>;
def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O, IMAGE_SAMPLE_C_CD_O_G16>;
def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O, IMAGE_SAMPLE_C_CD_CL_O_G16>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_noret, IMAGE_SAMPLE_D_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_noret, IMAGE_SAMPLE_D_CL_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_noret, IMAGE_SAMPLE_C_D_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_noret, IMAGE_SAMPLE_C_D_CL_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_O_noret, IMAGE_SAMPLE_D_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_O_noret, IMAGE_SAMPLE_D_CL_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_O_noret, IMAGE_SAMPLE_C_D_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_O_noret, IMAGE_SAMPLE_C_D_CL_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_noret, IMAGE_SAMPLE_CD_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_noret, IMAGE_SAMPLE_CD_CL_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_noret, IMAGE_SAMPLE_C_CD_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_noret, IMAGE_SAMPLE_C_CD_CL_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O_noret, IMAGE_SAMPLE_CD_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O_noret, IMAGE_SAMPLE_CD_CL_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O_noret, IMAGE_SAMPLE_C_CD_O_G16_noret>;
+def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O_noret, IMAGE_SAMPLE_C_CD_CL_O_G16_noret>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 12977af0d7e85..e0cbd908d1d6d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1190,8 +1190,13 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
// TODO: Should images get their own address space?
Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
- if (RsrcIntr->IsImage)
+ const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
+ if (RsrcIntr->IsImage) {
+ const AMDGPU::ImageDimIntrinsicInfo *Intr =
+ AMDGPU::getImageDimIntrinsicInfo(IntrID);
+ BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
Info.align.reset();
+ }
Value *RsrcArg = CI.getArgOperand(RsrcIntr->RsrcArg);
if (auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->getType())) {
@@ -1212,11 +1217,6 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
if (RsrcIntr->IsImage) {
unsigned MaxNumLanes = 4;
- const AMDGPU::ImageDimIntrinsicInfo *Intr
- = AMDGPU::getImageDimIntrinsicInfo(IntrID);
- const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
- AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
-
if (!BaseOpcode->Gather4) {
// If this isn't a gather, we may have excess loaded elements in the
// IR type. Check the dmask for the real number of elements loaded.
@@ -1250,7 +1250,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
Info.flags |= MachineMemOperand::MOStore;
} else {
- // Atomic
+ // Atomic or NoReturn Sampler
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID :
ISD::INTRINSIC_W_CHAIN;
Info.flags |= MachineMemOperand::MOLoad |
@@ -1259,9 +1259,14 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
switch (IntrID) {
default:
- Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
- // XXX - Should this be volatile without known ordering?
- Info.flags |= MachineMemOperand::MOVolatile;
+ if (RsrcIntr->IsImage && BaseOpcode->NoReturn) {
+ // Fake memory access type for no return sampler intrinsics
+ Info.memVT = MVT::i32;
+ } else {
+ // XXX - Should this be volatile without known ordering?
+ Info.flags |= MachineMemOperand::MOVolatile;
+ Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+ }
break;
case Intrinsic::amdgcn_raw_buffer_load_lds:
case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
@@ -7913,7 +7918,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
bool IsG16 = false;
bool IsA16 = false;
SDValue VData;
- int NumVDataDwords;
+ int NumVDataDwords = 0;
bool AdjustRetType = false;
bool IsAtomicPacked16Bit = false;
@@ -7962,7 +7967,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
}
NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
- } else {
+ } else if (!BaseOpcode->NoReturn) {
// Work out the num dwords based on the dmask popcount and underlying type
// and whether packing is supported.
MVT LoadVT = ResultTypes[0].getSimpleVT();
@@ -8255,7 +8260,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
}
- if (BaseOpcode->Store)
+ if (BaseOpcode->NoReturn)
return SDValue(NewNode, 0);
return constructRetValue(DAG, NewNode, OrigResultTypes, IsTexFail,
Subtarget->hasUnpackedD16VMem(), IsD16, DMaskLanes,
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 4c53a081cdb29..6293b7ca724b8 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -699,7 +699,8 @@ class SIInsertWaitcnts : public MachineFunctionPass {
// these should use VM_CNT.
if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
return VMEM_ACCESS;
- if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) {
+ if (Inst.mayStore() &&
+ (!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(Inst))) {
// FLAT and SCRATCH instructions may access scratch. Other VMEM
// instructions do not.
if (SIInstrInfo::isFLAT(Inst) && mayAccessScratchThroughFlat(Inst))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index cc1b9ac0c9ecd..b480d27454b60 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -475,6 +475,8 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
Offset = 0;
// Get appropriate operand, and compute width accordingly.
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
+ if (DataOpIdx == -1)
+ return false; // no return sampler
Width = getOpSize(LdSt, DataOpIdx);
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 77b17a0f2789b..70eae430ad005 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3945,6 +3945,14 @@ def G_AMDGPU_INTRIN_IMAGE_LOAD_D16 : AMDGPUGenericInstruction {
let mayStore = 1;
}
+def G_AMDGPU_INTRIN_IMAGE_LOAD_NORET : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins unknown:$intrin, variable_ops);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+ let mayStore = 1;
+}
+
// This is equivalent to the G_INTRINSIC*, but the operands may have
// been legalized depending on the subtarget requirements.
def G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPUGenericInstruction {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index af2f0bc1a6306..429c3ad335d21 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -404,6 +404,7 @@ struct MIMGBaseOpcodeInfo {
bool MSAA;
bool BVH;
bool A16;
+ bool NoReturn;
};
LLVM_READONLY
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
new file mode 100644
index 0000000000000..3790db6848c4c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
@@ -0,0 +1,479 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -global-isel=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-SDAG,GFX10,GFX10-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -global-isel=1 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-GISEL,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-delay-alu=0 -global-isel=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-SDAG,GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-delay-alu=0 -global-isel=1 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10PLUS-GISEL,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -amdgpu-enable-delay-alu=0 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -amdgpu-enable-delay-alu=0 -global-isel=1 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+
+define amdgpu_ps void @sample_1d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_1d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_1d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_2d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_2d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_2d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.2d.f32(i32 15, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_3d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %r) {
+; GFX10PLUS-LABEL: sample_3d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_3d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.3d.f32(i32 15, float %s, float %t, float %r, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_cube_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %face) {
+; GFX10PLUS-LABEL: sample_cube_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_cube_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.cube.f32(i32 15, float %s, float %t, float %face, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_1darray_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %slice) {
+; GFX10PLUS-LABEL: sample_1darray_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_1darray_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.1darray.f32(i32 15, float %s, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_2darray_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %slice) {
+; GFX10PLUS-LABEL: sample_2darray_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_2darray_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.2darray.f32(i32 15, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_b_1d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
+; GFX10PLUS-LABEL: sample_b_1d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample_b off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_b_1d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample_b off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.b.1d.f32(i32 15, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_b_2d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_b_2d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample_b off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_b_2d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample_b off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.b.2d.f32(i32 15, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_c_1d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
+; GFX10PLUS-LABEL: sample_c_1d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample_c off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_c_1d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample_c off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.c.1d.f32(i32 15, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_c_2d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_c_2d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: image_sample_c off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_c_2d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: image_sample_c off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.c.2d.f32(i32 15, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_d_1d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s) {
+; GFX10PLUS-LABEL: sample_d_1d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: image_sample_d off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_d_1d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_sample_d off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.d.1d.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_d_2d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
+; GFX10PLUS-LABEL: sample_d_2d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: image_sample_d off, v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_d_2d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_sample_d off, [v0, v1, v2, v[3:5]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.d.2d.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_l_1d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %lod) {
+; GFX10PLUS-LABEL: sample_l_1d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: image_sample_l off, v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_l_1d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_sample_l off, [v0, v1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.l.1d.f32(i32 15, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @sample_l_2d_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
+; GFX10PLUS-LABEL: sample_l_2d_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: image_sample_l off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_l_2d_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_sample_l off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.l.2d.f32(i32 15, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_ps <4 x float> @sample_noret_mix_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_noret_mix_1:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: sample_noret_mix_1:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_noret_mix_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-LABEL: sample_noret_mix_2:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-NEXT: v_mov_b32_e32 v4, v0
+; GFX10PLUS-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: sample_noret_mix_2:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-NEXT: v_mov_b32_e32 v4, v0
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
+main_body:
+ %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_noret_mix_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-SDAG-LABEL: sample_noret_mix_3:
+; GFX10PLUS-SDAG: ; %bb.0: ; %main_body
+; GFX10PLUS-SDAG-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-SDAG-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-SDAG-NEXT: image_sample v1, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX10PLUS-SDAG-NEXT: image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX10PLUS-GISEL-LABEL: sample_noret_mix_3:
+; GFX10PLUS-GISEL: ; %bb.0: ; %main_body
+; GFX10PLUS-GISEL-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-GISEL-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-GISEL-NEXT: image_sample v[1:4], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX10PLUS-GISEL-NEXT: image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-LABEL: sample_noret_mix_3:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-SDAG-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-SDAG-NEXT: image_sample v1, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x1
+; GFX12-SDAG-NEXT: image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: sample_noret_mix_3:
+; GFX12-GISEL: ; %bb.0: ; %main_body
+; GFX12-GISEL-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-GISEL-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-GISEL-NEXT: image_sample v[1:4], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x1
+; GFX12-GISEL-NEXT: image_sample v[0:3], v1, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+main_body:
+ %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %v.0 = extractelement <4 x float> %v, i32 0
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %u = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %v.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret <4 x float> %u
+}
+
+define amdgpu_ps <4 x float> @sample_noret_mix_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+; GFX10PLUS-SDAG-LABEL: sample_noret_mix_4:
+; GFX10PLUS-SDAG: ; %bb.0: ; %main_body
+; GFX10PLUS-SDAG-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-SDAG-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-SDAG-NEXT: image_sample v4, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX10PLUS-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(2)
+; GFX10PLUS-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX10PLUS-GISEL-LABEL: sample_noret_mix_4:
+; GFX10PLUS-GISEL: ; %bb.0: ; %main_body
+; GFX10PLUS-GISEL-NEXT: s_mov_b32 s12, exec_lo
+; GFX10PLUS-GISEL-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX10PLUS-GISEL-NEXT: image_sample v[4:7], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX10PLUS-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX10PLUS-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(2)
+; GFX10PLUS-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10PLUS-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX12-SDAG-LABEL: sample_noret_mix_4:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-SDAG-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-SDAG-NEXT: image_sample v4, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x1
+; GFX12-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x2
+; GFX12-SDAG-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: sample_noret_mix_4:
+; GFX12-GISEL: ; %bb.0: ; %main_body
+; GFX12-GISEL-NEXT: s_mov_b32 s12, exec_lo
+; GFX12-GISEL-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-GISEL-NEXT: image_sample v[4:7], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x1
+; GFX12-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_and_b32 exec_lo, exec_lo, s12
+; GFX12-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: image_sample off, v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x2
+; GFX12-GISEL-NEXT: image_sample off, v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+main_body:
+ %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %v.0 = extractelement <4 x float> %v, i32 0
+ %v.1 = extractelement <4 x float> %v, i32 0
+ %v.2 = extractelement <4 x float> %v, i32 0
+ %v.3 = extractelement <4 x float> %v, i32 0
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %v.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %u = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %v.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %u.0 = extractelement <4 x float> %u, i32 0
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %v.2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %v.3, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ call void @llvm.amdgcn.image.sample.noret.1d.f32(i32 15, float %u.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret <4 x float> %u
+}
+
+define amdgpu_ps void @sample_d_1d_g16_noret(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
+; GFX10PLUS-LABEL: sample_d_1d_g16_noret:
+; GFX10PLUS: ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10PLUS-NEXT: s_endpgm
+;
+; GFX12-LABEL: sample_d_1d_g16_noret:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.image.sample.noret.d.1d.f16.f32(i32 15, half %dsdh, half %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ ret void
+}
+
+declare void @llvm.amdgcn.image.sample.noret.1d.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.2d.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.3d.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.cube.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.1darray.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.2darray.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.noret.b.1d.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.b.2d.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.noret.c.1d.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.c.2d.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.noret.d.1d.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.d.2d.f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare void @llvm.amdgcn.image.sample.noret.l.1d.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+declare void @llvm.amdgcn.image.sample.noret.l.2d.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare void @llvm.amdgcn.image.sample.noret.d.1d.f16.f32(i32, half, half, float, <8 x i32>, <4 x i32>, i1, i32, i32) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX10-GISEL: {{.*}}
+; GFX10-SDAG: {{.*}}
+; GFX11: {{.*}}
+; GFX11-GISEL: {{.*}}
+; GFX11-SDAG: {{.*}}
More information about the llvm-commits
mailing list