[llvm] [AMDGPU] Fix s_buffer_load crash for illegal result types (PR #215483)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 08:24:12 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215483
>From dcf9ec6c7e8a8bb0873234307d5cf85ef9d37828 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 09:21:39 +0200
Subject: [PATCH 1/6] [AMDGPU] Fix s_buffer_load crash for illegal result types
i1, i4, v2i1, v3i16, v6i8, and i128 had no SBUFFER_LOAD selection pattern and crashed
Load a legal i32/vNi32 carrier instead and narrow/bitcast down to the requested type
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 76 ++-
...llvm.amdgcn.s.buffer.load.illegal-types.ll | 507 ++++++++++++++++++
2 files changed, 540 insertions(+), 43 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fb5c4279af4b7..ad19049f39d0c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1017,7 +1017,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_WO_CHAIN,
{MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
- MVT::i8},
+ MVT::i8, MVT::i1, MVT::i4, MVT::v2i1, MVT::v3i16,
+ MVT::v6i8},
Custom);
setOperationAction(ISD::INTRINSIC_W_CHAIN,
@@ -8404,53 +8405,15 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
case Intrinsic::amdgcn_s_buffer_load: {
- // Lower llvm.amdgcn.s.buffer.load.(i8, u8) intrinsics. First, we generate
- // s_buffer_load_u8 for signed and unsigned load instructions. Next, DAG
- // combiner tries to merge the s_buffer_load_u8 with a sext instruction
- // (performSignExtendInRegCombine()) and it replaces s_buffer_load_u8 with
- // s_buffer_load_i8.
- if (!Subtarget->hasScalarSubwordLoads())
- return;
+ // lowerSBuffer already handles every illegal result type.
SDValue Op = SDValue(N, 0);
SDValue Rsrc = Op.getOperand(1);
SDValue Offset = Op.getOperand(2);
SDValue CachePolicy = Op.getOperand(3);
EVT VT = Op.getValueType();
- assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n");
SDLoc DL(Op);
- MachineFunction &MF = DAG.getMachineFunction();
- const DataLayout &DataLayout = DAG.getDataLayout();
- Align Alignment =
- DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext()));
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(),
- MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
- MachineMemOperand::MOInvariant,
- VT.getStoreSize(), Alignment);
- SDValue LoadVal;
- if (!Offset->isDivergent()) {
- SDValue Ops[] = {DAG.getEntryNode(), // Chain
- Rsrc, // source register
- Offset, CachePolicy};
- SDValue BufferLoad = DAG.getMemIntrinsicNode(
- AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
- DAG.getVTList(MVT::i32, MVT::Other), Ops, VT, MMO);
- LoadVal = DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
- } else {
- SDValue Ops[] = {
- DAG.getEntryNode(), // Chain
- Rsrc, // rsrc
- DAG.getConstant(0, DL, MVT::i32), // vindex
- {}, // voffset
- {}, // soffset
- {}, // offset
- CachePolicy, // cachepolicy
- DAG.getTargetConstant(0, DL, MVT::i1), // idxen
- };
- setBufferOffsets(Offset, DAG, &Ops[3], Align(4));
- LoadVal = handleByteShortBufferLoads(DAG, VT, DL, Ops, MMO);
- }
- Results.push_back(LoadVal);
+ Results.push_back(lowerSBuffer(VT, VT, DL, DAG.getEntryNode(), Rsrc,
+ Offset, CachePolicy, DAG));
return;
}
case Intrinsic::amdgcn_dead: {
@@ -10884,8 +10847,9 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
- // Widen vec3 load to vec4.
+ // Widen vec3 load to vec4. Only 32-bit elements have a vec4 pattern.
if (VT.isVector() && VT.getVectorNumElements() == 3 &&
+ VT.getVectorElementType().getSizeInBits() == 32 &&
!Subtarget->hasScalarDwordx3Loads()) {
EVT WidenedVT =
EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
@@ -10900,6 +10864,32 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
return Subvector;
}
+ // No SBUFFER_LOAD pattern for this width (i1 is a legal type but still
+ // has none). Load a legal i32/vNi32 carrier and narrow/bitcast down.
+ if ((!isTypeLegal(VT) || VT == MVT::i1) && MemVT == VT) {
+ unsigned Bits = VT.getSizeInBits();
+ EVT CarrierVT = Bits <= 32 ? EVT(MVT::i32)
+ : EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+ divideCeil(Bits, 32));
+ SDValue CarrierLoad = DAG.getMemIntrinsicNode(
+ AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(CarrierVT, MVT::Other),
+ Ops, CarrierVT,
+ MF.getMachineMemOperand(MMO, 0, CarrierVT.getStoreSize()));
+ EVT CarrierIntVT =
+ EVT::getIntegerVT(*DAG.getContext(), CarrierVT.getSizeInBits());
+ SDValue AsInt = DAG.getNode(ISD::BITCAST, DL, CarrierIntVT, CarrierLoad);
+ EVT NarrowIntVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
+ SDValue Narrow = NarrowIntVT == CarrierIntVT
+ ? AsInt
+ : DAG.getNode(ISD::TRUNCATE, DL, NarrowIntVT, AsInt);
+ SDValue Result = VT == NarrowIntVT
+ ? Narrow
+ : DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
+ if (HasChainResult)
+ return DAG.getMergeValues({Result, CarrierLoad.getValue(1)}, DL);
+ return Result;
+ }
+
return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
DAG.getVTList(VT, MVT::Other), Ops, MemVT,
MMO);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
new file mode 100644
index 0000000000000..91ee72335474d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -0,0 +1,507 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=amdgpu6.00 | FileCheck %s -check-prefixes=GFX67,GFX6
+; RUN: llc < %s -mtriple=amdgpu7.01 | FileCheck %s -check-prefixes=GFX67,GFX7
+; RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck %s -check-prefixes=GFX8
+; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck %s -check-prefixes=GFX910,GFX9
+; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s -check-prefixes=GFX910,GFX10
+; RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GFX11
+; RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc < %s -mtriple=amdgpu12.50 -amdgpu-enable-vopd=0 -mattr=-wait-xcnt | FileCheck %s -check-prefixes=GFX1250
+
+; Result types with no direct SBUFFER_LOAD pattern used to ICE in the type
+; legalizer. GlobalISel is not covered: it hits a separate, pre-existing
+; assertion in AMDGPULegalizerInfo::legalizeSBufferLoad for these widths.
+
+define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_i1:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: s_buffer_load_dword s4, s[0:3], 0x0
+; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s0, s2
+; GFX67-NEXT: s_mov_b32 s1, s2
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_and_b32 s4, s4, 1
+; GFX67-NEXT: v_mov_b32_e32 v2, s4
+; GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_i1:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s0, s0, 1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_i1:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GFX910-NEXT: s_and_b32 s0, s0, 1
+; GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_i1:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_i1:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s0, s0, 1
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_i1:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s0, s0, 1
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32> %desc, i32 0, i32 0)
+ store i1 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_i4:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: s_buffer_load_dword s4, s[0:3], 0x0
+; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s0, s2
+; GFX67-NEXT: s_mov_b32 s1, s2
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_and_b32 s4, s4, 15
+; GFX67-NEXT: v_mov_b32_e32 v2, s4
+; GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_i4:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s0, s0, 15
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_i4:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GFX910-NEXT: s_and_b32 s0, s0, 15
+; GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_i4:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, s0, 15
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_i4:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s0, s0, 15
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_i4:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s0, s0, 15
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call i4 @llvm.amdgcn.s.buffer.load.i4(<4 x i32> %desc, i32 0, i32 0)
+ store i4 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_v2i1:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: s_buffer_load_dword s4, s[0:3], 0x0
+; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s0, s2
+; GFX67-NEXT: s_mov_b32 s1, s2
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_and_b32 s4, s4, 3
+; GFX67-NEXT: v_mov_b32_e32 v2, s4
+; GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_v2i1:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s0, s0, 3
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_byte v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_v2i1:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GFX910-NEXT: s_and_b32 s0, s0, 3
+; GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_v2i1:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, s0, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_v2i1:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s0, s0, 3
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_v2i1:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s0, s0, 3
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call <2 x i1> @llvm.amdgcn.s.buffer.load.v2i1(<4 x i32> %desc, i32 0, i32 0)
+ store <2 x i1> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX6-LABEL: s_buffer_load_v3i16:
+; GFX6: ; %bb.0: ; %main_body
+; GFX6-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s0, s2
+; GFX6-NEXT: s_mov_b32 s1, s2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s5
+; GFX6-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: s_buffer_load_v3i16:
+; GFX7: ; %bb.0: ; %main_body
+; GFX7-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s5
+; GFX7-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_v3i16:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-NEXT: flat_store_short v[2:3], v4
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_buffer_load_v3i16:
+; GFX9: ; %bb.0: ; %main_body
+; GFX9-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: global_store_short v[0:1], v2, off offset:4
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: s_buffer_load_v3i16:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v3, s0
+; GFX10-NEXT: global_store_short v[0:1], v2, off offset:4
+; GFX10-NEXT: global_store_dword v[0:1], v3, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_v3i16:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s1
+; GFX11-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX11-NEXT: global_store_b32 v[0:1], v3, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_v3i16:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: v_mov_b32_e32 v2, s1
+; GFX1200-NEXT: v_mov_b32_e32 v3, s0
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX1200-NEXT: global_store_b32 v[0:1], v3, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_v3i16:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s1
+; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX1250-NEXT: global_store_b32 v[0:1], v3, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call <3 x i16> @llvm.amdgcn.s.buffer.load.v3i16(<4 x i32> %desc, i32 0, i32 0)
+ store <3 x i16> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX6-LABEL: s_buffer_load_v6i8:
+; GFX6: ; %bb.0: ; %main_body
+; GFX6-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s0, s2
+; GFX6-NEXT: s_mov_b32 s1, s2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s5
+; GFX6-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: s_buffer_load_v6i8:
+; GFX7: ; %bb.0: ; %main_body
+; GFX7-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s5
+; GFX7-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_v6i8:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-NEXT: flat_store_short v[2:3], v4
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_buffer_load_v6i8:
+; GFX9: ; %bb.0: ; %main_body
+; GFX9-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: global_store_short v[0:1], v2, off offset:4
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: s_buffer_load_v6i8:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v3, s0
+; GFX10-NEXT: global_store_short v[0:1], v2, off offset:4
+; GFX10-NEXT: global_store_dword v[0:1], v3, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_v6i8:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s1
+; GFX11-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX11-NEXT: global_store_b32 v[0:1], v3, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_v6i8:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: v_mov_b32_e32 v2, s1
+; GFX1200-NEXT: v_mov_b32_e32 v3, s0
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX1200-NEXT: global_store_b32 v[0:1], v3, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_v6i8:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s1
+; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off offset:4
+; GFX1250-NEXT: global_store_b32 v[0:1], v3, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call <6 x i8> @llvm.amdgcn.s.buffer.load.v6i8(<4 x i32> %desc, i32 0, i32 0)
+ store <6 x i8> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_i128(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_i128:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GFX67-NEXT: s_mov_b32 s6, 0
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s4, s6
+; GFX67-NEXT: s_mov_b32 s5, s6
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: v_mov_b32_e32 v5, s3
+; GFX67-NEXT: v_mov_b32_e32 v4, s2
+; GFX67-NEXT: v_mov_b32_e32 v3, s1
+; GFX67-NEXT: v_mov_b32_e32 v2, s0
+; GFX67-NEXT: buffer_store_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_i128:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_i128:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GFX910-NEXT: v_mov_b32_e32 v5, s3
+; GFX910-NEXT: v_mov_b32_e32 v4, s2
+; GFX910-NEXT: v_mov_b32_e32 v3, s1
+; GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GFX910-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_i128:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-NEXT: v_mov_b32_e32 v4, s2
+; GFX11-NEXT: v_mov_b32_e32 v3, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_i128:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: v_mov_b32_e32 v5, s3
+; GFX1200-NEXT: v_mov_b32_e32 v4, s2
+; GFX1200-NEXT: v_mov_b32_e32 v3, s1
+; GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GFX1200-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_i128:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32> %desc, i32 0, i32 0)
+ store i128 %load, ptr addrspace(1) %out
+ ret void
+}
+
+declare i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32>, i32, i32)
+declare i4 @llvm.amdgcn.s.buffer.load.i4(<4 x i32>, i32, i32)
+declare <2 x i1> @llvm.amdgcn.s.buffer.load.v2i1(<4 x i32>, i32, i32)
+declare <3 x i16> @llvm.amdgcn.s.buffer.load.v3i16(<4 x i32>, i32, i32)
+declare <6 x i8> @llvm.amdgcn.s.buffer.load.v6i8(<4 x i32>, i32, i32)
+declare i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32>, i32, i32)
>From ef7c017a06e073328ba5973797d74958472f241d Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 13:28:27 +0200
Subject: [PATCH 2/6] Address comments
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 67 ++--
....amdgcn.ptr.s.buffer.load.illegal-types.ll | 348 ++++++++++++++++++
...llvm.amdgcn.s.buffer.load.illegal-types.ll | 119 ++++++
3 files changed, 505 insertions(+), 29 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ad19049f39d0c..cec3611b2cf8e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1023,9 +1023,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_W_CHAIN,
{MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
- MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
- MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
- MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
+ MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
+ MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other,
+ MVT::f16, MVT::i16, MVT::bf16, MVT::i8,
+ MVT::i128, MVT::i1, MVT::i4, MVT::v2i1,
+ MVT::v6i8},
Custom);
setOperationAction(ISD::INTRINSIC_VOID,
@@ -10824,6 +10826,39 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
MemVT.getStoreSize(), Alignment);
}
+ // No SBUFFER_LOAD pattern exists for these widths; a broader
+ // "!isTypeLegal(VT)" check would also swallow i8/i16 and 32-bit-element
+ // vec3 types, which have their own faster lowerings below.
+ auto NeedsCarrierLoad = [](EVT VT) {
+ return VT == MVT::i1 || VT == MVT::i4 || VT == MVT::v2i1 ||
+ VT == MVT::v3i16 || VT == MVT::v6i8 || VT == MVT::i128;
+ };
+ if (NeedsCarrierLoad(VT) && MemVT == VT) {
+ unsigned Bits = VT.getSizeInBits();
+ EVT CarrierVT = Bits <= 32 ? EVT(MVT::i32)
+ : EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+ divideCeil(Bits, 32));
+ Align CarrierAlign = DAG.getDataLayout().getABITypeAlign(
+ CarrierVT.getTypeForEVT(*DAG.getContext()));
+ MachineMemOperand *CarrierMMO =
+ MF.getMachineMemOperand(MMO->getPointerInfo(), MMO->getFlags(),
+ CarrierVT.getStoreSize(), CarrierAlign);
+ SDValue CarrierLoad = lowerSBuffer(CarrierVT, CarrierVT, DL, Chain, Rsrc,
+ Offset, CachePolicy, DAG, CarrierMMO);
+ EVT CarrierIntVT =
+ EVT::getIntegerVT(*DAG.getContext(), CarrierVT.getSizeInBits());
+ SDValue AsInt = DAG.getNode(ISD::BITCAST, DL, CarrierIntVT, CarrierLoad);
+ EVT NarrowIntVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
+ SDValue Narrow = NarrowIntVT == CarrierIntVT
+ ? AsInt
+ : DAG.getNode(ISD::TRUNCATE, DL, NarrowIntVT, AsInt);
+ SDValue Result =
+ VT == NarrowIntVT ? Narrow : DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
+ if (HasChainResult)
+ return DAG.getMergeValues({Result, CarrierLoad.getValue(1)}, DL);
+ return Result;
+ }
+
if (!Offset->isDivergent()) {
SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
@@ -10864,32 +10899,6 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
return Subvector;
}
- // No SBUFFER_LOAD pattern for this width (i1 is a legal type but still
- // has none). Load a legal i32/vNi32 carrier and narrow/bitcast down.
- if ((!isTypeLegal(VT) || VT == MVT::i1) && MemVT == VT) {
- unsigned Bits = VT.getSizeInBits();
- EVT CarrierVT = Bits <= 32 ? EVT(MVT::i32)
- : EVT::getVectorVT(*DAG.getContext(), MVT::i32,
- divideCeil(Bits, 32));
- SDValue CarrierLoad = DAG.getMemIntrinsicNode(
- AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(CarrierVT, MVT::Other),
- Ops, CarrierVT,
- MF.getMachineMemOperand(MMO, 0, CarrierVT.getStoreSize()));
- EVT CarrierIntVT =
- EVT::getIntegerVT(*DAG.getContext(), CarrierVT.getSizeInBits());
- SDValue AsInt = DAG.getNode(ISD::BITCAST, DL, CarrierIntVT, CarrierLoad);
- EVT NarrowIntVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
- SDValue Narrow = NarrowIntVT == CarrierIntVT
- ? AsInt
- : DAG.getNode(ISD::TRUNCATE, DL, NarrowIntVT, AsInt);
- SDValue Result = VT == NarrowIntVT
- ? Narrow
- : DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
- if (HasChainResult)
- return DAG.getMergeValues({Result, CarrierLoad.getValue(1)}, DL);
- return Result;
- }
-
return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
DAG.getVTList(VT, MVT::Other), Ops, MemVT,
MMO);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
new file mode 100644
index 0000000000000..f822af217f120
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX6
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX9
+; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX11
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200
+
+; llvm.amdgcn.ptr.s.buffer.load lowers through INTRINSIC_W_CHAIN, which was
+; missing Custom for these result types.
+
+define amdgpu_kernel void @ptr_s_buffer_load_i1(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
+; GFX6-LABEL: ptr_s_buffer_load_i1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_buffer_load_dword s4, s[0:3], s4
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_and_b32 s4, s4, 1
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_buffer_load_dword s7, s[0:3], s6 offset:0x0
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s0, s7, 1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s2, s2, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i1:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s2, s2, 1
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX1200-NEXT: s_endpgm
+ %load = call i1 @llvm.amdgcn.ptr.s.buffer.load.i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i1 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_i4(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
+; GFX6-LABEL: ptr_s_buffer_load_i4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_buffer_load_dword s4, s[0:3], s4
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_and_b32 s4, s4, 15
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i4:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_buffer_load_dword s7, s[0:3], s6 offset:0x0
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s0, s7, 15
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s2, s2, 15
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i4:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s2, s2, 15
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX1200-NEXT: s_endpgm
+ %load = call i4 @llvm.amdgcn.ptr.s.buffer.load.i4(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i4 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_v2i1(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
+; GFX6-LABEL: ptr_s_buffer_load_v2i1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_buffer_load_dword s4, s[0:3], s4
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_and_b32 s4, s4, 3
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_v2i1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_buffer_load_dword s7, s[0:3], s6 offset:0x0
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s0, s7, 3
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_byte v0, v1, s[4:5]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_v2i1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_v2i1:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_and_b32 s2, s2, 3
+; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
+; GFX1200-NEXT: s_endpgm
+ %load = call <2 x i1> @llvm.amdgcn.ptr.s.buffer.load.v2i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store <2 x i1> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_v6i8(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
+; GFX6-LABEL: ptr_s_buffer_load_v6i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], s4
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
+; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_v6i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: s_load_dword s10, s[8:9], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], s10 offset:0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: global_store_short v0, v1, s[6:7] offset:4
+; GFX9-NEXT: global_store_dword v0, v2, s[6:7]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_v6i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b16 v0, v1, s[2:3] offset:4
+; GFX11-NEXT: global_store_b32 v0, v2, s[2:3]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_v6i8:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
+; GFX1200-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s1
+; GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: global_store_b16 v0, v1, s[2:3] offset:4
+; GFX1200-NEXT: global_store_b32 v0, v2, s[2:3]
+; GFX1200-NEXT: s_endpgm
+ %load = call <6 x i8> @llvm.amdgcn.ptr.s.buffer.load.v6i8(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store <6 x i8> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_i128(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
+; GFX6-LABEL: ptr_s_buffer_load_i128:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], s4
+; GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: s_load_dword s12, s[8:9], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_buffer_load_dwordx4 s[4:7], s[0:3], s12 offset:0x0
+; GFX9-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[10:11]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i128:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
+; GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
+; GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX1200-NEXT: v_mov_b32_e32 v2, s2
+; GFX1200-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GFX1200-NEXT: s_endpgm
+ %load = call i128 @llvm.amdgcn.ptr.s.buffer.load.i128(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i128 %load, ptr addrspace(1) %out
+ ret void
+}
+
+declare i1 @llvm.amdgcn.ptr.s.buffer.load.i1(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
+declare i4 @llvm.amdgcn.ptr.s.buffer.load.i4(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
+declare <2 x i1> @llvm.amdgcn.ptr.s.buffer.load.v2i1(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
+declare <6 x i8> @llvm.amdgcn.ptr.s.buffer.load.v6i8(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
+declare i128 @llvm.amdgcn.ptr.s.buffer.load.i128(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
index 91ee72335474d..10c74b1f147d5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -499,6 +499,125 @@ main_body:
ret void
}
+; A divergent offset takes the MUBUF lowering instead of SMEM.
+define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %index, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_i1_divergent:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s0, s2
+; GFX67-NEXT: s_mov_b32 s1, s2
+; GFX67-NEXT: s_waitcnt vmcnt(0)
+; GFX67-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX67-NEXT: buffer_store_byte v0, v[1:2], s[0:3], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_i1_divergent:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT: flat_store_byte v[1:2], v0
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_i1_divergent:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX910-NEXT: s_waitcnt vmcnt(0)
+; GFX910-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX910-NEXT: global_store_byte v[1:2], v0, off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_i1_divergent:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: global_store_b8 v[1:2], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_i1_divergent:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1200-NEXT: global_store_b8 v[1:2], v0, off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_i1_divergent:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
+; GFX1250-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-NEXT: v_mov_b32_e32 v2, v1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1250-NEXT: global_store_b8 v[2:3], v0, off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32> %desc, i32 %index, i32 0)
+ store i1 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_buffer_load_i128_divergent(<4 x i32> inreg %desc, i32 %index, ptr addrspace(1) %out) {
+; GFX67-LABEL: s_buffer_load_i128_divergent:
+; GFX67: ; %bb.0: ; %main_body
+; GFX67-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s0, s2
+; GFX67-NEXT: s_mov_b32 s1, s2
+; GFX67-NEXT: s_waitcnt vmcnt(0)
+; GFX67-NEXT: buffer_store_dwordx4 v[3:6], v[1:2], s[0:3], 0 addr64
+; GFX67-NEXT: s_endpgm
+;
+; GFX8-LABEL: s_buffer_load_i128_divergent:
+; GFX8: ; %bb.0: ; %main_body
+; GFX8-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: flat_store_dwordx4 v[1:2], v[3:6]
+; GFX8-NEXT: s_endpgm
+;
+; GFX910-LABEL: s_buffer_load_i128_divergent:
+; GFX910: ; %bb.0: ; %main_body
+; GFX910-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GFX910-NEXT: s_waitcnt vmcnt(0)
+; GFX910-NEXT: global_store_dwordx4 v[1:2], v[3:6], off
+; GFX910-NEXT: s_endpgm
+;
+; GFX11-LABEL: s_buffer_load_i128_divergent:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: buffer_load_b128 v[3:6], v0, s[0:3], 0 offen
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b128 v[1:2], v[3:6], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: s_buffer_load_i128_divergent:
+; GFX1200: ; %bb.0: ; %main_body
+; GFX1200-NEXT: buffer_load_b128 v[3:6], v0, s[0:3], null offen
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: global_store_b128 v[1:2], v[3:6], off
+; GFX1200-NEXT: s_endpgm
+;
+; GFX1250-LABEL: s_buffer_load_i128_divergent:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: v_mov_b32_e32 v5, v2
+; GFX1250-NEXT: v_mov_b32_e32 v4, v1
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX1250-NEXT: s_endpgm
+main_body:
+ %load = call i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32> %desc, i32 %index, i32 0)
+ store i128 %load, ptr addrspace(1) %out
+ ret void
+}
+
declare i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32>, i32, i32)
declare i4 @llvm.amdgcn.s.buffer.load.i4(<4 x i32>, i32, i32)
declare <2 x i1> @llvm.amdgcn.s.buffer.load.v2i1(<4 x i32>, i32, i32)
>From fa1b6ac2e42294f47b6e044a47e62fa0d74675f1 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 17 Aug 2026 08:13:54 +0200
Subject: [PATCH 3/6] Address comments
---
.../llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll | 14 ++++----------
.../llvm.amdgcn.s.buffer.load.illegal-types.ll | 7 -------
2 files changed, 4 insertions(+), 17 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
index f822af217f120..5613580e0aa3f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX6
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX9
-; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX11
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200
+; RUN: llc -mtriple=amdgpu6.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX6
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX9
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX11
+; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200
; llvm.amdgcn.ptr.s.buffer.load lowers through INTRINSIC_W_CHAIN, which was
; missing Custom for these result types.
@@ -340,9 +340,3 @@ define amdgpu_kernel void @ptr_s_buffer_load_i128(ptr addrspace(1) %out, ptr add
store i128 %load, ptr addrspace(1) %out
ret void
}
-
-declare i1 @llvm.amdgcn.ptr.s.buffer.load.i1(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
-declare i4 @llvm.amdgcn.ptr.s.buffer.load.i4(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
-declare <2 x i1> @llvm.amdgcn.ptr.s.buffer.load.v2i1(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
-declare <6 x i8> @llvm.amdgcn.ptr.s.buffer.load.v6i8(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
-declare i128 @llvm.amdgcn.ptr.s.buffer.load.i128(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
index 10c74b1f147d5..958c6d1b90a9d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -617,10 +617,3 @@ main_body:
store i128 %load, ptr addrspace(1) %out
ret void
}
-
-declare i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32>, i32, i32)
-declare i4 @llvm.amdgcn.s.buffer.load.i4(<4 x i32>, i32, i32)
-declare <2 x i1> @llvm.amdgcn.s.buffer.load.v2i1(<4 x i32>, i32, i32)
-declare <3 x i16> @llvm.amdgcn.s.buffer.load.v3i16(<4 x i32>, i32, i32)
-declare <6 x i8> @llvm.amdgcn.s.buffer.load.v6i8(<4 x i32>, i32, i32)
-declare i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32>, i32, i32)
>From 2018e228fbc3a14fb365e742020677e423696360 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 17 Aug 2026 10:43:52 +0200
Subject: [PATCH 4/6] tests
---
...llvm.amdgcn.s.buffer.load.illegal-types.ll | 24 ++++++++++++-------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
index 958c6d1b90a9d..8206c785a335f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -66,8 +66,9 @@ define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 1
@@ -135,8 +136,9 @@ define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i4:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 15
@@ -204,8 +206,9 @@ define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v2i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 3
@@ -305,8 +308,9 @@ define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(
;
; GFX1250-LABEL: s_buffer_load_v3i16:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -407,8 +411,9 @@ define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v6i8:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -485,8 +490,9 @@ define amdgpu_ps void @s_buffer_load_i128(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_i128:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
@@ -547,8 +553,9 @@ define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %in
;
; GFX1250-LABEL: s_buffer_load_i1_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GFX1250-NEXT: v_mov_b32_e32 v3, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
@@ -604,8 +611,9 @@ define amdgpu_ps void @s_buffer_load_i128_divergent(<4 x i32> inreg %desc, i32 %
;
; GFX1250-LABEL: s_buffer_load_i128_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: v_mov_b32_e32 v5, v2
; GFX1250-NEXT: v_mov_b32_e32 v4, v1
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
>From 345c8f5cb1fa36057e404d4ec4f5b0e9c4e5e4b7 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 2 Sep 2026 12:50:13 +0200
Subject: [PATCH 5/6] Address comments
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 45 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 34 +-
....amdgcn.ptr.s.buffer.load.illegal-types.ll | 942 ++++++++++++++++++
...llvm.amdgcn.s.buffer.load.illegal-types.ll | 717 ++++++++++++-
4 files changed, 1696 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 513a6487b39a2..f758d2851f6d0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -7659,6 +7659,51 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
unsigned Size = Ty.getSizeInBits();
MachineFunction &MF = B.getMF();
bool HasMMO = !MI.memoperands_empty();
+
+ // No S_BUFFER_LOAD exists for these widths. Load a legal i32 carrier and
+ // narrow it down, mirroring SITargetLowering::lowerSBuffer.
+ const LLT NarrowTys[] = {LLT::integer(1), LLT::integer(4),
+ LLT::fixed_vector(2, LLT::integer(1)),
+ LLT::fixed_vector(3, LLT::integer(16)),
+ LLT::fixed_vector(6, LLT::integer(8))};
+ if (is_contained(NarrowTys, Ty)) {
+ LLT CarrierTy = LLT::scalarOrVector(
+ ElementCount::getFixed(divideCeil(Size, 32)), LLT::integer(32));
+ Register Carrier = B.getMRI()->createGenericVirtualRegister(CarrierTy);
+
+ Observer.changingInstr(MI);
+ MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_LOAD));
+ MI.getOperand(0).setReg(Carrier);
+ MI.removeOperand(1);
+ castBufferRsrcArgToV4I32(MI, B, 1);
+ MachinePointerInfo PtrInfo =
+ HasMMO ? (*MI.memoperands_begin())->getPointerInfo()
+ : MachinePointerInfo();
+ MI.setMemRefs(
+ MF, MF.getMachineMemOperand(PtrInfo,
+ MachineMemOperand::MOLoad |
+ MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOInvariant,
+ CarrierTy.getSizeInBytes(),
+ Align(CarrierTy.getSizeInBytes())));
+ Observer.changedInstr(MI);
+
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ if (CarrierTy.isVector()) {
+ // Truncating through a non-dword scalar produces illegal G_UNMERGE_VALUES
+ // later, so reinterpret the carrier as a wider vector of the requested
+ // element type and drop the trailing elements instead.
+ LLT EltTy = Ty.getElementType();
+ LLT WideTy = LLT::fixed_vector(
+ CarrierTy.getSizeInBits() / EltTy.getSizeInBits(), EltTy);
+ B.buildDeleteTrailingVectorElements(OrigDst,
+ B.buildBitcast(WideTy, Carrier));
+ } else {
+ B.buildCast(OrigDst, B.buildTrunc(LLT::integer(Size), Carrier));
+ }
+ return true;
+ }
+
unsigned Opc = 0;
if (Size < 32 && ST.hasScalarSubwordLoads()) {
assert(Size == 8 || Size == 16);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index cec3611b2cf8e..3e98000a116d3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -8407,15 +8407,13 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
case Intrinsic::amdgcn_s_buffer_load: {
- // lowerSBuffer already handles every illegal result type.
+ // The i8 lowering that used to live here duplicated lowerSBuffer, which
+ // now covers the illegal result types marked Custom above.
SDValue Op = SDValue(N, 0);
- SDValue Rsrc = Op.getOperand(1);
- SDValue Offset = Op.getOperand(2);
- SDValue CachePolicy = Op.getOperand(3);
EVT VT = Op.getValueType();
- SDLoc DL(Op);
- Results.push_back(lowerSBuffer(VT, VT, DL, DAG.getEntryNode(), Rsrc,
- Offset, CachePolicy, DAG));
+ Results.push_back(lowerSBuffer(VT, VT, SDLoc(Op), DAG.getEntryNode(),
+ Op.getOperand(1), Op.getOperand(2),
+ Op.getOperand(3), DAG));
return;
}
case Intrinsic::amdgcn_dead: {
@@ -10829,31 +10827,19 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
// No SBUFFER_LOAD pattern exists for these widths; a broader
// "!isTypeLegal(VT)" check would also swallow i8/i16 and 32-bit-element
// vec3 types, which have their own faster lowerings below.
- auto NeedsCarrierLoad = [](EVT VT) {
- return VT == MVT::i1 || VT == MVT::i4 || VT == MVT::v2i1 ||
- VT == MVT::v3i16 || VT == MVT::v6i8 || VT == MVT::i128;
- };
- if (NeedsCarrierLoad(VT) && MemVT == VT) {
+ if (MemVT == VT && (VT == MVT::i1 || VT == MVT::i4 || VT == MVT::v2i1 ||
+ VT == MVT::v3i16 || VT == MVT::v6i8 || VT == MVT::i128)) {
unsigned Bits = VT.getSizeInBits();
EVT CarrierVT = Bits <= 32 ? EVT(MVT::i32)
: EVT::getVectorVT(*DAG.getContext(), MVT::i32,
divideCeil(Bits, 32));
- Align CarrierAlign = DAG.getDataLayout().getABITypeAlign(
- CarrierVT.getTypeForEVT(*DAG.getContext()));
MachineMemOperand *CarrierMMO =
- MF.getMachineMemOperand(MMO->getPointerInfo(), MMO->getFlags(),
- CarrierVT.getStoreSize(), CarrierAlign);
+ MF.getMachineMemOperand(MMO, 0, CarrierVT.getStoreSize());
SDValue CarrierLoad = lowerSBuffer(CarrierVT, CarrierVT, DL, Chain, Rsrc,
Offset, CachePolicy, DAG, CarrierMMO);
- EVT CarrierIntVT =
- EVT::getIntegerVT(*DAG.getContext(), CarrierVT.getSizeInBits());
- SDValue AsInt = DAG.getNode(ISD::BITCAST, DL, CarrierIntVT, CarrierLoad);
EVT NarrowIntVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
- SDValue Narrow = NarrowIntVT == CarrierIntVT
- ? AsInt
- : DAG.getNode(ISD::TRUNCATE, DL, NarrowIntVT, AsInt);
- SDValue Result =
- VT == NarrowIntVT ? Narrow : DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
+ SDValue Result = DAG.getBitcast(
+ VT, DAG.getBitcastedAnyExtOrTrunc(CarrierLoad, DL, NarrowIntVT));
if (HasChainResult)
return DAG.getMergeValues({Result, CarrierLoad.getValue(1)}, DL);
return Result;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
index 5613580e0aa3f..5ca6cf51452d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.illegal-types.ll
@@ -3,6 +3,10 @@
; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX9
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX11
; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GISEL-GFX6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GISEL-GFX9
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GISEL-GFX11
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GISEL-GFX1200
; llvm.amdgcn.ptr.s.buffer.load lowers through INTRINSIC_W_CHAIN, which was
; missing Custom for these result types.
@@ -67,6 +71,67 @@ define amdgpu_kernel void @ptr_s_buffer_load_i1(ptr addrspace(1) %out, ptr addrs
; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i1:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dword s6, s[8:9], 0x8
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_buffer_load_dword s0, s[0:3], s6
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i1:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_buffer_load_dword s0, s[0:3], s6 offset:0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i1:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i1:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
%load = call i1 @llvm.amdgcn.ptr.s.buffer.load.i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
store i1 %load, ptr addrspace(1) %out
ret void
@@ -132,6 +197,67 @@ define amdgpu_kernel void @ptr_s_buffer_load_i4(ptr addrspace(1) %out, ptr addrs
; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i4:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dword s6, s[8:9], 0x8
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_buffer_load_dword s0, s[0:3], s6
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i4:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_buffer_load_dword s0, s[0:3], s6 offset:0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i4:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i4:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
%load = call i4 @llvm.amdgcn.ptr.s.buffer.load.i4(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
store i4 %load, ptr addrspace(1) %out
ret void
@@ -197,6 +323,67 @@ define amdgpu_kernel void @ptr_s_buffer_load_v2i1(ptr addrspace(1) %out, ptr add
; GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1200-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_v2i1:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dword s6, s[8:9], 0x8
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_buffer_load_dword s0, s[0:3], s6
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_v2i1:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_buffer_load_dword s0, s[0:3], s6 offset:0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_v2i1:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_v2i1:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
%load = call <2 x i1> @llvm.amdgcn.ptr.s.buffer.load.v2i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
store <2 x i1> %load, ptr addrspace(1) %out
ret void
@@ -266,6 +453,122 @@ define amdgpu_kernel void @ptr_s_buffer_load_v6i8(ptr addrspace(1) %out, ptr add
; GFX1200-NEXT: global_store_b16 v0, v1, s[2:3] offset:4
; GFX1200-NEXT: global_store_b32 v0, v2, s[2:3]
; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_v6i8:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dword s6, s[8:9], 0x8
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], s6
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX6-NEXT: s_lshr_b32 s8, s0, 24
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:1
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:2
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-GFX6-NEXT: s_lshr_b32 s9, s1, 8
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:3
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:4
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s9
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:5
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_v6i8:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], s6 offset:0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GISEL-GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5]
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GISEL-GFX9-NEXT: s_lshr_b32 s6, s0, 24
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5] offset:1
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5] offset:2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GISEL-GFX9-NEXT: s_lshr_b32 s7, s1, 8
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5] offset:3
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5] offset:4
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GISEL-GFX9-NEXT: global_store_byte v0, v1, s[4:5] offset:5
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_v6i8:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s0
+; GISEL-GFX11-NEXT: s_lshr_b32 s6, s0, 24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v3, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v4, s3
+; GISEL-GFX11-NEXT: s_lshr_b32 s7, s1, 8
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v5, s6
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s1
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v6, s7
+; GISEL-GFX11-NEXT: s_clause 0x5
+; GISEL-GFX11-NEXT: global_store_b8 v0, v1, s[4:5]
+; GISEL-GFX11-NEXT: global_store_b8 v0, v3, s[4:5] offset:1
+; GISEL-GFX11-NEXT: global_store_b8 v0, v4, s[4:5] offset:2
+; GISEL-GFX11-NEXT: global_store_b8 v0, v5, s[4:5] offset:3
+; GISEL-GFX11-NEXT: global_store_b8 v0, v2, s[4:5] offset:4
+; GISEL-GFX11-NEXT: global_store_b8 v0, v6, s[4:5] offset:5
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_v6i8:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GISEL-GFX1200-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX1200-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX1200-NEXT: s_lshr_b32 s6, s0, 24
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s6
+; GISEL-GFX1200-NEXT: s_lshr_b32 s7, s1, 8
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v6, s7
+; GISEL-GFX1200-NEXT: s_clause 0x5
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v1, s[4:5]
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v3, s[4:5] offset:1
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v4, s[4:5] offset:2
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v5, s[4:5] offset:3
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v2, s[4:5] offset:4
+; GISEL-GFX1200-NEXT: global_store_b8 v0, v6, s[4:5] offset:5
+; GISEL-GFX1200-NEXT: s_endpgm
%load = call <6 x i8> @llvm.amdgcn.ptr.s.buffer.load.v6i8(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
store <6 x i8> %load, ptr addrspace(1) %out
ret void
@@ -336,6 +639,645 @@ define amdgpu_kernel void @ptr_s_buffer_load_i128(ptr addrspace(1) %out, ptr add
; GFX1200-NEXT: v_mov_b32_e32 v2, s2
; GFX1200-NEXT: global_store_b128 v4, v[0:3], s[4:5]
; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i128:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dword s6, s[8:9], 0x8
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], s6
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v3, s3
+; GISEL-GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i128:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s6, s[8:9], 0x20
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], s6 offset:0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GISEL-GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i128:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GISEL-GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i128:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: s_load_b32 s6, s[4:5], 0x20
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
+ %load = call i128 @llvm.amdgcn.ptr.s.buffer.load.i128(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i128 %load, ptr addrspace(1) %out
+ ret void
+}
+
+; A divergent offset takes the MUBUF lowering instead of SMEM.
+define amdgpu_kernel void @ptr_s_buffer_load_i1_divergent(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: ptr_s_buffer_load_i1_divergent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i1_divergent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i1_divergent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i1_divergent:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i1_divergent:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX6-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i1_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i1_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i1_divergent:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
+ %offset = call i32 @llvm.amdgcn.workitem.id.x()
+ %load = call i1 @llvm.amdgcn.ptr.s.buffer.load.i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i1 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_i4_divergent(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: ptr_s_buffer_load_i4_divergent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i4_divergent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i4_divergent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i4_divergent:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i4_divergent:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX6-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX6-NEXT: v_and_b32_e32 v0, 15, v0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i4_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i4_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 15, v0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i4_divergent:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 15, v0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
+ %offset = call i32 @llvm.amdgcn.workitem.id.x()
+ %load = call i4 @llvm.amdgcn.ptr.s.buffer.load.i4(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store i4 %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_v2i1_divergent(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.l, v0.l, 3
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: v_and_b16 v0.l, v0.l, 3
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX6-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX6-NEXT: v_and_b32_e32 v0, 3, v0
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 3, v0
+; GISEL-GFX9-NEXT: global_store_byte v1, v0, s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 3, v0
+; GISEL-GFX11-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_v2i1_divergent:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 3, v0
+; GISEL-GFX1200-NEXT: global_store_b8 v1, v0, s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
+ %offset = call i32 @llvm.amdgcn.workitem.id.x()
+ %load = call <2 x i1> @llvm.amdgcn.ptr.s.buffer.load.v2i1(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store <2 x i1> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_v6i8_divergent(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[0:3], 0 offen
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT: buffer_store_short v1, off, s[0:3], 0 offset:4
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_load_dwordx2 v[0:1], v0, s[0:3], 0 offen
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_store_short v2, v1, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1] offset:4
+; GFX11-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1200-NEXT: v_mov_b32_e32 v2, 0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: s_clause 0x1
+; GFX1200-NEXT: global_store_b16 v2, v1, s[0:1] offset:4
+; GFX1200-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[0:3], 0 offen
+; GISEL-GFX6-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GISEL-GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GISEL-GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GISEL-GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GISEL-GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GISEL-GFX6-NEXT: buffer_store_byte v1, off, s[4:7], 0 offset:4
+; GISEL-GFX6-NEXT: buffer_store_byte v2, off, s[4:7], 0 offset:1
+; GISEL-GFX6-NEXT: buffer_store_byte v3, off, s[4:7], 0 offset:2
+; GISEL-GFX6-NEXT: buffer_store_byte v4, off, s[4:7], 0 offset:3
+; GISEL-GFX6-NEXT: buffer_store_byte v5, off, s[4:7], 0 offset:5
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: buffer_load_dwordx2 v[0:1], v0, s[0:3], 0 offen
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GISEL-GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GISEL-GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GISEL-GFX9-NEXT: global_store_byte v2, v0, s[4:5]
+; GISEL-GFX9-NEXT: global_store_byte_d16_hi v2, v0, s[4:5] offset:2
+; GISEL-GFX9-NEXT: global_store_byte v2, v1, s[4:5] offset:4
+; GISEL-GFX9-NEXT: global_store_byte v2, v3, s[4:5] offset:1
+; GISEL-GFX9-NEXT: global_store_byte v2, v4, s[4:5] offset:3
+; GISEL-GFX9-NEXT: global_store_byte v2, v5, s[4:5] offset:5
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GISEL-GFX11-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GISEL-GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GISEL-GFX11-NEXT: s_clause 0x5
+; GISEL-GFX11-NEXT: global_store_b8 v2, v0, s[4:5]
+; GISEL-GFX11-NEXT: global_store_b8 v2, v3, s[4:5] offset:1
+; GISEL-GFX11-NEXT: global_store_d16_hi_b8 v2, v0, s[4:5] offset:2
+; GISEL-GFX11-NEXT: global_store_b8 v2, v4, s[4:5] offset:3
+; GISEL-GFX11-NEXT: global_store_b8 v2, v1, s[4:5] offset:4
+; GISEL-GFX11-NEXT: global_store_b8 v2, v5, s[4:5] offset:5
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_v6i8_divergent:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GISEL-GFX1200-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GISEL-GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GISEL-GFX1200-NEXT: s_clause 0x5
+; GISEL-GFX1200-NEXT: global_store_b8 v2, v0, s[4:5]
+; GISEL-GFX1200-NEXT: global_store_b8 v2, v3, s[4:5] offset:1
+; GISEL-GFX1200-NEXT: global_store_d16_hi_b8 v2, v0, s[4:5] offset:2
+; GISEL-GFX1200-NEXT: global_store_b8 v2, v4, s[4:5] offset:3
+; GISEL-GFX1200-NEXT: global_store_b8 v2, v1, s[4:5] offset:4
+; GISEL-GFX1200-NEXT: global_store_b8 v2, v5, s[4:5] offset:5
+; GISEL-GFX1200-NEXT: s_endpgm
+ %offset = call i32 @llvm.amdgcn.workitem.id.x()
+ %load = call <6 x i8> @llvm.amdgcn.ptr.s.buffer.load.v6i8(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
+ store <6 x i8> %load, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @ptr_s_buffer_load_i128_divergent(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
+; GFX6-LABEL: ptr_s_buffer_load_i128_divergent:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX9-LABEL: ptr_s_buffer_load_i128_divergent:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-LABEL: ptr_s_buffer_load_i128_divergent:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX1200-LABEL: ptr_s_buffer_load_i128_divergent:
+; GFX1200: ; %bb.0:
+; GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX1200-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1200-NEXT: v_mov_b32_e32 v4, 0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1200-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_kmcnt 0x0
+; GFX1200-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: ptr_s_buffer_load_i128_divergent:
+; GISEL-GFX6: ; %bb.0:
+; GISEL-GFX6-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x4
+; GISEL-GFX6-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GISEL-GFX6-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: ptr_s_buffer_load_i128_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x10
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: ptr_s_buffer_load_i128_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: ptr_s_buffer_load_i128_divergent:
+; GISEL-GFX1200: ; %bb.0:
+; GISEL-GFX1200-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-GFX1200-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: global_store_b128 v4, v[0:3], s[4:5]
+; GISEL-GFX1200-NEXT: s_endpgm
+ %offset = call i32 @llvm.amdgcn.workitem.id.x()
%load = call i128 @llvm.amdgcn.ptr.s.buffer.load.i128(ptr addrspace(8) %rsrc, i32 %offset, i32 0)
store i128 %load, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
index 8206c785a335f..f612f158dbdc3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -7,10 +7,17 @@
; RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GFX11
; RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GFX1200
; RUN: llc < %s -mtriple=amdgpu12.50 -amdgpu-enable-vopd=0 -mattr=-wait-xcnt | FileCheck %s -check-prefixes=GFX1250
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu6.00 | FileCheck %s -check-prefixes=GISEL-GFX67,GISEL-GFX6
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu7.01 | FileCheck %s -check-prefixes=GISEL-GFX67,GISEL-GFX7
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu8.02 | FileCheck %s -check-prefixes=GISEL-GFX8
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu9.00 | FileCheck %s -check-prefixes=GISEL-GFX910,GISEL-GFX9
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu10.10 | FileCheck %s -check-prefixes=GISEL-GFX910,GISEL-GFX10
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GISEL-GFX11
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-vopd=0 | FileCheck %s -check-prefixes=GISEL-GFX1200
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-vopd=0 -mattr=-wait-xcnt | FileCheck %s -check-prefixes=GISEL-GFX1250
; Result types with no direct SBUFFER_LOAD pattern used to ICE in the type
-; legalizer. GlobalISel is not covered: it hits a separate, pre-existing
-; assertion in AMDGPULegalizerInfo::legalizeSBufferLoad for these widths.
+; legalizer and in AMDGPULegalizerInfo::legalizeSBufferLoad.
define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1) %out) {
; GFX67-LABEL: s_buffer_load_i1:
@@ -66,9 +73,8 @@ define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 1
@@ -76,6 +82,68 @@ define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1)
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_i1:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX67-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_i1:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: flat_store_byte v[0:1], v2
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_i1:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX910-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_i1:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_i1:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_i1:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 1
+; GISEL-GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32> %desc, i32 0, i32 0)
store i1 %load, ptr addrspace(1) %out
@@ -136,9 +204,8 @@ define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i4:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 15
@@ -146,6 +213,68 @@ define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1)
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_i4:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX67-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_i4:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: flat_store_byte v[0:1], v2
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_i4:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX910-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_i4:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_i4:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_i4:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 15
+; GISEL-GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call i4 @llvm.amdgcn.s.buffer.load.i4(<4 x i32> %desc, i32 0, i32 0)
store i4 %load, ptr addrspace(1) %out
@@ -206,9 +335,8 @@ define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v2i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 3
@@ -216,6 +344,68 @@ define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: global_store_b8 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX67-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: flat_store_byte v[0:1], v2
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+; GISEL-GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX910-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX910-NEXT: global_store_byte v[0:1], v2, off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_v2i1:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 3
+; GISEL-GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call <2 x i1> @llvm.amdgcn.s.buffer.load.v2i1(<4 x i32> %desc, i32 0, i32 0)
store <2 x i1> %load, ptr addrspace(1) %out
@@ -308,9 +498,8 @@ define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(
;
; GFX1250-LABEL: s_buffer_load_v3i16:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -319,6 +508,127 @@ define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(
; GFX1250-NEXT: global_store_b16 v[0:1], v2, off offset:4
; GFX1250-NEXT: global_store_b32 v[0:1], v3, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX6: ; %bb.0: ; %main_body
+; GISEL-GFX6-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX6-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_lshr_b32 s6, s4, 16
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-GFX6-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-GFX6-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:2
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX6-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX7-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX7: ; %bb.0: ; %main_body
+; GISEL-GFX7-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GISEL-GFX7-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX7-NEXT: s_lshr_b32 s6, s4, 16
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-GFX7-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-GFX7-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:2
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX7-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GISEL-GFX7-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: flat_store_short v[0:1], v2
+; GISEL-GFX8-NEXT: v_add_u32_e32 v2, vcc, 2, v0
+; GISEL-GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v0
+; GISEL-GFX8-NEXT: flat_store_short v[2:3], v4
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GISEL-GFX8-NEXT: flat_store_short v[0:1], v2
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX9: ; %bb.0: ; %main_body
+; GISEL-GFX9-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX9-NEXT: global_store_short v[0:1], v2, off
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-GFX9-NEXT: global_store_short v[0:1], v2, off offset:2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GISEL-GFX9-NEXT: global_store_short v[0:1], v2, off offset:4
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX10-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX10: ; %bb.0: ; %main_body
+; GISEL-GFX10-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v4, s1
+; GISEL-GFX10-NEXT: global_store_short v[0:1], v2, off
+; GISEL-GFX10-NEXT: global_store_short v[0:1], v3, off offset:2
+; GISEL-GFX10-NEXT: global_store_short v[0:1], v4, off offset:4
+; GISEL-GFX10-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v2.h, s2
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GISEL-GFX11-NEXT: s_clause 0x2
+; GISEL-GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GISEL-GFX11-NEXT: global_store_d16_hi_b16 v[0:1], v2, off offset:2
+; GISEL-GFX11-NEXT: global_store_b16 v[0:1], v3, off offset:4
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s0
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.h, s2
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s1
+; GISEL-GFX1200-NEXT: s_clause 0x2
+; GISEL-GFX1200-NEXT: global_store_b16 v[0:1], v2, off
+; GISEL-GFX1200-NEXT: global_store_d16_hi_b16 v[0:1], v2, off offset:2
+; GISEL-GFX1200-NEXT: global_store_b16 v[0:1], v3, off offset:4
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_v3i16:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: s_lshr_b32 s2, s0, 16
+; GISEL-GFX1250-NEXT: v_mov_b16_e32 v2.l, s0
+; GISEL-GFX1250-NEXT: v_mov_b16_e32 v2.h, s2
+; GISEL-GFX1250-NEXT: v_mov_b16_e32 v3.l, s1
+; GISEL-GFX1250-NEXT: s_clause 0x2
+; GISEL-GFX1250-NEXT: global_store_b16 v[0:1], v2, off
+; GISEL-GFX1250-NEXT: global_store_d16_hi_b16 v[0:1], v2, off offset:2
+; GISEL-GFX1250-NEXT: global_store_b16 v[0:1], v3, off offset:4
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call <3 x i16> @llvm.amdgcn.s.buffer.load.v3i16(<4 x i32> %desc, i32 0, i32 0)
store <3 x i16> %load, ptr addrspace(1) %out
@@ -411,9 +721,8 @@ define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v6i8:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -422,6 +731,208 @@ define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1
; GFX1250-NEXT: global_store_b16 v[0:1], v2, off offset:4
; GFX1250-NEXT: global_store_b32 v[0:1], v3, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX6-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX6: ; %bb.0: ; %main_body
+; GISEL-GFX6-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GISEL-GFX6-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX6-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX6-NEXT: s_lshr_b32 s6, s4, 8
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-GFX6-NEXT: s_lshr_b32 s8, s4, 24
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:1
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:2
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GISEL-GFX6-NEXT: s_lshr_b32 s9, s5, 8
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:3
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GISEL-GFX6-NEXT: s_waitcnt expcnt(0)
+; GISEL-GFX6-NEXT: v_mov_b32_e32 v2, s9
+; GISEL-GFX6-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:5
+; GISEL-GFX6-NEXT: s_endpgm
+;
+; GISEL-GFX7-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX7: ; %bb.0: ; %main_body
+; GISEL-GFX7-NEXT: s_buffer_load_dwordx2 s[4:5], s[0:3], 0x0
+; GISEL-GFX7-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX7-NEXT: s_lshr_b32 s6, s4, 8
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-GFX7-NEXT: s_lshr_b32 s7, s4, 16
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-GFX7-NEXT: s_lshr_b32 s8, s4, 24
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:1
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:2
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GISEL-GFX7-NEXT: s_lshr_b32 s9, s5, 8
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:3
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GISEL-GFX7-NEXT: v_mov_b32_e32 v2, s9
+; GISEL-GFX7-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:5
+; GISEL-GFX7-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX8-NEXT: flat_store_byte v[0:1], v2
+; GISEL-GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v0
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX8-NEXT: flat_store_byte v[2:3], v4
+; GISEL-GFX8-NEXT: v_add_u32_e32 v2, vcc, 2, v0
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GISEL-GFX8-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX8-NEXT: flat_store_byte v[2:3], v4
+; GISEL-GFX8-NEXT: v_add_u32_e32 v2, vcc, 3, v0
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-GFX8-NEXT: flat_store_byte v[2:3], v4
+; GISEL-GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s1
+; GISEL-GFX8-NEXT: v_add_u32_e32 v0, vcc, 5, v0
+; GISEL-GFX8-NEXT: flat_store_byte v[2:3], v4
+; GISEL-GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX8-NEXT: flat_store_byte v[0:1], v2
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX9: ; %bb.0: ; %main_body
+; GISEL-GFX9-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GISEL-GFX9-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off offset:1
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off offset:2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-GFX9-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off offset:3
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off offset:4
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GISEL-GFX9-NEXT: global_store_byte v[0:1], v2, off offset:5
+; GISEL-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX10-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX10: ; %bb.0: ; %main_body
+; GISEL-GFX10-NEXT: s_buffer_load_dwordx2 s[0:1], s[0:3], 0x0
+; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX10-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX10-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v2, off
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v3, off offset:4
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v4, off offset:1
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v5, off offset:2
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v6, off offset:3
+; GISEL-GFX10-NEXT: global_store_byte v[0:1], v7, off offset:5
+; GISEL-GFX10-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX11-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX11-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v6, s4
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v7, s5
+; GISEL-GFX11-NEXT: s_clause 0x5
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v4, off offset:1
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v5, off offset:2
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v6, off offset:3
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v3, off offset:4
+; GISEL-GFX11-NEXT: global_store_b8 v[0:1], v7, off offset:5
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX1200-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1200-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX1200-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v6, s4
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v7, s5
+; GISEL-GFX1200-NEXT: s_clause 0x5
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v4, off offset:1
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v5, off offset:2
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v6, off offset:3
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v3, off offset:4
+; GISEL-GFX1200-NEXT: global_store_b8 v[0:1], v7, off offset:5
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_v6i8:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: s_lshr_b32 s2, s0, 8
+; GISEL-GFX1250-NEXT: s_lshr_b32 s3, s0, 16
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1250-NEXT: s_lshr_b32 s4, s0, 24
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX1250-NEXT: s_lshr_b32 s5, s1, 8
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v6, s4
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v7, s5
+; GISEL-GFX1250-NEXT: s_clause 0x5
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v2, off
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v4, off offset:1
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v5, off offset:2
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v6, off offset:3
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v3, off offset:4
+; GISEL-GFX1250-NEXT: global_store_b8 v[0:1], v7, off offset:5
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call <6 x i8> @llvm.amdgcn.s.buffer.load.v6i8(<4 x i32> %desc, i32 0, i32 0)
store <6 x i8> %load, ptr addrspace(1) %out
@@ -490,15 +1001,83 @@ define amdgpu_ps void @s_buffer_load_i128(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_i128:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_i128:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GISEL-GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX67-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: buffer_store_dwordx4 v[2:5], v[0:1], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_i128:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GISEL-GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_i128:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x0
+; GISEL-GFX910-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX910-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX910-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_i128:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_i128:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0
+; GISEL-GFX1200-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v5, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v4, s2
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-GFX1200-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_i128:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
+; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GISEL-GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GISEL-GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32> %desc, i32 0, i32 0)
store i128 %load, ptr addrspace(1) %out
@@ -553,9 +1132,8 @@ define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %in
;
; GFX1250-LABEL: s_buffer_load_i1_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GFX1250-NEXT: v_mov_b32_e32 v3, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
@@ -563,6 +1141,61 @@ define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %in
; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-NEXT: global_store_b8 v[2:3], v0, off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX67-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX67-NEXT: buffer_store_byte v0, v[1:2], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX8-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX8-NEXT: flat_store_byte v[1:2], v0
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GISEL-GFX910-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX910-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX910-NEXT: global_store_byte v[1:2], v0, off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX11-NEXT: global_store_b8 v[1:2], v0, off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX1200-NEXT: global_store_b8 v[1:2], v0, off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_i1_divergent:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v4, v1
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v5, v2
+; GISEL-GFX1250-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-GFX1250-NEXT: global_store_b8 v[4:5], v0, off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call i1 @llvm.amdgcn.s.buffer.load.i1(<4 x i32> %desc, i32 %index, i32 0)
store i1 %load, ptr addrspace(1) %out
@@ -611,15 +1244,63 @@ define amdgpu_ps void @s_buffer_load_i128_divergent(<4 x i32> inreg %desc, i32 %
;
; GFX1250-LABEL: s_buffer_load_i128_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: v_mov_b32_e32 v5, v2
; GFX1250-NEXT: v_mov_b32_e32 v4, v1
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX1250-NEXT: s_endpgm
+;
+; GISEL-GFX67-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX67: ; %bb.0: ; %main_body
+; GISEL-GFX67-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GISEL-GFX67-NEXT: s_mov_b32 s2, 0
+; GISEL-GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GISEL-GFX67-NEXT: s_mov_b64 s[0:1], 0
+; GISEL-GFX67-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX67-NEXT: buffer_store_dwordx4 v[3:6], v[1:2], s[0:3], 0 addr64
+; GISEL-GFX67-NEXT: s_endpgm
+;
+; GISEL-GFX8-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX8: ; %bb.0: ; %main_body
+; GISEL-GFX8-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GISEL-GFX8-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX8-NEXT: flat_store_dwordx4 v[1:2], v[3:6]
+; GISEL-GFX8-NEXT: s_endpgm
+;
+; GISEL-GFX910-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX910: ; %bb.0: ; %main_body
+; GISEL-GFX910-NEXT: buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; GISEL-GFX910-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX910-NEXT: global_store_dwordx4 v[1:2], v[3:6], off
+; GISEL-GFX910-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX11: ; %bb.0: ; %main_body
+; GISEL-GFX11-NEXT: buffer_load_b128 v[3:6], v0, s[0:3], 0 offen
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-NEXT: global_store_b128 v[1:2], v[3:6], off
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX1200-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX1200: ; %bb.0: ; %main_body
+; GISEL-GFX1200-NEXT: buffer_load_b128 v[3:6], v0, s[0:3], null offen
+; GISEL-GFX1200-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1200-NEXT: global_store_b128 v[1:2], v[3:6], off
+; GISEL-GFX1200-NEXT: s_endpgm
+;
+; GISEL-GFX1250-LABEL: s_buffer_load_i128_divergent:
+; GISEL-GFX1250: ; %bb.0: ; %main_body
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v4, v1
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v5, v2
+; GISEL-GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
+; GISEL-GFX1250-NEXT: s_wait_loadcnt 0x0
+; GISEL-GFX1250-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GISEL-GFX1250-NEXT: s_endpgm
main_body:
%load = call i128 @llvm.amdgcn.s.buffer.load.i128(<4 x i32> %desc, i32 %index, i32 0)
store i128 %load, ptr addrspace(1) %out
>From 650a8ca5607a6b9f77609de50e3eb69750fecb98 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 2 Sep 2026 17:23:56 +0200
Subject: [PATCH 6/6] fix
---
...llvm.amdgcn.s.buffer.load.illegal-types.ll | 48 ++++++++++++-------
1 file changed, 32 insertions(+), 16 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
index f612f158dbdc3..88c709eddb8ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.illegal-types.ll
@@ -73,8 +73,9 @@ define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 1
@@ -135,8 +136,9 @@ define amdgpu_ps void @s_buffer_load_i1(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GISEL-GFX1250-LABEL: s_buffer_load_i1:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 1
@@ -204,8 +206,9 @@ define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GFX1250-LABEL: s_buffer_load_i4:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 15
@@ -266,8 +269,9 @@ define amdgpu_ps void @s_buffer_load_i4(<4 x i32> inreg %desc, ptr addrspace(1)
;
; GISEL-GFX1250-LABEL: s_buffer_load_i4:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 15
@@ -335,8 +339,9 @@ define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v2i1:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s0, s0, 3
@@ -397,8 +402,9 @@ define amdgpu_ps void @s_buffer_load_v2i1(<4 x i32> inreg %desc, ptr addrspace(1
;
; GISEL-GFX1250-LABEL: s_buffer_load_v2i1:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: s_and_b32 s0, s0, 3
@@ -498,8 +504,9 @@ define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(
;
; GFX1250-LABEL: s_buffer_load_v3i16:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -616,8 +623,9 @@ define amdgpu_ps void @s_buffer_load_v3i16(<4 x i32> inreg %desc, ptr addrspace(
;
; GISEL-GFX1250-LABEL: s_buffer_load_v3i16:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: s_lshr_b32 s2, s0, 16
@@ -721,8 +729,9 @@ define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_v6i8:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
@@ -911,8 +920,9 @@ define amdgpu_ps void @s_buffer_load_v6i8(<4 x i32> inreg %desc, ptr addrspace(1
;
; GISEL-GFX1250-LABEL: s_buffer_load_v6i8:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: s_lshr_b32 s2, s0, 8
@@ -1001,8 +1011,9 @@ define amdgpu_ps void @s_buffer_load_i128(<4 x i32> inreg %desc, ptr addrspace(1
;
; GFX1250-LABEL: s_buffer_load_i128:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
@@ -1070,8 +1081,9 @@ define amdgpu_ps void @s_buffer_load_i128(<4 x i32> inreg %desc, ptr addrspace(1
;
; GISEL-GFX1250-LABEL: s_buffer_load_i128:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x0 nv
; GISEL-GFX1250-NEXT: s_wait_kmcnt 0x0
; GISEL-GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
@@ -1132,8 +1144,9 @@ define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %in
;
; GFX1250-LABEL: s_buffer_load_i1_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GFX1250-NEXT: v_mov_b32_e32 v3, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
@@ -1187,8 +1200,9 @@ define amdgpu_ps void @s_buffer_load_i1_divergent(<4 x i32> inreg %desc, i32 %in
;
; GISEL-GFX1250-LABEL: s_buffer_load_i1_divergent:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GISEL-GFX1250-NEXT: v_mov_b32_e32 v4, v1
; GISEL-GFX1250-NEXT: v_mov_b32_e32 v5, v2
@@ -1244,8 +1258,9 @@ define amdgpu_ps void @s_buffer_load_i128_divergent(<4 x i32> inreg %desc, i32 %
;
; GFX1250-LABEL: s_buffer_load_i128_divergent:
; GFX1250: ; %bb.0: ; %main_body
-; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: v_mov_b32_e32 v5, v2
; GFX1250-NEXT: v_mov_b32_e32 v4, v1
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
@@ -1293,8 +1308,9 @@ define amdgpu_ps void @s_buffer_load_i128_divergent(<4 x i32> inreg %desc, i32 %
;
; GISEL-GFX1250-LABEL: s_buffer_load_i128_divergent:
; GISEL-GFX1250: ; %bb.0: ; %main_body
-; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-GFX1250-NEXT: v_mov_b32_e32 v4, v1
; GISEL-GFX1250-NEXT: v_mov_b32_e32 v5, v2
; GISEL-GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
More information about the llvm-commits
mailing list