[llvm-branch-commits] [llvm] [AMDGPU] Lower loads and stores for address space 13 (PR #209541)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 30 09:55:52 PDT 2026
================
@@ -0,0 +1,300 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of the VGPR "as memory" address space (13) on a
+; movrel-capable subtarget (gfx12). A load/store of a uniform (SGPR) pointer
+; lowers to an M0-relative move (v_movrels_b32 / v_movreld_b32) over the wave's
+; vector registers, with the dword index (pointer >> 2) placed in M0.
+
+define i32 @load_i32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define i64 @load_i64(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %p
+ %y = add i64 %x, 1
+ ret i64 %y
+}
+
+define <2 x float> @load_v2f32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_v2f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_add_f32 v0, 0x42280000, v0 :: v_dual_add_f32 v1, 0x42280000, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x float>, ptr addrspace(13) %p
+ %y = fadd <2 x float> %x, <float 42.0, float 42.0>
+ ret <2 x float> %y
+}
+
+define <3 x float> @load_v3f32(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_v3f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 64
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_v3f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 64
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %p.2 = getelementptr i32, ptr addrspace(13) %p, i32 16
+ %x = load <3 x float>, ptr addrspace(13) %p
+ %y = load <3 x float>, ptr addrspace(13) %p.2
+ %z = fadd <3 x float> %x, %y
+ ret <3 x float> %z
+}
+
+define void @store_i32(ptr addrspace(13) inreg %p, i32 %x) {
+; GFX12-LABEL: store_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i64(ptr addrspace(13) inreg %p, i64 %x, i64 %y) {
+; GFX12-LABEL: store_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %z = add i64 %x, %y
+ store i64 %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v3i32(ptr addrspace(13) inreg %p, <3 x i32> %x, <3 x i32> %y) {
+; GFX12-SDAG-LABEL: store_v3i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v3i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = add <3 x i32> %x, %y
+ store <3 x i32> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v8f16(ptr addrspace(13) inreg %p, <8 x half> %x, <8 x half> %y) {
+; GFX12-SDAG-LABEL: store_v8f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-SDAG-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-SDAG-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-SDAG-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v8f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-GISEL-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-GISEL-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = fadd <8 x half> %x, %y
+ store <8 x half> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @copy_i32(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %src
+ store i32 %x, ptr addrspace(13) %dst
+ ret void
+}
+
+define void @copy_v2i32_unaligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_v2i32_unaligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x i32>, ptr addrspace(13) %src, align 4
+ store <2 x i32> %x, ptr addrspace(13) %dst, align 4
+ ret void
+}
+
+define void @copy_i64_aligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i64_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %src, align 8
+ store i64 %x, ptr addrspace(13) %dst, align 8
+ ret void
+}
----------------
doru1004 wrote:
Done
https://github.com/llvm/llvm-project/pull/209541
More information about the llvm-branch-commits
mailing list