[llvm] [AMDGPU] Restrict SMRD load patterns to SMEM-accessible address spaces (PR #227423)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 12:01:21 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: alexioslyrakis-amd
<details>
<summary>Changes</summary>
The SMRD load PatFrags (smrd_load, smrd_*extload*) had no address space
restriction and relied on isUniformLoad() to reject ineligible loads.
isUniformLoad() accepts any uniform load with an invariant memory operand,
so a uniform !invariant.load from LDS or scratch was selected as a scalar
load. The 32-bit address was then zero-extended by Expand32BitAddress() (the
path meant for the 32-bit constant address space) and used as a global
address:
```
%v = load i32, ptr addrspace(3) %p, align 4, !invariant.load !0
s_lshl2_add_u32 s0, s0, s1
s_mov_b32 s1, 0
s_load_dword s0, s[0:1], 0x0 ; reads global memory at the LDS offset
```
Restrict SMRDLoadPat to the flat, global, constant and 32-bit constant
address spaces (LoadAddress_flat), so the patterns never match loads from
LDS, GDS or scratch. This matches the GlobalISel register bank selection,
which already excludes LOCAL/REGION/PRIVATE for G_LOAD before calling
isScalarLoadLegal().
---
Full diff: https://github.com/llvm/llvm-project/pull/227423.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SMInstructions.td (+5)
- (added) llvm/test/CodeGen/AMDGPU/smem-invariant-load-addrspace.ll (+236)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 19aeafe9b30cc..dcc308ff563e1 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -829,6 +829,11 @@ def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
//===----------------------------------------------------------------------===//
class SMRDLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{ return isUniformLoad(N);}]> {
+ // Scalar loads can only access memory through a 64-bit address, so never
+ // consider loads from LDS, GDS or scratch.
+ let IsLoad = 1;
+ let AddressSpaces = LoadAddress_flat.AddrSpaces;
+
let GISelPredicateCode = [{
if (!MI.hasOneMemOperand())
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/smem-invariant-load-addrspace.ll b/llvm/test/CodeGen/AMDGPU/smem-invariant-load-addrspace.ll
new file mode 100644
index 0000000000000..24664e805a3fe
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/smem-invariant-load-addrspace.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+
+; A uniform !invariant.load may only be selected as a scalar (SMEM) load if SMEM
+; can reach its address space. LDS and scratch are not accessible through SMEM;
+; selecting s_load_dword for them reads global memory at the LDS/scratch offset.
+
+define amdgpu_kernel void @uniform_invariant_lds_load(i32 %i, ptr addrspace(3) %q, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_lds_load:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_lshl2_add_u32 s0, s0, s1
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ds_read_b32 v1, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v0, v1, s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_lds_load:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GISEL-NEXT: s_add_u32 s0, s1, s0
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GISEL-NEXT: s_endpgm
+ %p = getelementptr float, ptr addrspace(3) %q, i32 %i
+ %v = load float, ptr addrspace(3) %p, align 4, !invariant.load !0
+ store float %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @uniform_invariant_lds_load_v4i32(i32 %i, ptr addrspace(3) %q, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_lds_load_v4i32:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_lshl4_add_u32 s0, s0, s1
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ds_read_b128 v[2:5], v0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_lds_load_v4i32:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_lshl_b32 s0, s0, 4
+; GISEL-NEXT: s_add_u32 s0, s1, s0
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: ds_read_b128 v[0:3], v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
+; GISEL-NEXT: s_endpgm
+ %p = getelementptr <4 x i32>, ptr addrspace(3) %q, i32 %i
+ %v = load <4 x i32>, ptr addrspace(3) %p, align 16, !invariant.load !0
+ store <4 x i32> %v, ptr addrspace(1) %out, align 16
+ ret void
+}
+
+define amdgpu_kernel void @uniform_invariant_private_load(ptr addrspace(5) %q, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_private_load:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: scratch_load_dword v1, off, s2
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_private_load:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dword s2, s[4:5], 0x0
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: scratch_load_dword v0, off, s2
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
+ %v = load i32, ptr addrspace(5) %q, align 4, !invariant.load !0
+ store i32 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; Fill LDS, synchronize, then every lane reads the same element.
+define amdgpu_kernel void @uniform_invariant_lds_load_after_barrier(i32 %idx, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_lds_load_after_barrier:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-NEXT: v_lshlrev_b32_e32 v1, 2, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v2, s1, v1
+; SDAG-NEXT: s_lshl2_add_u32 s0, s0, s1
+; SDAG-NEXT: ds_write_b32 v2, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_barrier
+; SDAG-NEXT: ds_read_b32 v0, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v1, v0, s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_lds_load_after_barrier:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 2, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GISEL-NEXT: v_add_u32_e32 v2, s1, v1
+; GISEL-NEXT: s_add_u32 s0, s1, s0
+; GISEL-NEXT: ds_write_b32 v2, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_barrier
+; GISEL-NEXT: ds_read_b32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GISEL-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %w = getelementptr inbounds i32, ptr addrspace(3) %lds, i32 %tid
+ store i32 %tid, ptr addrspace(3) %w, align 4
+ fence syncscope("workgroup") release
+ call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ %p = getelementptr inbounds i32, ptr addrspace(3) %lds, i32 %idx
+ %v = load i32, ptr addrspace(3) %p, align 4, !invariant.load !0
+ %o = getelementptr inbounds i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %v, ptr addrspace(1) %o, align 4
+ ret void
+}
+
+; Uniform invariant loads from SMEM-accessible address spaces are still scalar.
+define amdgpu_kernel void @uniform_invariant_global_load(ptr addrspace(1) %g, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_global_load:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_load_dword s0, s[0:1], 0x0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-NEXT: global_store_dword v0, v1, s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_global_load:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_load_dword s0, s[0:1], 0x0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GISEL-NEXT: s_endpgm
+ %v = load i32, ptr addrspace(1) %g, align 4, !invariant.load !0
+ store i32 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @uniform_invariant_constant_load(ptr addrspace(4) %c, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_constant_load:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_load_dword s0, s[0:1], 0x0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-NEXT: global_store_dword v0, v1, s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_constant_load:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: s_load_dword s0, s[0:1], 0x0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GISEL-NEXT: s_endpgm
+ %v = load i32, ptr addrspace(4) %c, align 4, !invariant.load !0
+ store i32 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @uniform_invariant_flat_load(ptr %f, ptr addrspace(1) %out) {
+; SDAG-LABEL: uniform_invariant_flat_load:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: s_load_dword s0, s[0:1], 0x0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-NEXT: global_store_dword v0, v1, s[2:3]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: uniform_invariant_flat_load:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GISEL-NEXT: s_endpgm
+ %v = load i32, ptr %f, align 4, !invariant.load !0
+ store i32 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare void @llvm.amdgcn.s.barrier()
+
+!0 = !{}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
``````````
</details>
https://github.com/llvm/llvm-project/pull/227423
More information about the llvm-commits
mailing list