[llvm] [AMDGPU] Handle amdgpu.last.use metadata (PR #83816)
Mirko BrkuĊĦanin via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 4 02:27:05 PST 2024
https://github.com/mbrkusanin updated https://github.com/llvm/llvm-project/pull/83816
>From 47734172af154d07d6b3a13bfcb3901b78b51462 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 1 Mar 2024 18:25:40 +0100
Subject: [PATCH 1/4] Add nontemporal+volatile tests
---
.../memory-legalizer-flat-nontemporal.ll | 165 ++++++++++++++
.../memory-legalizer-global-nontemporal.ll | 158 ++++++++++++++
.../memory-legalizer-local-nontemporal.ll | 179 +++++++++++++++
.../memory-legalizer-private-nontemporal.ll | 203 ++++++++++++++++++
4 files changed, 705 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 9f0556023aa24a..35a9ab72b04362 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -684,5 +684,170 @@ entry:
ret void
}
+define amdgpu_kernel void @flat_nontemporal_volatile_load(
+; GFX7-LABEL: flat_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: flat_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
+; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
+; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: flat_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: flat_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-CU-NEXT: s_endpgm
+ ptr %in, ptr %out) {
+entry:
+ %val = load volatile i32, ptr %in, align 4, !nontemporal !0
+ store i32 %val, ptr %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index b2751326fa784b..672f803a12235d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -674,5 +674,163 @@ entry:
ret void
}
+define amdgpu_kernel void @global_nontemporal_volatile_load(
+; GFX6-LABEL: global_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0x100f000
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: buffer_load_dword v0, off, s[4:7], 0 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: s_mov_b32 s4, s2
+; GFX6-NEXT: s_mov_b32 s5, s3
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s1
+; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s2
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s3
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: global_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[0:1] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: global_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: global_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: global_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(1) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(1) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 0eb2f7b1d694ec..16ede2665400ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -717,5 +717,184 @@ entry:
ret void
}
+define amdgpu_kernel void @local_nontemporal_volatile_load(
+; GFX6-LABEL: local_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT: s_mov_b32 m0, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: ds_read_b32 v2, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_clause 0x1
+; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: ds_read_b32 v0, v0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_clause 0x1
+; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: ds_read_b32 v0, v0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
+; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-NOTTGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: local_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4
+; GFX940-TGSPLIT-NEXT: ds_read_b32 v0, v0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_clause 0x1
+; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-WGP-NEXT: ds_load_b32 v0, v0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: local_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_clause 0x1
+; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-CU-NEXT: ds_load_b32 v0, v0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: local_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-WGP-NEXT: ds_load_b32 v0, v0
+; GFX12-WGP-NEXT: s_wait_dscnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: local_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-CU-NEXT: ds_load_b32 v0, v0
+; GFX12-CU-NEXT: s_wait_dscnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(3) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(3) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index 069443791b3e1b..e0eff90c4f40ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -813,5 +813,208 @@ entry:
ret void
}
+define amdgpu_kernel void @private_nontemporal_volatile_load(
+; GFX6-LABEL: private_nontemporal_volatile_load:
+; GFX6: ; %bb.0: ; %entry
+; GFX6-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX6-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX6-NEXT: s_add_u32 s8, s8, s7
+; GFX6-NEXT: s_addc_u32 s9, s9, 0
+; GFX6-NEXT: s_mov_b32 s3, 0x100f000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_volatile_load:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT: s_add_u32 s8, s8, s7
+; GFX7-NEXT: s_addc_u32 s9, s9, 0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: buffer_load_dword v2, v0, s[8:11], 0 offen glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_store_dword v[0:1], v2
+; GFX7-NEXT: s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX10-WGP: ; %bb.0: ; %entry
+; GFX10-WGP-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT: s_clause 0x1
+; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT: s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT: s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT: s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_volatile_load:
+; GFX10-CU: ; %bb.0: ; %entry
+; GFX10-CU-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT: s_clause 0x1
+; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT: s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT: s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT: s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_volatile_load:
+; SKIP-CACHE-INV: ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[4:5]
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
+; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s4, s3
+; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s5, 0
+; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen glc
+; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT: s_endpgm
+;
+; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s8, s8, s7
+; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s9, s9, 0
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX90A-TGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX90A-TGSPLIT: ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[10:11], s[2:3]
+; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90A-TGSPLIT-NEXT: s_add_u32 s8, s8, s7
+; GFX90A-TGSPLIT-NEXT: s_addc_u32 s9, s9, 0
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-TGSPLIT-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen glc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-TGSPLIT-NEXT: s_endpgm
+;
+; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
+; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: scratch_load_dword v0, off, s4 sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-NOTTGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NOTTGSPLIT-NEXT: s_endpgm
+;
+; GFX940-TGSPLIT-LABEL: private_nontemporal_volatile_load:
+; GFX940-TGSPLIT: ; %bb.0: ; %entry
+; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, 0
+; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX940-TGSPLIT-NEXT: scratch_load_dword v0, off, s4 sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX940-TGSPLIT-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-TGSPLIT-NEXT: s_endpgm
+;
+; GFX11-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX11-WGP: ; %bb.0: ; %entry
+; GFX11-WGP-NEXT: s_clause 0x1
+; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: scratch_load_b32 v0, off, s2 glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-WGP-NEXT: s_nop 0
+; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-WGP-NEXT: s_endpgm
+;
+; GFX11-CU-LABEL: private_nontemporal_volatile_load:
+; GFX11-CU: ; %bb.0: ; %entry
+; GFX11-CU-NEXT: s_clause 0x1
+; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX11-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: scratch_load_b32 v0, off, s2 glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-CU-NEXT: s_nop 0
+; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-CU-NEXT: s_endpgm
+;
+; GFX12-WGP-LABEL: private_nontemporal_volatile_load:
+; GFX12-WGP: ; %bb.0: ; %entry
+; GFX12-WGP-NEXT: s_clause 0x1
+; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-WGP-NEXT: s_nop 0
+; GFX12-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-WGP-NEXT: s_endpgm
+;
+; GFX12-CU-LABEL: private_nontemporal_volatile_load:
+; GFX12-CU: ; %bb.0: ; %entry
+; GFX12-CU-NEXT: s_clause 0x1
+; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-CU-NEXT: s_nop 0
+; GFX12-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-CU-NEXT: s_endpgm
+ ptr addrspace(5) %in, ptr addrspace(1) %out) {
+entry:
+ %val = load volatile i32, ptr addrspace(5) %in, align 4, !nontemporal !0
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
!0 = !{i32 1}
declare i32 @llvm.amdgcn.workitem.id.x()
>From c66560c981dcc680e8087b6abb1645aa40123945 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 1 Mar 2024 18:29:18 +0100
Subject: [PATCH 2/4] Propely set nontemporal when volatile is also present
Iterator MI can advance in insertWait() but we need original
instruction to set temporal hint. Just move it before handling
volatile.
---
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 10 +++++-----
.../AMDGPU/memory-legalizer-flat-nontemporal.ll | 4 ++--
.../AMDGPU/memory-legalizer-global-nontemporal.ll | 4 ++--
.../AMDGPU/memory-legalizer-private-nontemporal.ll | 4 ++--
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 4069a368f68719..73c23f0f987c3d 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -2392,6 +2392,11 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
bool Changed = false;
+ if (IsNonTemporal) {
+ // Set non-temporal hint for all cache levels.
+ Changed |= setTH(MI, AMDGPU::CPol::TH_NT);
+ }
+
if (IsVolatile) {
Changed |= setScope(MI, AMDGPU::CPol::SCOPE_SYS);
@@ -2407,11 +2412,6 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
Position::AFTER);
}
- if (IsNonTemporal) {
- // Set non-temporal hint for all cache levels.
- Changed |= setTH(MI, AMDGPU::CPol::TH_NT);
- }
-
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 35a9ab72b04362..f6fac11d634c5b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -824,7 +824,7 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-WGP-NEXT: s_wait_dscnt 0x0
@@ -836,7 +836,7 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-CU-NEXT: s_wait_dscnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index 672f803a12235d..c95ff2c691a3cb 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -807,7 +807,7 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-WGP-NEXT: s_nop 0
@@ -819,7 +819,7 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-CU-NEXT: s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index e0eff90c4f40ec..30296a9c3d8963 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -989,7 +989,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, 0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-WGP-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-WGP-NEXT: s_nop 0
@@ -1003,7 +1003,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
; GFX12-CU-NEXT: v_mov_b32_e32 v1, 0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-CU-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-CU-NEXT: s_nop 0
>From b585a1bfbb64a3cff270f4a9c402950169dd38a9 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Mon, 4 Mar 2024 11:03:10 +0100
Subject: [PATCH 3/4] [AMDGPU] Handle amdgpu.last.use metadata
Convert !amdgpu.last.use metadata into Machine Memory Operand for last use
and handle it in SIMemoryLegalizer similary to nontemporal and volatile.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +-
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 56 ++++++++++++++-
.../AMDGPU/memory-legalizer-flat-lastuse.ll | 66 +++++++++++++++++
.../AMDGPU/memory-legalizer-global-lastuse.ll | 65 +++++++++++++++++
.../memory-legalizer-private-lastuse.ll | 70 +++++++++++++++++++
5 files changed, 259 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0df0b5cdf0f392..72c79cc58f8ea9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16181,9 +16181,12 @@ bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
MachineMemOperand::Flags
SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
// Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
if (I.getMetadata("amdgpu.noclobber"))
- return MONoClobber;
- return MachineMemOperand::MONone;
+ Flags |= MONoClobber;
+ if (I.getMetadata("amdgpu.last.use"))
+ Flags |= MOLastUse;
+ return Flags;
}
bool SITargetLowering::checkForPhysRegDependency(
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 73c23f0f987c3d..ff3e6bc9f8b08f 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -98,6 +98,7 @@ class SIMemOpInfo final {
bool IsCrossAddressSpaceOrdering = false;
bool IsVolatile = false;
bool IsNonTemporal = false;
+ bool IsLastUse = false;
SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
SIAtomicScope Scope = SIAtomicScope::SYSTEM,
@@ -107,13 +108,15 @@ class SIMemOpInfo final {
AtomicOrdering FailureOrdering =
AtomicOrdering::SequentiallyConsistent,
bool IsVolatile = false,
- bool IsNonTemporal = false)
+ bool IsNonTemporal = false,
+ bool IsLastUse = false)
: Ordering(Ordering), FailureOrdering(FailureOrdering),
Scope(Scope), OrderingAddrSpace(OrderingAddrSpace),
InstrAddrSpace(InstrAddrSpace),
IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
IsVolatile(IsVolatile),
- IsNonTemporal(IsNonTemporal) {
+ IsNonTemporal(IsNonTemporal),
+ IsLastUse(IsLastUse) {
if (Ordering == AtomicOrdering::NotAtomic) {
assert(Scope == SIAtomicScope::NONE &&
@@ -201,6 +204,12 @@ class SIMemOpInfo final {
return IsNonTemporal;
}
+ /// \returns True if memory access of the machine instruction used to
+ /// create this SIMemOpInfo is last use, false otherwise.
+ bool isLastUse() const {
+ return IsLastUse;
+ }
+
/// \returns True if ordering constraint of the machine instruction used to
/// create this SIMemOpInfo is unordered or higher, false otherwise.
bool isAtomic() const {
@@ -316,6 +325,12 @@ class SICacheControl {
return false;
};
+ /// Update \p MI memory instruction to indicate it is a last use. Return true
+ /// iff the instruction was modified.
+ virtual bool enableLastUse(MachineInstr &MI, bool IsLastUse) const {
+ return false;
+ }
+
/// Inserts any necessary instructions at position \p Pos relative
/// to instruction \p MI to ensure memory instructions before \p Pos of kind
/// \p Op associated with address spaces \p AddrSpace have completed. Used
@@ -592,6 +607,10 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
// MI. \returns Returns true if \p MI is modified, false otherwise.
bool setScope(const MachineBasicBlock::iterator MI,
AMDGPU::CPol::CPol Value) const;
+ // Checks if CPol operand is present in instruction \p MI and if current Scope
+ // policy is same as \p Value.
+ bool isScope(const MachineBasicBlock::iterator MI,
+ AMDGPU::CPol::CPol Value) const;
// Stores with system scope (SCOPE_SYS) need to wait for:
// - loads or atomics(returning) - wait for {LOAD|SAMPLE|BVH|KM}CNT==0
@@ -618,6 +637,9 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
bool IsNonTemporal) const override;
bool expandSystemScopeStore(MachineBasicBlock::iterator &MI) const override;
+
+ bool enableLastUse(MachineInstr &MI,
+ bool IsLastUse) const override;
};
class SIMemoryLegalizer final : public MachineFunctionPass {
@@ -745,12 +767,14 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
bool IsNonTemporal = true;
bool IsVolatile = false;
+ bool IsLastUse = false;
// Validator should check whether or not MMOs cover the entire set of
// locations accessed by the memory instruction.
for (const auto &MMO : MI->memoperands()) {
IsNonTemporal &= MMO->isNonTemporal();
IsVolatile |= MMO->isVolatile();
+ IsLastUse |= MMO->getFlags() & MOLastUse;
InstrAddrSpace |=
toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace());
AtomicOrdering OpOrdering = MMO->getSuccessOrdering();
@@ -792,7 +816,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
}
return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
- IsNonTemporal);
+ IsNonTemporal, IsLastUse);
}
std::optional<SIMemOpInfo>
@@ -2209,6 +2233,15 @@ bool SIGfx12CacheControl::setScope(const MachineBasicBlock::iterator MI,
return false;
}
+bool SIGfx12CacheControl::isScope(const MachineBasicBlock::iterator MI,
+ AMDGPU::CPol::CPol Value) const {
+ MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol);
+ if (!CPol)
+ return false;
+
+ return (CPol->getImm() & AMDGPU::CPol::SCOPE) == Value;
+}
+
bool SIGfx12CacheControl::insertWaitsBeforeSystemScopeStore(
const MachineBasicBlock::iterator MI) const {
// TODO: implement flag for frontend to give us a hint not to insert waits.
@@ -2415,6 +2448,16 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
return Changed;
}
+bool SIGfx12CacheControl::enableLastUse(MachineInstr &MI,
+ bool IsLastUse) const {
+ assert(MI.mayLoad() && !MI.mayStore());
+
+ if (IsLastUse && !isScope(MI, AMDGPU::CPol::SCOPE_SYS))
+ return setTH(MI, AMDGPU::CPol::TH_LU);;
+
+ return false;
+}
+
bool SIGfx12CacheControl::expandSystemScopeStore(
MachineBasicBlock::iterator &MI) const {
MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol);
@@ -2471,12 +2514,19 @@ bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
return Changed;
}
+ // enableVolatileAndOrNonTemporal can insert instructions and advance iterator
+ // MI and we need original instruction for enabling last use.
+ MachineInstr &Inst = *MI;
+
// Atomic instructions already bypass caches to the scope specified by the
// SyncScope operand. Only non-atomic volatile and nontemporal instructions
// need additional treatment.
Changed |= CC->enableVolatileAndOrNonTemporal(MI, MOI.getInstrAddrSpace(),
SIMemOp::LOAD, MOI.isVolatile(),
MOI.isNonTemporal());
+
+ Changed |= CC->enableLastUse(Inst, MOI.isLastUse());
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
new file mode 100644
index 00000000000000..e3ce16fe741f7b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid
+ %val = load i32, ptr %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+define amdgpu_kernel void @flat_last_use_volatile_load(ptr %in, ptr %out) {
+; GFX12-LABEL: flat_last_use_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
new file mode 100644
index 00000000000000..56b1a366708127
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid
+ %val = load i32, ptr addrspace(1) %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @global_last_use_volatile_load(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: global_last_use_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
new file mode 100644
index 00000000000000..f54e7819e1b56b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s
+
+define amdgpu_kernel void @private_nontemporal_load_0(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_load_0:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_load_1(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_load_1:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, v0, s2 th:TH_LOAD_LU
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %val.gep = getelementptr inbounds i32, ptr addrspace(5) %in, i32 %tid
+ %val = load i32, ptr addrspace(5) %val.gep, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_volatile_load(ptr addrspace(5) %in, ptr addrspace(1) %out) {
+; GFX12-LABEL: private_nontemporal_volatile_load:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: scratch_load_b32 v0, off, s2 scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
+entry:
+ %val = load volatile i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{}
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-CU: {{.*}}
+; GFX12-WGP: {{.*}}
>From 39b54116a07e4b9db2ca574c7f481732223fdeda Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Mon, 4 Mar 2024 11:26:09 +0100
Subject: [PATCH 4/4] clang-format
---
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 40 +++++++++-----------
1 file changed, 17 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index ff3e6bc9f8b08f..534a9380b8b77b 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -100,23 +100,20 @@ class SIMemOpInfo final {
bool IsNonTemporal = false;
bool IsLastUse = false;
- SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
- SIAtomicScope Scope = SIAtomicScope::SYSTEM,
- SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC,
- SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL,
- bool IsCrossAddressSpaceOrdering = true,
- AtomicOrdering FailureOrdering =
- AtomicOrdering::SequentiallyConsistent,
- bool IsVolatile = false,
- bool IsNonTemporal = false,
- bool IsLastUse = false)
- : Ordering(Ordering), FailureOrdering(FailureOrdering),
- Scope(Scope), OrderingAddrSpace(OrderingAddrSpace),
- InstrAddrSpace(InstrAddrSpace),
- IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
- IsVolatile(IsVolatile),
- IsNonTemporal(IsNonTemporal),
- IsLastUse(IsLastUse) {
+ SIMemOpInfo(
+ AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
+ SIAtomicScope Scope = SIAtomicScope::SYSTEM,
+ SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC,
+ SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL,
+ bool IsCrossAddressSpaceOrdering = true,
+ AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent,
+ bool IsVolatile = false, bool IsNonTemporal = false,
+ bool IsLastUse = false)
+ : Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
+ OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace),
+ IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
+ IsVolatile(IsVolatile), IsNonTemporal(IsNonTemporal),
+ IsLastUse(IsLastUse) {
if (Ordering == AtomicOrdering::NotAtomic) {
assert(Scope == SIAtomicScope::NONE &&
@@ -206,9 +203,7 @@ class SIMemOpInfo final {
/// \returns True if memory access of the machine instruction used to
/// create this SIMemOpInfo is last use, false otherwise.
- bool isLastUse() const {
- return IsLastUse;
- }
+ bool isLastUse() const { return IsLastUse; }
/// \returns True if ordering constraint of the machine instruction used to
/// create this SIMemOpInfo is unordered or higher, false otherwise.
@@ -638,8 +633,7 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
bool expandSystemScopeStore(MachineBasicBlock::iterator &MI) const override;
- bool enableLastUse(MachineInstr &MI,
- bool IsLastUse) const override;
+ bool enableLastUse(MachineInstr &MI, bool IsLastUse) const override;
};
class SIMemoryLegalizer final : public MachineFunctionPass {
@@ -2453,7 +2447,7 @@ bool SIGfx12CacheControl::enableLastUse(MachineInstr &MI,
assert(MI.mayLoad() && !MI.mayStore());
if (IsLastUse && !isScope(MI, AMDGPU::CPol::SCOPE_SYS))
- return setTH(MI, AMDGPU::CPol::TH_LU);;
+ return setTH(MI, AMDGPU::CPol::TH_LU);
return false;
}
More information about the llvm-commits
mailing list