[clang] [llvm] [AMDGPU] Emit a waitcnt instruction after each memory instruction (PR #68932)
Jun Wang via cfe-commits
cfe-commits at lists.llvm.org
Wed Nov 22 11:19:28 PST 2023
https://github.com/jwanggit86 updated https://github.com/llvm/llvm-project/pull/68932
>From e393477607cb94b45a3b9a5db2aea98fb8af2a86 Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Thu, 12 Oct 2023 16:45:59 -0500
Subject: [PATCH 01/11] [AMDGPU] Emit a waitcnt instruction after each memory
instruction
This patch implements a new command-line option for the backend, namely,
amdgpu-waitcnt-for-all-mem-op. When this option is specified, a "waitcnt 0"
instruction is generated after each memory load/store instruction.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 30 ++-
.../CodeGen/AMDGPU/insert_waitcnt_for_all.ll | 222 ++++++++++++++++++
2 files changed, 251 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index ede4841b8a5fd7d..728be7c61fa2217 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -52,6 +52,10 @@ static cl::opt<bool> ForceEmitZeroFlag(
cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
cl::init(false), cl::Hidden);
+static cl::opt<bool> EmitForAllMemOpFlag(
+ "amdgpu-waitcnt-for-all-mem-op",
+ cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
+
namespace {
// Class of object that encapsulates latest instruction counter score
// associated with the operand. Used for determining whether
@@ -388,6 +392,8 @@ class SIInsertWaitcnts : public MachineFunctionPass {
// message.
DenseSet<MachineInstr *> ReleaseVGPRInsts;
+ bool insertWaitcntAfterMemOp(MachineFunction &MF);
+
public:
static char ID;
@@ -1809,6 +1815,23 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
return HasVMemLoad && UsesVgprLoadedOutside;
}
+bool SIInsertWaitcnts::insertWaitcntAfterMemOp(MachineFunction &MF) {
+ bool Modified = false;
+
+ for (auto &MBB : MF) {
+ for (auto It = MBB.begin(); It != MBB.end();) {
+ bool IsMemOp = It->mayLoadOrStore();
+ ++It;
+ if (IsMemOp) {
+ BuildMI(MBB, It, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0);
+ Modified = true;
+ }
+ }
+ }
+
+ return Modified;
+}
+
bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
TII = ST->getInstrInfo();
@@ -1819,6 +1842,12 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
MLI = &getAnalysis<MachineLoopInfo>();
PDT = &getAnalysis<MachinePostDominatorTree>();
+ bool Modified = false;
+
+ if (EmitForAllMemOpFlag) {
+ Modified = insertWaitcntAfterMemOp(MF);
+ }
+
ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
for (auto T : inst_counter_types())
ForceEmitWaitcnt[T] = false;
@@ -1847,7 +1876,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
TrackedWaitcntSet.clear();
BlockInfos.clear();
- bool Modified = false;
if (!MFI->isEntryFunction()) {
// Wait for any outstanding memory operations that the input registers may
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
new file mode 100644
index 000000000000000..4580b9074ada3cc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
@@ -0,0 +1,222 @@
+; Testing the -amdgpu-waitcnt-for-all-mem-op option
+; COM: llc -mtriple=amdgcn -mcpu=hawaii -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
+; COM: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global,+enable-flat-scratch -amdgpu-use-divergent-register-indexing -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
+
+; from atomicrmw-expand.ll
+; covers flat_load, flat_atomic
+define void @syncscope_workgroup_nortn(ptr %addr, float %val) {
+; GFX90A-LABEL: syncscope_workgroup_nortn:
+; GFX90A: ; %bb.0:
+; GFX90A: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A: .LBB0_1: ; %atomicrmw.start
+; GFX90A: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst
+ ret void
+}
+
+; from atomicrmw-nand.ll
+; covers global_atomic, global_load
+define i32 @atomic_nand_i32_global(ptr addrspace(1) %ptr) nounwind {
+; GFX9-LABEL: atomic_nand_i32_global:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB1_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NOT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-NEXT: v_not_b32_e32 v2, v3
+; GFX9-NEXT: v_or_b32_e32 v2, -5, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB1_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw nand ptr addrspace(1) %ptr, i32 4 seq_cst
+ ret i32 %result
+}
+
+; from bf16.ll
+; covers buffer_load, buffer_store, flat_load, flat_store, global_load, global_store
+define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+;
+; GFX7-LABEL: test_load_store:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: test_load_store:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_load_ushort v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_store_short v[2:3], v0
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: test_load_store:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_ushort v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_load_store:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_ushort v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+ %val = load bfloat, ptr addrspace(1) %in
+ store bfloat %val, ptr addrspace(1) %out
+ ret void
+}
+
+; from scratch-simple.ll
+; covers scratch_load, scratch_store
+;
+; GFX9-FLATSCR-LABEL: {{^}}vs_main:
+; GFX9-FLATSCR: scratch_store_dwordx4 off, v[{{[0-9:]+}}],
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLATSCR: scratch_load_dword {{v[0-9]+}}, {{v[0-9]+}}, off
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_vs float @vs_main(i32 %idx) {
+ %v1 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
+ %v2 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
+ %r = fadd float %v1, %v2
+ ret float %r
+}
+
+; from udiv.ll
+; covers s_load
+define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
+; GFX9-LABEL: udiv_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NOT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
+ %r = udiv i32 %x, %y
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32)
+
+; from smrd.ll
+; covers s_buffer_load
+; GFX9-LABEL: {{^}}smrd_sgpr_offset:
+; GFX9: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 {
+main_body:
+ %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
+ ret float %r
+}
+
+; from atomic_load_add.ll
+; covers s_load, ds_add
+; GFX9-LABEL: atomic_add_local:
+; GFX9: ; %bb.1:
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9: ds_add_u32 v0, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
+ %unused = atomicrmw volatile add ptr addrspace(3) %local, i32 5 seq_cst
+ ret void
+}
+
+declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32 immarg)
+
+; from atomic_optimizations_buffer.ll
+; covers buffer_atomic
+; GFX9-LABEL: add_i32_constant:
+; GFX9: ; %bb.1:
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9: buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
+entry:
+ %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)
+ store i32 %old, ptr addrspace(1) %out
+ ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32, i16, <8 x i32>, i32, i32)
+
+; from llvm.amdgcn.image.load.a16.ll
+; covers image_load
+; GFX9-LABEL: {{^}}load.f32.1d:
+; GFX9: image_load v0, v0, s[0:7] dmask:0x1 unorm a16
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_ps <4 x float> @load.f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
+main_body:
+ %x = extractelement <2 x i16> %coords, i32 0
+ %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 1, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x float> %v
+}
+
+declare void @llvm.amdgcn.image.store.1d.v4f32.i16(<4 x float>, i32, i16, <8 x i32>, i32, i32)
+
+; from llvm.amdgcn.image.store.a16.ll
+; covers image_store
+define amdgpu_ps void @store_f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <4 x float> %val) {
+; GFX9-LABEL: store_f32_1d:
+; GFX9: ; %bb.0: ; %main_body
+; GFX9-NEXT: image_store v[1:4], v0, s[0:7] dmask:0x1 unorm a16
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+main_body:
+ %x = extractelement <2 x i16> %coords, i32 0
+ call void @llvm.amdgcn.image.store.1d.v4f32.i16(<4 x float> %val, i32 1, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
+ ret void
+}
+
+declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32, i32, <8 x i32>, i32, i32)
+
+; from llvm.amdgcn.image.atomic.dim.ll
+; covers image_atomic
+; GFX90A-LABEL: {{^}}atomic_swap_1d:
+; GFX90A: image_atomic_swap v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+main_body:
+ %v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+ %out = bitcast i32 %v to float
+ ret float %out
+}
+
+
+
+
>From 68d7dec2b2802f6477e8a65efaf9e4724c77cffb Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Mon, 6 Nov 2023 16:48:53 -0600
Subject: [PATCH 02/11] (1) change cmdline option name to
amdgpu-precise-memory-op (2) option is added for both clang and llc
---
clang/include/clang/Driver/Options.td | 3 +++
clang/test/Driver/amdgpu-features.c | 6 ++++++
llvm/lib/Target/AMDGPU/AMDGPU.td | 6 ++++++
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 5 +++++
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 10 +++++++---
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll | 14 +++++++-------
6 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index b2f2bcb6ac37910..16fe33f51752a94 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4723,6 +4723,9 @@ defm tgsplit : SimpleMFlag<"tgsplit", "Enable", "Disable",
defm wavefrontsize64 : SimpleMFlag<"wavefrontsize64",
"Specify wavefront size 64", "Specify wavefront size 32",
" mode (AMDGPU only)">;
+defm amdgpu_precise_memory_op : SimpleMFlag<"amdgpu-precise-memory-op",
+ "Enable", "Disable",
+ " precise memory mode (AMDGPU only)", m_amdgpu_Features_Group>;
defm unsafe_fp_atomics : BoolOption<"m", "unsafe-fp-atomics",
TargetOpts<"AllowAMDGPUUnsafeFPAtomics">, DefaultFalse,
diff --git a/clang/test/Driver/amdgpu-features.c b/clang/test/Driver/amdgpu-features.c
index a516bc6b7ff2004..57d31ccedd8783e 100644
--- a/clang/test/Driver/amdgpu-features.c
+++ b/clang/test/Driver/amdgpu-features.c
@@ -32,3 +32,9 @@
// RUN: %clang -### -target amdgcn -mcpu=gfx1010 -mno-cumode %s 2>&1 | FileCheck --check-prefix=NO-CUMODE %s
// NO-CUMODE: "-target-feature" "-cumode"
+
+// RUN: %clang -### -target amdgcn -mcpu=gfx1010 -mamdgpu-precise-memory-op %s 2>&1 | FileCheck --check-prefix=PREC-MEM %s
+// PREC-MEM: "-target-feature" "+amdgpu-precise-memory-op"
+
+// RUN: %clang -### -target amdgcn -mcpu=gfx1010 -mno-amdgpu-precise-memory-op %s 2>&1 | FileCheck --check-prefix=NO-PREC-MEM %s
+// NO-PREC-MEM: "-target-feature" "-amdgpu-precise-memory-op"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index bf244bb024a7240..df668cb2846b131 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -167,6 +167,12 @@ def FeatureCuMode : SubtargetFeature<"cumode",
"Enable CU wavefront execution mode"
>;
+def FeaturePreciseMemory : SubtargetFeature<"amdgpu-precise-memory-op",
+ "EnablePreciseMemory",
+ "true",
+ "Enable precise memory mode"
+>;
+
def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
"SGPRInitBug",
"true",
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 4c624ed59427c95..0d5098a6b162ee1 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -87,6 +87,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool EnableTgSplit = false;
bool EnableCuMode = false;
bool TrapHandler = false;
+ bool EnablePreciseMemory = false;
// Used as options.
bool EnableLoadStoreOpt = false;
@@ -578,6 +579,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
return EnableCuMode;
}
+ bool isPreciseMemoryEnabled() const {
+ return EnablePreciseMemory;
+ }
+
bool hasFlatAddressSpace() const {
return FlatAddressSpace;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 728be7c61fa2217..760848643d0751c 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -56,6 +56,10 @@ static cl::opt<bool> EmitForAllMemOpFlag(
"amdgpu-waitcnt-for-all-mem-op",
cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
+static cl::opt<bool> PreciseMemOpFlag(
+ "amdgpu-precise-memory-op",
+ cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
+
namespace {
// Class of object that encapsulates latest instruction counter score
// associated with the operand. Used for determining whether
@@ -393,7 +397,6 @@ class SIInsertWaitcnts : public MachineFunctionPass {
DenseSet<MachineInstr *> ReleaseVGPRInsts;
bool insertWaitcntAfterMemOp(MachineFunction &MF);
-
public:
static char ID;
@@ -1631,6 +1634,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
bool FlushVmCnt = Block.getFirstTerminator() == Inst &&
isPreheaderToFlush(Block, ScoreBrackets);
+
// Generate an s_waitcnt instruction to be placed before Inst, if needed.
Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
FlushVmCnt);
@@ -1844,8 +1848,8 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
bool Modified = false;
- if (EmitForAllMemOpFlag) {
- Modified = insertWaitcntAfterMemOp(MF);
+ if (ST->isPreciseMemoryEnabled() || PreciseMemOpFlag) {
+ Modified |= insertWaitcntAfterMemOp(MF);
}
ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
index 4580b9074ada3cc..79ce0c505ad9763 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
@@ -1,10 +1,10 @@
-; Testing the -amdgpu-waitcnt-for-all-mem-op option
-; COM: llc -mtriple=amdgcn -mcpu=hawaii -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
-; COM: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
-; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global,+enable-flat-scratch -amdgpu-use-divergent-register-indexing -amdgpu-waitcnt-for-all-mem-op -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
+; Testing the -amdgpu-precise-memory-op option
+; COM: llc -mtriple=amdgcn -mcpu=hawaii -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
+; COM: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global,+enable-flat-scratch -amdgpu-use-divergent-register-indexing -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
; from atomicrmw-expand.ll
; covers flat_load, flat_atomic
>From f34726f36287d7fa077c9aeb031648fef826ded0 Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Mon, 6 Nov 2023 17:26:50 -0600
Subject: [PATCH 03/11] Remove some obsolete code.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 760848643d0751c..8f2a9cc984ada22 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -52,10 +52,6 @@ static cl::opt<bool> ForceEmitZeroFlag(
cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
cl::init(false), cl::Hidden);
-static cl::opt<bool> EmitForAllMemOpFlag(
- "amdgpu-waitcnt-for-all-mem-op",
- cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
-
static cl::opt<bool> PreciseMemOpFlag(
"amdgpu-precise-memory-op",
cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
>From 53475cc3c237c44d9b960a1a4470fdd7212fabca Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Mon, 6 Nov 2023 18:28:37 -0600
Subject: [PATCH 04/11] Fix code formatting for GCNSubtarget.h.
---
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 0d5098a6b162ee1..fdcd44a4d6ccb82 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -579,9 +579,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
return EnableCuMode;
}
- bool isPreciseMemoryEnabled() const {
- return EnablePreciseMemory;
- }
+ bool isPreciseMemoryEnabled() const { return EnablePreciseMemory; }
bool hasFlatAddressSpace() const {
return FlatAddressSpace;
>From e39db24c98b5f036f3a8cd5bfdc13c9f53238b1d Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Mon, 6 Nov 2023 19:00:09 -0600
Subject: [PATCH 05/11] Fix code format.
---
clang/include/clang/Driver/Options.td | 7 ++++---
llvm/lib/Target/AMDGPU/AMDGPU.td | 8 +++-----
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 7 ++++---
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 16fe33f51752a94..d4374dec59e1bad 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4723,9 +4723,10 @@ defm tgsplit : SimpleMFlag<"tgsplit", "Enable", "Disable",
defm wavefrontsize64 : SimpleMFlag<"wavefrontsize64",
"Specify wavefront size 64", "Specify wavefront size 32",
" mode (AMDGPU only)">;
-defm amdgpu_precise_memory_op : SimpleMFlag<"amdgpu-precise-memory-op",
- "Enable", "Disable",
- " precise memory mode (AMDGPU only)", m_amdgpu_Features_Group>;
+defm amdgpu_precise_memory_op
+ : SimpleMFlag<"amdgpu-precise-memory-op", "Enable", "Disable",
+ " precise memory mode (AMDGPU only)",
+ m_amdgpu_Features_Group>;
defm unsafe_fp_atomics : BoolOption<"m", "unsafe-fp-atomics",
TargetOpts<"AllowAMDGPUUnsafeFPAtomics">, DefaultFalse,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index df668cb2846b131..cb3ccd760d15235 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -167,11 +167,9 @@ def FeatureCuMode : SubtargetFeature<"cumode",
"Enable CU wavefront execution mode"
>;
-def FeaturePreciseMemory : SubtargetFeature<"amdgpu-precise-memory-op",
- "EnablePreciseMemory",
- "true",
- "Enable precise memory mode"
->;
+def FeaturePreciseMemory
+ : SubtargetFeature<"amdgpu-precise-memory-op", "EnablePreciseMemory",
+ "true", "Enable precise memory mode">;
def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
"SGPRInitBug",
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 8f2a9cc984ada22..3d2d4c64eb0cfef 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -52,9 +52,10 @@ static cl::opt<bool> ForceEmitZeroFlag(
cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
cl::init(false), cl::Hidden);
-static cl::opt<bool> PreciseMemOpFlag(
- "amdgpu-precise-memory-op",
- cl::desc("Emit s_waitcnt 0 after each memory operation"), cl::init(false));
+static cl::opt<bool>
+ PreciseMemOpFlag("amdgpu-precise-memory-op",
+ cl::desc("Emit s_waitcnt 0 after each memory operation"),
+ cl::init(false));
namespace {
// Class of object that encapsulates latest instruction counter score
>From 70f117771e696365aaec53bf50f1e53b567c0d2e Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Mon, 6 Nov 2023 19:50:48 -0600
Subject: [PATCH 06/11] Fix formatting.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 3d2d4c64eb0cfef..faff98903140a06 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -394,6 +394,7 @@ class SIInsertWaitcnts : public MachineFunctionPass {
DenseSet<MachineInstr *> ReleaseVGPRInsts;
bool insertWaitcntAfterMemOp(MachineFunction &MF);
+
public:
static char ID;
@@ -1631,7 +1632,6 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
bool FlushVmCnt = Block.getFirstTerminator() == Inst &&
isPreheaderToFlush(Block, ScoreBrackets);
-
// Generate an s_waitcnt instruction to be placed before Inst, if needed.
Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
FlushVmCnt);
>From fc486ba14d3e7f8f3560f933f5feb17fd2523f68 Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Thu, 9 Nov 2023 17:34:23 -0600
Subject: [PATCH 07/11] Remove the command-line option for llc because it is
not really necessary,given that a command-line option has been added for
clang.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 7 +------
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll | 12 ++++++------
2 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index faff98903140a06..5e3be5a8e20c5c2 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -52,11 +52,6 @@ static cl::opt<bool> ForceEmitZeroFlag(
cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
cl::init(false), cl::Hidden);
-static cl::opt<bool>
- PreciseMemOpFlag("amdgpu-precise-memory-op",
- cl::desc("Emit s_waitcnt 0 after each memory operation"),
- cl::init(false));
-
namespace {
// Class of object that encapsulates latest instruction counter score
// associated with the operand. Used for determining whether
@@ -1845,7 +1840,7 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
bool Modified = false;
- if (ST->isPreciseMemoryEnabled() || PreciseMemOpFlag) {
+ if (ST->isPreciseMemoryEnabled()) {
Modified |= insertWaitcntAfterMemOp(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
index 79ce0c505ad9763..fb2449dfa72bfb9 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
@@ -1,10 +1,10 @@
; Testing the -amdgpu-precise-memory-op option
-; COM: llc -mtriple=amdgcn -mcpu=hawaii -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
-; COM: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
-; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global,+enable-flat-scratch -amdgpu-use-divergent-register-indexing -amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
+; COM: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
+; COM: llc -mtriple=amdgcn -mcpu=tonga -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global,+enable-flat-scratch,+amdgpu-precise-memory-op -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
; from atomicrmw-expand.ll
; covers flat_load, flat_atomic
>From 907a538a45eace8f9dd22f40a00982263e8fdceb Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Thu, 9 Nov 2023 20:54:52 -0600
Subject: [PATCH 08/11] Integrating inserting "s_waitcnt 0" with existing
SIInsertWaintcnt pass.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 5e3be5a8e20c5c2..a67a2b736c73bd1 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -388,7 +388,7 @@ class SIInsertWaitcnts : public MachineFunctionPass {
// message.
DenseSet<MachineInstr *> ReleaseVGPRInsts;
- bool insertWaitcntAfterMemOp(MachineFunction &MF);
+ // bool insertWaitcntAfterMemOp(MachineFunction &MF);
public:
static char ID;
@@ -1710,6 +1710,13 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
}
++Iter;
+ if (ST->isPreciseMemoryEnabled() && Inst.mayLoadOrStore()) {
+ auto builder =
+ BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
+ .addImm(0);
+ OldWaitcntInstr = builder.getInstr();
+ Modified = true;
+ }
}
if (Block.getFirstTerminator() == Block.end() &&
@@ -1811,6 +1818,7 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
return HasVMemLoad && UsesVgprLoadedOutside;
}
+#if 0
bool SIInsertWaitcnts::insertWaitcntAfterMemOp(MachineFunction &MF) {
bool Modified = false;
@@ -1827,6 +1835,7 @@ bool SIInsertWaitcnts::insertWaitcntAfterMemOp(MachineFunction &MF) {
return Modified;
}
+#endif
bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
@@ -1838,11 +1847,11 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
MLI = &getAnalysis<MachineLoopInfo>();
PDT = &getAnalysis<MachinePostDominatorTree>();
- bool Modified = false;
-
+#if 0
if (ST->isPreciseMemoryEnabled()) {
Modified |= insertWaitcntAfterMemOp(MF);
}
+#endif
ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
for (auto T : inst_counter_types())
@@ -1873,6 +1882,8 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
TrackedWaitcntSet.clear();
BlockInfos.clear();
+ bool Modified = false;
+
if (!MFI->isEntryFunction()) {
// Wait for any outstanding memory operations that the input registers may
// depend on. We can't track them and it's better to do the wait after the
>From a505836d11c9b865a3282484867c7dabd4a2d15f Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Tue, 21 Nov 2023 17:36:26 -0600
Subject: [PATCH 09/11] Made changes based on code review: (1) for GFX10+
generate a s_waitcnt_vscnt 0 as well (2) split test files into two. (3)
removed unused code.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 39 ++++---------
.../CodeGen/AMDGPU/insert_waitcnt_for_all.ll | 27 +--------
.../AMDGPU/insert_waitcnt_for_all_2.ll | 57 +++++++++++++++++++
3 files changed, 70 insertions(+), 53 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index a67a2b736c73bd1..3e5d8736592ad8d 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -388,8 +388,6 @@ class SIInsertWaitcnts : public MachineFunctionPass {
// message.
DenseSet<MachineInstr *> ReleaseVGPRInsts;
- // bool insertWaitcntAfterMemOp(MachineFunction &MF);
-
public:
static char ID;
@@ -1607,6 +1605,8 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
VCCZCorrect = false;
}
+ bool IsGFX10Plus = AMDGPU::isGFX10Plus(*ST);
+
// Walk over the instructions.
MachineInstr *OldWaitcntInstr = nullptr;
@@ -1711,10 +1711,16 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
++Iter;
if (ST->isPreciseMemoryEnabled() && Inst.mayLoadOrStore()) {
- auto builder =
+ auto Builder =
BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
.addImm(0);
- OldWaitcntInstr = builder.getInstr();
+ if (IsGFX10Plus) {
+ Builder =
+ BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT))
+ .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
+ .addImm(0);
+ }
+ OldWaitcntInstr = Builder.getInstr();
Modified = true;
}
}
@@ -1818,25 +1824,6 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
return HasVMemLoad && UsesVgprLoadedOutside;
}
-#if 0
-bool SIInsertWaitcnts::insertWaitcntAfterMemOp(MachineFunction &MF) {
- bool Modified = false;
-
- for (auto &MBB : MF) {
- for (auto It = MBB.begin(); It != MBB.end();) {
- bool IsMemOp = It->mayLoadOrStore();
- ++It;
- if (IsMemOp) {
- BuildMI(MBB, It, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0);
- Modified = true;
- }
- }
- }
-
- return Modified;
-}
-#endif
-
bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
TII = ST->getInstrInfo();
@@ -1847,12 +1834,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
MLI = &getAnalysis<MachineLoopInfo>();
PDT = &getAnalysis<MachinePostDominatorTree>();
-#if 0
- if (ST->isPreciseMemoryEnabled()) {
- Modified |= insertWaitcntAfterMemOp(MF);
- }
-#endif
-
ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
for (auto T : inst_counter_types())
ForceEmitWaitcnt[T] = false;
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
index fb2449dfa72bfb9..aef8e81e0bb3df8 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
@@ -1,6 +1,4 @@
; Testing the -amdgpu-precise-memory-op option
-; COM: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
-; COM: llc -mtriple=amdgcn -mcpu=tonga -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX90A
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
@@ -55,28 +53,6 @@ define i32 @atomic_nand_i32_global(ptr addrspace(1) %ptr) nounwind {
; covers buffer_load, buffer_store, flat_load, flat_store, global_load, global_store
define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
;
-; GFX7-LABEL: test_load_store:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_load_store:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_ushort v0, v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_store_short v[2:3], v0
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
; GFX9-LABEL: test_load_store:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -91,8 +67,11 @@ define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_ushort v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%val = load bfloat, ptr addrspace(1) %in
store bfloat %val, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
new file mode 100644
index 000000000000000..97adaec68c34723
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
@@ -0,0 +1,57 @@
+; Testing the -amdgpu-precise-memory-op option
+; RUN: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX7
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX8
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+amdgpu-precise-memory-op -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
+
+; from bf16.ll
+; covers buffer_load, buffer_store, flat_load, flat_store, global_load, global_store
+define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+;
+; GFX7-LABEL: test_load_store:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: test_load_store:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_load_ushort v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: flat_store_short v[2:3], v0
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: test_load_store:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_ushort v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_store_short v[2:3], v0, off
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_load_store:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: global_load_ushort v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_store_short v[2:3], v0, off
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+ %val = load bfloat, ptr addrspace(1) %in
+ store bfloat %val, ptr addrspace(1) %out
+ ret void
+}
+
>From 8456eb00b14358cb56e8f1fcf4d83cf7403724ed Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Tue, 21 Nov 2023 19:43:41 -0600
Subject: [PATCH 10/11] Fix code formatting.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 3e5d8736592ad8d..289fd147a984011 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1715,10 +1715,10 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
.addImm(0);
if (IsGFX10Plus) {
- Builder =
- BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT))
- .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
- .addImm(0);
+ Builder =
+ BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT))
+ .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
+ .addImm(0);
}
OldWaitcntInstr = Builder.getInstr();
Modified = true;
>From c935ed0ca3a7945bc756c5424678d18a9453acb3 Mon Sep 17 00:00:00 2001
From: Jun Wang <jun.wang7 at amd.com>
Date: Wed, 22 Nov 2023 13:16:05 -0600
Subject: [PATCH 11/11] Code changed based on code review: (1) update
OldWaitcntInstr such that it points to the inserted s_waitcnt even when the
s_waitcnt is followed by an s_waitcnt_vscnt (2) updated testcases
accordingly.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 ++----
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll | 1 -
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll | 1 -
3 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 289fd147a984011..a555a799be39678 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1714,13 +1714,14 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
auto Builder =
BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
.addImm(0);
+ OldWaitcntInstr = Builder.getInstr();
+
if (IsGFX10Plus) {
Builder =
BuildMI(Block, Iter, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT))
.addReg(AMDGPU::SGPR_NULL, RegState::Undef)
.addImm(0);
}
- OldWaitcntInstr = Builder.getInstr();
Modified = true;
}
}
@@ -1860,9 +1861,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
TRI->getEncodingValue(AMDGPU::SGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK;
Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
- TrackedWaitcntSet.clear();
- BlockInfos.clear();
-
bool Modified = false;
if (!MFI->isEntryFunction()) {
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
index aef8e81e0bb3df8..8d90a962369f510 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all.ll
@@ -68,7 +68,6 @@ define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX10-NEXT: global_load_ushort v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
index 97adaec68c34723..f4bccb2554daa57 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_all_2.ll
@@ -45,7 +45,6 @@ define void @test_load_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX10-NEXT: global_load_ushort v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
More information about the cfe-commits
mailing list