[llvm] [AMDGPU] SIWholeQuadMode: avoid execz effects in exact regions (PR #101157)
Carl Ritson via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 31 01:49:34 PDT 2024
https://github.com/perlfu updated https://github.com/llvm/llvm-project/pull/101157
>From b452ca59290b918ee619041fb2404afd1e3df501 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Tue, 30 Jul 2024 18:22:02 +0900
Subject: [PATCH 1/2] [AMDGPU] SIWholeQuadMode: avoid execz effects in exact
regions
Exact mode regions within WQM may have EXEC=0 in divergent
control flow. This occurs if a branch is only taken by helper
lanes and an instruction requiring WQM disabling is encountered.
The current code extends the exact region as far as possible;
however, this can result in it including instructions with
unwanted side effects at EXEC=0.
In particular readfirstlane combined with scalar loads
can produce invalid memory accesses in this circumstance.
Workaround this by shrinking exact regions to only the instructions
requiring WQM disabling when unwanted side effects are present.
Eventually we should branch over these regions when EXEC=0, but
this requires visibility of CFG/divergence information not
currently available.
---
llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 26 ++++++++++++----
llvm/test/CodeGen/AMDGPU/wqm.ll | 35 +++++++++++-----------
2 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index df7906ebd8a7e..63f6464a727aa 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -1364,8 +1364,26 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
llvm_unreachable("Unknown state");
break;
}
+ char StartState = State & StateStrict ? NonStrictState : State;
+ bool WQMToExact =
+ StartState == StateWQM && (Needs & StateExact) && !(Needs & StateWQM);
+ bool ExactToWQM = StartState == StateExact && (Needs & StateWQM) &&
+ !(Needs & StateExact);
+ bool PreferLast = Needs == StateWQM;
+ // Exact regions in divergent control flow may run at EXEC=0, so try to
+ // exclude instructions with unexpected effects from them.
+ // FIXME: ideally we would branch over these when EXEC=0,
+ // but this requires updating implicit values, live intervals and CFG.
+ if (WQMToExact || ExactToWQM) {
+ for (MachineBasicBlock::iterator I = First; I != II; ++I) {
+ if (TII->hasUnwantedEffectsWhenEXECEmpty(*I)) {
+ PreferLast = WQMToExact;
+ break;
+ }
+ }
+ }
MachineBasicBlock::iterator Before =
- prepareInsertion(MBB, First, II, Needs == StateWQM, SaveSCC);
+ prepareInsertion(MBB, First, II, PreferLast, SaveSCC);
if (State & StateStrict) {
assert(State == StateStrictWWM || State == StateStrictWQM);
@@ -1385,9 +1403,8 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
toStrictMode(MBB, Before, SavedNonStrictReg, Needs);
State = Needs;
-
} else {
- if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
+ if (WQMToExact) {
if (!WQMFromExec && (OutNeeds & StateWQM)) {
assert(!SavedWQMReg);
SavedWQMReg = MRI->createVirtualRegister(BoolRC);
@@ -1395,8 +1412,7 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
toExact(MBB, Before, SavedWQMReg);
State = StateExact;
- } else if (State == StateExact && (Needs & StateWQM) &&
- !(Needs & StateExact)) {
+ } else if (ExactToWQM) {
assert(WQMFromExec == (SavedWQMReg == 0));
toWQM(MBB, Before, SavedWQMReg);
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index ee71502cdd94e..90508ee37e662 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -3551,13 +3551,13 @@ define amdgpu_ps float @short_exact_regions(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc
; GFX9-W64-NEXT: s_cbranch_execz .LBB59_2
; GFX9-W64-NEXT: ; %bb.1: ; %if
-; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13]
; GFX9-W64-NEXT: global_load_dword v0, v[1:2], off
; GFX9-W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9-W64-NEXT: v_readfirstlane_b32 s18, v0
-; GFX9-W64-NEXT: s_buffer_load_dword s18, s[8:11], s18 offset:0x0
+; GFX9-W64-NEXT: v_readfirstlane_b32 s16, v0
+; GFX9-W64-NEXT: s_buffer_load_dword s16, s[8:11], s16 offset:0x0
; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-W64-NEXT: v_mov_b32_e32 v0, s18
+; GFX9-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13]
; GFX9-W64-NEXT: buffer_store_dwordx4 v[3:6], v0, s[0:3], 0 idxen
; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17]
; GFX9-W64-NEXT: .LBB59_2: ; %endif
@@ -3581,13 +3581,13 @@ define amdgpu_ps float @short_exact_regions(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-W32-NEXT: v_cmpx_gt_u32_e32 16, v0
; GFX10-W32-NEXT: s_cbranch_execz .LBB59_2
; GFX10-W32-NEXT: ; %bb.1: ; %if
-; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12
; GFX10-W32-NEXT: global_load_dword v0, v[1:2], off
; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
-; GFX10-W32-NEXT: v_readfirstlane_b32 s15, v0
-; GFX10-W32-NEXT: s_buffer_load_dword s15, s[8:11], s15 offset:0x0
+; GFX10-W32-NEXT: v_readfirstlane_b32 s14, v0
+; GFX10-W32-NEXT: s_buffer_load_dword s14, s[8:11], s14 offset:0x0
; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-W32-NEXT: v_mov_b32_e32 v0, s15
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s14
+; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12
; GFX10-W32-NEXT: buffer_store_dwordx4 v[3:6], v0, s[0:3], 0 idxen
; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14
; GFX10-W32-NEXT: .LBB59_2: ; %endif
@@ -3633,20 +3633,19 @@ define amdgpu_ps float @short_exact_regions_2(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec
; GFX9-W64-NEXT: s_wqm_b64 exec, exec
; GFX9-W64-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x3
-; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]
-; GFX9-W64-NEXT: global_load_dword v0, v[1:2], off
-; GFX9-W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9-W64-NEXT: image_sample v5, v3, s[0:7], s[8:11] dmask:0x4
-; GFX9-W64-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6 killed $sgpr7
-; GFX9-W64-NEXT: ; kill: killed $vgpr3
+; GFX9-W64-NEXT: global_load_dword v5, v[1:2], off
+; GFX9-W64-NEXT: ; kill: killed $vgpr0
; GFX9-W64-NEXT: ; kill: killed $vgpr1 killed $vgpr2
; GFX9-W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9-W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-W64-NEXT: image_sample v0, v3, s[0:7], s[8:11] dmask:0x4
+; GFX9-W64-NEXT: s_waitcnt vmcnt(1)
+; GFX9-W64-NEXT: v_readfirstlane_b32 s0, v5
; GFX9-W64-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX9-W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v5
+; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v0
; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-W64-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]
; GFX9-W64-NEXT: ; return to shader part epilog
;
; GFX10-W32-LABEL: short_exact_regions_2:
@@ -3654,17 +3653,17 @@ define amdgpu_ps float @short_exact_regions_2(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo
; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX10-W32-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D
-; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX10-W32-NEXT: global_load_dword v0, v[1:2], off
; GFX10-W32-NEXT: s_waitcnt vmcnt(1)
; GFX10-W32-NEXT: image_sample v1, v3, s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_1D
; GFX10-W32-NEXT: s_waitcnt vmcnt(1)
; GFX10-W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-W32-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
; GFX10-W32-NEXT: v_add_f32_e32 v0, v4, v1
-; GFX10-W32-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-W32-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX10-W32-NEXT: ; return to shader part epilog
main_body:
%tex1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
>From 885ed8af9480a1946a9afc93eac5a273c64b9f3c Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 31 Jul 2024 16:24:49 +0900
Subject: [PATCH 2/2] Refine condition for shortening exact region to only
require shortening if exact will be ended and exited.
---
llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 2 +-
llvm/test/CodeGen/AMDGPU/wqm.ll | 19 ++++++++++---------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 63f6464a727aa..9a51cbbb9f6b8 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -1374,7 +1374,7 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
// exclude instructions with unexpected effects from them.
// FIXME: ideally we would branch over these when EXEC=0,
// but this requires updating implicit values, live intervals and CFG.
- if (WQMToExact || ExactToWQM) {
+ if ((WQMToExact && (OutNeeds & StateWQM)) || ExactToWQM) {
for (MachineBasicBlock::iterator I = First; I != II; ++I) {
if (TII->hasUnwantedEffectsWhenEXECEmpty(*I)) {
PreferLast = WQMToExact;
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index 90508ee37e662..6b4c2da772cdc 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -3633,19 +3633,20 @@ define amdgpu_ps float @short_exact_regions_2(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec
; GFX9-W64-NEXT: s_wqm_b64 exec, exec
; GFX9-W64-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x3
-; GFX9-W64-NEXT: global_load_dword v5, v[1:2], off
-; GFX9-W64-NEXT: ; kill: killed $vgpr0
-; GFX9-W64-NEXT: ; kill: killed $vgpr1 killed $vgpr2
+; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]
+; GFX9-W64-NEXT: global_load_dword v0, v[1:2], off
; GFX9-W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9-W64-NEXT: image_sample v0, v3, s[0:7], s[8:11] dmask:0x4
+; GFX9-W64-NEXT: image_sample v5, v3, s[0:7], s[8:11] dmask:0x4
+; GFX9-W64-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6 killed $sgpr7
+; GFX9-W64-NEXT: ; kill: killed $vgpr3
+; GFX9-W64-NEXT: ; kill: killed $vgpr1 killed $vgpr2
; GFX9-W64-NEXT: s_waitcnt vmcnt(1)
-; GFX9-W64-NEXT: v_readfirstlane_b32 s0, v5
+; GFX9-W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-W64-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX9-W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v5
; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-W64-NEXT: v_add_f32_e32 v0, s0, v0
-; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]
; GFX9-W64-NEXT: ; return to shader part epilog
;
; GFX10-W32-LABEL: short_exact_regions_2:
@@ -3653,17 +3654,17 @@ define amdgpu_ps float @short_exact_regions_2(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo
; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX10-W32-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D
+; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX10-W32-NEXT: global_load_dword v0, v[1:2], off
; GFX10-W32-NEXT: s_waitcnt vmcnt(1)
; GFX10-W32-NEXT: image_sample v1, v3, s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_1D
; GFX10-W32-NEXT: s_waitcnt vmcnt(1)
; GFX10-W32-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-W32-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
; GFX10-W32-NEXT: v_add_f32_e32 v0, v4, v1
+; GFX10-W32-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x0
; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-W32-NEXT: v_add_f32_e32 v0, s0, v0
-; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX10-W32-NEXT: ; return to shader part epilog
main_body:
%tex1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
More information about the llvm-commits
mailing list