[llvm] relax D16 war (PR #194477)
Guo Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 27 16:50:56 PDT 2026
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/194477
>From 5e53438b5a4101413ac3b9a698138f1772eee9f0 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Mon, 27 Apr 2026 12:56:23 -0400
Subject: [PATCH] relax D16 war
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 48 +++++++++++++++++--
.../branch-relaxation-inst-size-gfx11.ll | 4 +-
llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll | 1 -
llvm/test/CodeGen/AMDGPU/function-args.ll | 30 ++++++++----
4 files changed, 66 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index f00b99075b022..eca63ebe4676e 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -803,6 +803,8 @@ class WaitcntBrackets {
void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCPhysReg Reg,
AMDGPU::Waitcnt &Wait) const;
+ MCPhysReg determineVGPR16Dependency(const MachineInstr &MI,
+ MCPhysReg Reg) const;
void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
AMDGPU::Waitcnt &Wait) const;
AMDGPU::Waitcnt determineAsyncWait(unsigned N);
@@ -920,10 +922,6 @@ class WaitcntBrackets {
assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
if (!Context->TRI.isInAllocatableClass(Reg))
return {{}, {}};
- const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
- unsigned Size = Context->TRI.getRegSizeInBits(*RC);
- if (Size == 16 && Context->ST.hasD16Writes32BitVgpr())
- Reg = Context->TRI.get32BitRegister(Reg);
return Context->TRI.regunits(Reg);
}
@@ -1643,6 +1641,46 @@ void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
}
}
+MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
+ MCPhysReg Reg) const {
+ const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
+ unsigned Size = Context->TRI.getRegSizeInBits(*RC);
+
+ if (!(Size == 16) || !Context->ST.hasD16Writes32BitVgpr())
+ return Reg;
+
+ // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
+ // check dependency on the other half
+ Register Reg32 = Context->TRI.get32BitRegister(Reg);
+ Register OtherHalf = Register(Context->TRI.getSubReg(
+ Reg32,
+ AMDGPU::isHi16Reg(Reg, Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16));
+
+ AMDGPU::Waitcnt Wait;
+ for (MCRegUnit RU : regunits(OtherHalf))
+ for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
+ determineWaitForScore(T, getVMemScore(toVMEMID(RU), T), Wait);
+
+ // no wait on otherhalf
+ if (!Wait.hasWait())
+ return Reg;
+
+ if (Context->TII.isVALU(MI))
+ return Reg32;
+
+ // if hi/lo16 mixed events
+ WaitEventSet MIEvents = Context->getEventsFor(MI);
+ WaitEventSet OtherHalfEvents;
+ for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
+ if (Wait.get(T) != ~0u)
+ OtherHalfEvents |= Context->getWaitEvents(T);
+ }
+ WaitEventSet Events = MIEvents & OtherHalfEvents;
+ if (Events.twoOrMore())
+ return Reg32;
+ return Reg;
+}
+
void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
VMEMID TID,
AMDGPU::Waitcnt &Wait) const {
@@ -2650,6 +2688,8 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
if (Op.isImplicit() && MI.mayLoadOrStore())
continue;
+ Reg = ScoreBrackets.determineVGPR16Dependency(MI, Reg);
+
ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST, Reg, Wait);
if (Op.isDef())
ScoreBrackets.determineWaitForPhysReg(AMDGPU::VM_VSRC, Reg, Wait);
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx11.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx11.ll
index 6bebc8f5d0d18..dd389375b0d77 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx11.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation-inst-size-gfx11.ll
@@ -23,9 +23,9 @@ define amdgpu_kernel void @long_forward_branch_gfx11plus(ptr addrspace(1) %in, p
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_d16_b16 v0, v1, s[0:1]
; GFX11-NEXT: global_load_d16_hi_b16 v0, v1, s[0:1] offset:2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_waitcnt vmcnt(1)
; GFX11-NEXT: global_store_b16 v1, v0, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_store_d16_hi_b16 v1, v0, s[2:3] offset:2
; GFX11-NEXT: .LBB0_2: ; %bb3
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index 76f84756dc3c1..062f682bf52ad 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -1093,7 +1093,6 @@ define <2 x i16> @chain_hi_to_lo_group_may_alias_store(ptr addrspace(3) %ptr, pt
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x7b
; GFX11-TRUE16-NEXT: ds_load_u16_d16_hi v2, v0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_store_b16 v1, v2
; GFX11-TRUE16-NEXT: ds_load_u16_d16 v2, v0 offset:2
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 3928a44595761..07dcad993ffe2 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -2680,14 +2680,16 @@ define void @void_func_v32i32_i1_i8_i16_bf16(<32 x i32> %arg0, i1 %arg1, i8 %arg
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: buffer_store_b8 v16, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: buffer_store_b8 v32, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b16 v32, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: buffer_store_b16 v33, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b16 v33, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4166,44 +4168,52 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: buffer_store_b8 v32, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v32, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-TRUE16-NEXT: buffer_store_b8 v33, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v33, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: buffer_store_b8 v34, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v34, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: buffer_store_b8 v35, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v35, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: buffer_store_b8 v36, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v36, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-TRUE16-NEXT: buffer_store_b8 v37, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v37, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: buffer_store_b8 v38, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v38, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: buffer_store_b8 v39, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_store_d16_hi_b8 v39, off, s[0:3], 0 dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
More information about the llvm-commits
mailing list