[llvm] [AMDGPU] Improve execz branch removal for async load-to-LDS (PR #192341)
Vigneshwar Jayakumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 13:49:27 PDT 2026
https://github.com/VigneshwarJ updated https://github.com/llvm/llvm-project/pull/192341
>From 3b14aff9110752716e027421c9597a95fc024975 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 15 Apr 2026 12:33:35 -0500
Subject: [PATCH 1/3] changes
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 12 +++++-
.../AMDGPU/remove-execz-branch-async-dma.ll | 42 +++++++++++++++++++
2 files changed, 53 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index a496c9a4daa71..7e98b70c73a2b 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -428,7 +428,17 @@ class BranchWeightCostModel {
if (TII.isWaitcnt(MI.getOpcode()))
return false;
- ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
+ // Async loads to LDS use the issue cost (1 cycle) instead of the
+ // scheduling model's resource latency so the cost model can remove the
+ // execz branch when the block is cheap enough. Only loads are
+ // overridden; async stores risk writing zeros to global memory for
+ // OOB LDS source addresses. Async loads have a vdst operand (LDS
+ // destination); stores do not.
+ if (SIInstrInfo::usesASYNC_CNT(MI) &&
+ TII.getNamedOperand(MI, AMDGPU::OpName::vdst))
+ ThenCyclesCost += 1;
+ else
+ ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
// Consider `P = N/D` to be the probability of execz being false (skipping
// the then-block) The transformation is profitable if always executing the
diff --git a/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll b/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
new file mode 100644
index 0000000000000..c51b172f52b6b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s
+
+ at lds = external addrspace(3) global [0 x i8], align 16
+
+declare void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1), ptr addrspace(3), i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.s.wait.asynccnt(i16 immarg)
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+; The cost model treats async DMA at its 1-cycle issue cost rather than the
+; scheduling model's full memory latency. When the guarded block contains
+; only cheap ALU and the async load (no waitcnts), the branch is removed.
+;
+; CHECK-LABEL: async_load_no_waitcnt:
+; CHECK-NOT: s_cbranch_execz
+; CHECK: global_load_async_to_lds_b128
+; CHECK: s_or_b32 exec_lo
+define amdgpu_ps void @async_load_no_waitcnt(
+ ptr addrspace(1) inreg %src,
+ i32 %bound
+) {
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %lds_off = shl nuw nsw i32 %tid, 4
+ %lds_ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 %lds_off
+ %off = sext i32 %tid to i64
+ %gep = getelementptr i8, ptr addrspace(1) %src, i64 %off
+ %cmp = icmp slt i32 %tid, %bound
+ br i1 %cmp, label %do_load, label %skip
+
+do_load:
+ tail call void @llvm.amdgcn.global.load.async.to.lds.b128(
+ ptr addrspace(1) %gep,
+ ptr addrspace(3) %lds_ptr,
+ i32 0, i32 0)
+ br label %join
+
+skip:
+ br label %join
+
+join:
+ tail call void @llvm.amdgcn.s.wait.asynccnt(i16 0)
+ ret void
+}
>From 3214e77a0a6725c1ac4a1bbf08d6a3bc9e9316ff Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 15 Apr 2026 15:26:58 -0500
Subject: [PATCH 2/3] [AMDGPU] Improve execz branch removal for async
load-to-LDS
Override the costs of async LDS loads to 1 cycle in SIPreEmitPeepholes
execz branch weight's cost model. HW drops the load for OOB LDS
destinations.
As per the gfx1250 LDS spec, the HW drops async loads when the
LDS destination address is out of range, so executing at EXEC=0 is
safe. Async stores (global_store_async_from_lds) are excluded because
the HW reads zeros from OOB LDS sources and writes them to global
memory.
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 37 ++++++++++++++++----
1 file changed, 30 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 7e98b70c73a2b..04d5d9a2db131 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -401,6 +401,30 @@ bool SIPreEmitPeephole::getBlockDestinations(
return true;
}
+static bool isAsyncLoadToLDS(unsigned Opc) {
+ switch (Opc) {
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
+ case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
+ return true;
+ default:
+ return false;
+ }
+}
+
namespace {
class BranchWeightCostModel {
const SIInstrInfo &TII;
@@ -428,14 +452,13 @@ class BranchWeightCostModel {
if (TII.isWaitcnt(MI.getOpcode()))
return false;
- // Async loads to LDS use the issue cost (1 cycle) instead of the
+ // gfx1250 async loads to LDS: use issue cost (1 cycle) instead of the
// scheduling model's resource latency so the cost model can remove the
- // execz branch when the block is cheap enough. Only loads are
- // overridden; async stores risk writing zeros to global memory for
- // OOB LDS source addresses. Async loads have a vdst operand (LDS
- // destination); stores do not.
- if (SIInstrInfo::usesASYNC_CNT(MI) &&
- TII.getNamedOperand(MI, AMDGPU::OpName::vdst))
+ // execz branch when the block is cheap enough. The HW drops async
+ // loads when the LDS destination address is OOB, so executing at
+ // EXEC=0 is safe. Async stores are NOT overridden: the HW reads
+ // zeros from OOB LDS sources and writes them to global memory.
+ if (isAsyncLoadToLDS(MI.getOpcode()))
ThenCyclesCost += 1;
else
ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
>From b637870e5920d3475358bbe69690bbe001ffea1f Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 15 Apr 2026 15:46:08 -0500
Subject: [PATCH 3/3] updated tests to autogenereate
---
.../AMDGPU/remove-execz-branch-async-dma.ll | 34 +++++++++----------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll b/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
index c51b172f52b6b..faf63dbefdcda 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
+++ b/llvm/test/CodeGen/AMDGPU/remove-execz-branch-async-dma.ll
@@ -1,35 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s
- at lds = external addrspace(3) global [0 x i8], align 16
-
-declare void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1), ptr addrspace(3), i32 immarg, i32 immarg)
-declare void @llvm.amdgcn.s.wait.asynccnt(i16 immarg)
-declare i32 @llvm.amdgcn.workitem.id.x()
-
; The cost model treats async DMA at its 1-cycle issue cost rather than the
; scheduling model's full memory latency. When the guarded block contains
; only cheap ALU and the async load (no waitcnts), the branch is removed.
;
-; CHECK-LABEL: async_load_no_waitcnt:
-; CHECK-NOT: s_cbranch_execz
-; CHECK: global_load_async_to_lds_b128
-; CHECK: s_or_b32 exec_lo
define amdgpu_ps void @async_load_no_waitcnt(
- ptr addrspace(1) inreg %src,
+; CHECK-LABEL: async_load_no_waitcnt:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; CHECK-NEXT: s_mov_b32 s2, exec_lo
+; CHECK-NEXT: v_cmpx_lt_i32_e64 s0, v1
+; CHECK-NEXT: ; %bb.1: ; %do_load
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1]
+; CHECK-NEXT: ; %bb.2: ; %join
+; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; CHECK-NEXT: s_wait_asynccnt 0x0
+; CHECK-NEXT: s_endpgm
+ ptr addrspace(1) inreg %gaddr,
+ ptr addrspace(3) %laddr,
i32 %bound
) {
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %lds_off = shl nuw nsw i32 %tid, 4
- %lds_ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 %lds_off
- %off = sext i32 %tid to i64
- %gep = getelementptr i8, ptr addrspace(1) %src, i64 %off
%cmp = icmp slt i32 %tid, %bound
br i1 %cmp, label %do_load, label %skip
do_load:
tail call void @llvm.amdgcn.global.load.async.to.lds.b128(
- ptr addrspace(1) %gep,
- ptr addrspace(3) %lds_ptr,
+ ptr addrspace(1) %gaddr,
+ ptr addrspace(3) %laddr,
i32 0, i32 0)
br label %join
More information about the llvm-commits
mailing list