[llvm] [AMDGPU] Let LDSDMA ops cross a sched_barrier when the mask allows it (PR #203255)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 07:20:58 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/203255
>From 88bca2e791eab836ff927a8599fa69c3d6a9f8a5 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 11 Jun 2026 14:41:55 +0200
Subject: [PATCH 1/2] [AMDGPU] Let LDSDMA ops cross a sched_barrier when the
mask allows it
A mask allowing LDSDMA (0x800) still kept LDSDMA ops pinned, because canAddMI matched them as VMEM and invertSchedBarrierMask left the DS bit set
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 9 +++--
.../test/CodeGen/AMDGPU/sched-ldsdma-mask.mir | 38 +++++++++++++++++++
.../AMDGPU/sched.barrier.inverted.mask.ll | 8 ++--
3 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 6c95dc76f4f6f..04639f55e3935 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2472,11 +2472,11 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
Result = true;
else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) &&
- MI.mayLoad() && TII->isVMEM(MI))
+ MI.mayLoad() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
Result = true;
else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) &&
- MI.mayStore() && TII->isVMEM(MI))
+ MI.mayStore() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
Result = true;
else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) &&
@@ -2679,9 +2679,10 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE &
~SchedGroupMask::LDSDMA;
- // DS_READ, DS_WRITE implies DS.
+ // DS_READ, DS_WRITE, LDSDMA implies DS.
else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE)
+ (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE ||
+ (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS;
LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask
diff --git a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
index 1ab3e349fc0a0..c1924706a0751 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
@@ -10,6 +10,7 @@
define amdgpu_kernel void @sched_group_barrier_mask_2048_async_loads() { ret void }
define amdgpu_kernel void @sched_group_barrier_mask_16_async_loads() { ret void }
define amdgpu_kernel void @sched_group_barrier_mask_2048_interleave_2load_1valu() { ret void }
+ define amdgpu_kernel void @sched_barrier_mask_2049_allows_ldsdma() { ret void }
...
---
@@ -340,3 +341,40 @@ body: |
S_ENDPGM 0, implicit %30, implicit %31, implicit %32
...
+
+# A SCHED_BARRIER whose mask allows LDSDMA (bit 0x800) together with ALU
+# (bit 0x1) must let an LDSDMA op be scheduled across it. The second
+# async load is hoisted above the barrier.
+
+---
+name: sched_barrier_mask_2049_allows_ldsdma
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: sched_barrier_mask_2049_allows_ldsdma
+ ; CHECK: $exec = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF1]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF2]], [[DEF]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF3]], [[DEF4]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[DEF3]], implicit $exec
+ ; CHECK-NEXT: SCHED_BARRIER 2049
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_1]]
+ $exec = IMPLICIT_DEF
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 %1, %0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %10:vgpr_32 = V_ADD_U32_e32 %3, %4, implicit $exec
+ SCHED_BARRIER 2049
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 %2, %0, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %11:vgpr_32 = V_ADD_U32_e32 %10, %3, implicit $exec
+ S_ENDPGM 0, implicit %11
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index ca802ac586668..71476ab29029b 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -41,8 +41,8 @@ entry:
ret void
}
-; Inverted 1935: 011110001111
-; GCN: After Inverting, SchedGroup Mask: 1935
+; Inverted 1807: 011100001111
+; GCN: After Inverting, SchedGroup Mask: 1807
define amdgpu_kernel void @invert16() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 16) #1
@@ -104,8 +104,8 @@ entry:
ret void
}
-; Inverted 2031: 011111101111
-; GCN: After Inverting, SchedGroup Mask: 2031
+; Inverted 1903: 011101101111
+; GCN: After Inverting, SchedGroup Mask: 1903
define amdgpu_kernel void @invert2048() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 2048) #1
>From 5ffbcec374e0e3ca4bc76de709036a24cd9a92f7 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 5 Aug 2026 15:32:47 +0200
Subject: [PATCH 2/2] fix test
---
llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir | 36 ++++++++++----------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir b/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
index 7854f2141abb9..e2c0e4567391a 100644
--- a/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
+++ b/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
@@ -323,21 +323,21 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_load
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
; MAXOCC-LABEL: name: async_load_async_load
@@ -485,20 +485,20 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 2, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
@@ -543,20 +543,20 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait1
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 0, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
@@ -600,21 +600,21 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait2
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 2, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
More information about the llvm-commits
mailing list