[Mlir-commits] [llvm] [mlir] [AMDGPU] Let LDSDMA ops cross a sched_barrier when the mask allows it (PR #203255)
Arseniy Obolenskiy
llvmlistbot at llvm.org
Thu Aug 6 21:16:40 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/203255
>From 88bca2e791eab836ff927a8599fa69c3d6a9f8a5 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 11 Jun 2026 14:41:55 +0200
Subject: [PATCH 1/4] [AMDGPU] Let LDSDMA ops cross a sched_barrier when the
mask allows it
A mask allowing LDSDMA (0x800) still kept LDSDMA ops pinned, because canAddMI matched them as VMEM and invertSchedBarrierMask left the DS bit set
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 9 +++--
.../test/CodeGen/AMDGPU/sched-ldsdma-mask.mir | 38 +++++++++++++++++++
.../AMDGPU/sched.barrier.inverted.mask.ll | 8 ++--
3 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 6c95dc76f4f6f..04639f55e3935 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2472,11 +2472,11 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
Result = true;
else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) &&
- MI.mayLoad() && TII->isVMEM(MI))
+ MI.mayLoad() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
Result = true;
else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) &&
- MI.mayStore() && TII->isVMEM(MI))
+ MI.mayStore() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
Result = true;
else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) &&
@@ -2679,9 +2679,10 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE &
~SchedGroupMask::LDSDMA;
- // DS_READ, DS_WRITE implies DS.
+ // DS_READ, DS_WRITE, LDSDMA implies DS.
else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE)
+ (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE ||
+ (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS;
LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask
diff --git a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
index 1ab3e349fc0a0..c1924706a0751 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
@@ -10,6 +10,7 @@
define amdgpu_kernel void @sched_group_barrier_mask_2048_async_loads() { ret void }
define amdgpu_kernel void @sched_group_barrier_mask_16_async_loads() { ret void }
define amdgpu_kernel void @sched_group_barrier_mask_2048_interleave_2load_1valu() { ret void }
+ define amdgpu_kernel void @sched_barrier_mask_2049_allows_ldsdma() { ret void }
...
---
@@ -340,3 +341,40 @@ body: |
S_ENDPGM 0, implicit %30, implicit %31, implicit %32
...
+
+# A SCHED_BARRIER whose mask allows LDSDMA (bit 0x800) together with ALU
+# (bit 0x1) must let an LDSDMA op be scheduled across it. The second
+# async load is hoisted above the barrier.
+
+---
+name: sched_barrier_mask_2049_allows_ldsdma
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: sched_barrier_mask_2049_allows_ldsdma
+ ; CHECK: $exec = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF1]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF2]], [[DEF]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF3]], [[DEF4]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[DEF3]], implicit $exec
+ ; CHECK-NEXT: SCHED_BARRIER 2049
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_1]]
+ $exec = IMPLICIT_DEF
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 %1, %0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %10:vgpr_32 = V_ADD_U32_e32 %3, %4, implicit $exec
+ SCHED_BARRIER 2049
+ GLOBAL_LOAD_ASYNC_TO_LDS_B32 %2, %0, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ %11:vgpr_32 = V_ADD_U32_e32 %10, %3, implicit $exec
+ S_ENDPGM 0, implicit %11
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index ca802ac586668..71476ab29029b 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -41,8 +41,8 @@ entry:
ret void
}
-; Inverted 1935: 011110001111
-; GCN: After Inverting, SchedGroup Mask: 1935
+; Inverted 1807: 011100001111
+; GCN: After Inverting, SchedGroup Mask: 1807
define amdgpu_kernel void @invert16() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 16) #1
@@ -104,8 +104,8 @@ entry:
ret void
}
-; Inverted 2031: 011111101111
-; GCN: After Inverting, SchedGroup Mask: 2031
+; Inverted 1903: 011101101111
+; GCN: After Inverting, SchedGroup Mask: 1903
define amdgpu_kernel void @invert2048() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 2048) #1
>From 5ffbcec374e0e3ca4bc76de709036a24cd9a92f7 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 5 Aug 2026 15:32:47 +0200
Subject: [PATCH 2/4] fix test
---
llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir | 36 ++++++++++----------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir b/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
index 7854f2141abb9..e2c0e4567391a 100644
--- a/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
+++ b/llvm/test/CodeGen/AMDGPU/ldsdmacnt_sched.mir
@@ -323,21 +323,21 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_load
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
; MAXOCC-LABEL: name: async_load_async_load
@@ -485,20 +485,20 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 2, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
@@ -543,20 +543,20 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait1
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 0, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
@@ -600,21 +600,21 @@ tracksRegLiveness: true
body: |
bb.0:
; DEFAULT-LABEL: name: async_load_async_wait2
- ; DEFAULT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT: SCHED_GROUP_BARRIER 32, 1, 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
; DEFAULT-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF2]], [[DEF3]], implicit $exec
- ; DEFAULT-NEXT: SCHED_GROUP_BARRIER 32, 1, 0
; DEFAULT-NEXT: SCHED_GROUP_BARRIER 2, 6, 0
+ ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+ ; DEFAULT-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR [[DEF]], [[DEF1]], [[DEF4]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
; DEFAULT-NEXT: S_WAIT_ASYNCCNT 2, implicit-def dead $asynccnt, implicit $asynccnt
; DEFAULT-NEXT: S_ENDPGM 0
;
>From 359c3abb992ec1a381ce88b1425a1f45144a8778 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 6 Aug 2026 06:13:13 +0200
Subject: [PATCH 3/4] upd masks docs
---
llvm/docs/AMDGPUUsage.rst | 14 ++++++++++----
mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 4 ++++
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 87ba811e1a686..a33868290011e 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1906,10 +1906,16 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
- 0x0002: VALU instructions may be scheduled across sched_barrier.
- 0x0004: SALU instructions may be scheduled across sched_barrier.
- 0x0008: MFMA/WMMA instructions may be scheduled across sched_barrier.
- - 0x0010: All VMEM instructions may be scheduled across sched_barrier.
- - 0x0020: VMEM read instructions may be scheduled across sched_barrier.
- - 0x0040: VMEM write instructions may be scheduled across sched_barrier.
- - 0x0080: All DS instructions may be scheduled across sched_barrier.
+ - 0x0010: All VMEM instructions may be scheduled across sched_barrier. This
+ includes LDSDMA instructions.
+ - 0x0020: VMEM read instructions may be scheduled across sched_barrier. This
+ does not include LDSDMA instructions; only loads targeting VGPRs are
+ allowed.
+ - 0x0040: VMEM write instructions may be scheduled across sched_barrier. This
+ does not include LDSDMA instructions; only stores sourcing from VGPRs are
+ allowed.
+ - 0x0080: All DS instructions may be scheduled across sched_barrier. This
+ includes LDSDMA instructions.
- 0x0100: All DS read instructions may be scheduled across sched_barrier.
- 0x0200: All DS write instructions may be scheduled across sched_barrier.
- 0x0400: All Transcendental (e.g. V_EXP) instructions may be scheduled across sched_barrier.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index f1195e9c644d5..a4488a990d6cd 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -765,6 +765,10 @@ def ROCDL_SchedBarrier : ROCDL_ConcreteNonMemIntrOp<"sched.barrier", [], 0, [0],
intrinsic's documented mask values and the AMDGPU backend's
`SchedGroupMask` enum.
+ `all_vmem` and `all_ds` are supersets that also include LDSDMA
+ instructions, while `vmem_read`, `vmem_write`, `ds_read`, and `ds_write`
+ do not.
+
Example:
```mlir
// Scheduling barrier with no instructions allowed to cross.
>From e5d096acdc987da9cf1afa2f277ec8751f268d9b Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 7 Aug 2026 06:16:27 +0200
Subject: [PATCH 4/4] correct wording
---
llvm/docs/AMDGPUUsage.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index a33868290011e..15731c83e69b8 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1909,11 +1909,11 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
- 0x0010: All VMEM instructions may be scheduled across sched_barrier. This
includes LDSDMA instructions.
- 0x0020: VMEM read instructions may be scheduled across sched_barrier. This
- does not include LDSDMA instructions; only loads targeting VGPRs are
- allowed.
+ does not include LDSDMA loads, even though they also read from global
+ memory; only loads targeting VGPRs are classified as VMEM read.
- 0x0040: VMEM write instructions may be scheduled across sched_barrier. This
- does not include LDSDMA instructions; only stores sourcing from VGPRs are
- allowed.
+ does not include LDSDMA stores, even though they also write to global
+ memory; only stores sourcing from VGPRs are classified as VMEM write.
- 0x0080: All DS instructions may be scheduled across sched_barrier. This
includes LDSDMA instructions.
- 0x0100: All DS read instructions may be scheduled across sched_barrier.
More information about the Mlir-commits
mailing list