[llvm] [AMDGPU] Clear DS aggregate in inverted sched_barrier mask when LDSDMA allowed (PR #207779)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 7 07:41:23 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/207779
>From 813a43b9b1e13c989aeeec315531c834261ef9c7 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 6 Jul 2026 18:59:50 +0200
Subject: [PATCH 1/3] [AMDGPU] Clear DS aggregate in inverted sched_barrier
mask when LDSDMA allowed
The DS clause was missing the LDSDMA check that the VMEM clause has
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 5 +++--
llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll | 8 ++++----
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index a7db2b5957a04..b1734d77c67e4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2822,9 +2822,10 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE &
~SchedGroupMask::LDSDMA;
- // DS_READ, DS_WRITE implies DS.
+ // DS_READ, DS_WRITE, LDSDMA implies DS.
else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE)
+ (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE ||
+ (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
InvertedMask &= ~SchedGroupMask::DS;
LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index e96485a27db33..b80d7377b8a55 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -41,8 +41,8 @@ entry:
ret void
}
-; Inverted 1935: 011110001111
-; GCN: After Inverting, SchedGroup Mask: 1935
+; Inverted 1807: 011100001111
+; GCN: After Inverting, SchedGroup Mask: 1807
define amdgpu_kernel void @invert16() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 16) #1
@@ -104,8 +104,8 @@ entry:
ret void
}
-; Inverted 2031: 011111101111
-; GCN: After Inverting, SchedGroup Mask: 2031
+; Inverted 1903: 011101101111
+; GCN: After Inverting, SchedGroup Mask: 1903
define amdgpu_kernel void @invert2048() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 2048) #1
>From 9a8f1b8e4a3ca1313be83609eb73977e4b773e0a Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 7 Jul 2026 06:07:10 +0200
Subject: [PATCH 2/3] Address comment
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 49 +++-----
.../AMDGPU/sched.barrier.inverted.mask.ll | 106 +++---------------
2 files changed, 31 insertions(+), 124 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index b1734d77c67e4..197b5657233ab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2796,37 +2796,24 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
// allowed past the SCHED_BARRIER.
SchedGroupMask InvertedMask = ~Mask;
- // ALU implies VALU, SALU, MFMA, TRANS.
- if ((InvertedMask & SchedGroupMask::ALU) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::VALU & ~SchedGroupMask::SALU &
- ~SchedGroupMask::MFMA & ~SchedGroupMask::TRANS;
- // VALU, SALU, MFMA, TRANS implies ALU.
- else if ((InvertedMask & SchedGroupMask::VALU) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::SALU) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::MFMA) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::TRANS) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::ALU;
-
- // VMEM implies VMEM_READ, VMEM_WRITE, LDSDMA.
- if ((InvertedMask & SchedGroupMask::VMEM) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::VMEM_READ & ~SchedGroupMask::VMEM_WRITE &
- ~SchedGroupMask::LDSDMA;
- // VMEM_READ, VMEM_WRITE, LDSDMA implies VMEM.
- else if ((InvertedMask & SchedGroupMask::VMEM_READ) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::VMEM_WRITE) ==
- SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::VMEM;
-
- // DS implies DS_READ, DS_WRITE, LDSDMA.
- if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE &
- ~SchedGroupMask::LDSDMA;
- // DS_READ, DS_WRITE, LDSDMA implies DS.
- else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE ||
- (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
- InvertedMask &= ~SchedGroupMask::DS;
+ static constexpr std::pair<SchedGroupMask, SchedGroupMask> ImpliedGroups[] = {
+ {SchedGroupMask::ALU, SchedGroupMask::VALU | SchedGroupMask::SALU |
+ SchedGroupMask::MFMA | SchedGroupMask::TRANS},
+ {SchedGroupMask::VMEM, SchedGroupMask::VMEM_READ |
+ SchedGroupMask::VMEM_WRITE |
+ SchedGroupMask::LDSDMA},
+ {SchedGroupMask::DS, SchedGroupMask::DS_READ | SchedGroupMask::DS_WRITE |
+ SchedGroupMask::LDSDMA},
+ };
+
+ for (auto [Aggregate, Members] : ImpliedGroups) {
+ // Aggregate allowed past the barrier implies all its members are too.
+ if ((InvertedMask & Aggregate) == SchedGroupMask::NONE)
+ InvertedMask &= ~Members;
+ // Any member allowed past the barrier implies the aggregate is too.
+ else if ((InvertedMask & Members) != Members)
+ InvertedMask &= ~Aggregate;
+ }
LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask
<< "\n");
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index b80d7377b8a55..131e52ba63e67 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -2,112 +2,32 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx600 < %s -debug-only=igrouplp 2>&1 | FileCheck --check-prefixes=GCN %s
-
-
-
-; Inverted 3056: 101111110000
+; Barriers are processed in reverse order
+; GCN: After Inverting, SchedGroup Mask: 1903
+; GCN: After Inverting, SchedGroup Mask: 3070
+; GCN: After Inverting, SchedGroup Mask: 3455
+; GCN: After Inverting, SchedGroup Mask: 3711
+; GCN: After Inverting, SchedGroup Mask: 1151
+; GCN: After Inverting, SchedGroup Mask: 4015
+; GCN: After Inverting, SchedGroup Mask: 4047
+; GCN: After Inverting, SchedGroup Mask: 1807
+; GCN: After Inverting, SchedGroup Mask: 4086
+; GCN: After Inverting, SchedGroup Mask: 4090
+; GCN: After Inverting, SchedGroup Mask: 4092
; GCN: After Inverting, SchedGroup Mask: 3056
-define amdgpu_kernel void @invert1() #0 {
+define amdgpu_kernel void @invert() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 1) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 4092: 111111111100
-; GCN: After Inverting, SchedGroup Mask: 4092
-define amdgpu_kernel void @invert2() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 2) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 4090: 111111111010
-; GCN: After Inverting, SchedGroup Mask: 4090
-define amdgpu_kernel void @invert4() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 4) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 4086: 111111110110
-; GCN: After Inverting, SchedGroup Mask: 4086
-define amdgpu_kernel void @invert8() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 8) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 1807: 011100001111
-; GCN: After Inverting, SchedGroup Mask: 1807
-define amdgpu_kernel void @invert16() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 16) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 4047: 111111001111
-; GCN: After Inverting, SchedGroup Mask: 4047
-define amdgpu_kernel void @invert32() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 32) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 4015: 111110101111
-; GCN: After Inverting, SchedGroup Mask: 4015
-define amdgpu_kernel void @invert64() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 64) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 3199: 110001111111
-; GCN: After Inverting, SchedGroup Mask: 1151
-define amdgpu_kernel void @invert128() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 128) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 3711: 111001111111
-; GCN: After Inverting, SchedGroup Mask: 3711
-define amdgpu_kernel void @invert256() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 256) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 3455: 110101111111
-; GCN: After Inverting, SchedGroup Mask: 3455
-define amdgpu_kernel void @invert512() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 512) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 3070: 101111111110
-; GCN: After Inverting, SchedGroup Mask: 3070
-define amdgpu_kernel void @invert1024() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 1024) #1
- call void @llvm.amdgcn.s.nop(i16 0) #1
- ret void
-}
-
-; Inverted 1903: 011101101111
-; GCN: After Inverting, SchedGroup Mask: 1903
-define amdgpu_kernel void @invert2048() #0 {
-entry:
call void @llvm.amdgcn.sched.barrier(i32 2048) #1
call void @llvm.amdgcn.s.nop(i16 0) #1
ret void
>From 71a53c2509c116b45ad9994086cb5c764a1ee0e2 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 7 Jul 2026 16:31:54 +0200
Subject: [PATCH 3/3] revert test
---
.../AMDGPU/sched.barrier.inverted.mask.ll | 106 +++++++++++++++---
1 file changed, 93 insertions(+), 13 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index 131e52ba63e67..b80d7377b8a55 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -2,32 +2,112 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx600 < %s -debug-only=igrouplp 2>&1 | FileCheck --check-prefixes=GCN %s
-; Barriers are processed in reverse order
-; GCN: After Inverting, SchedGroup Mask: 1903
-; GCN: After Inverting, SchedGroup Mask: 3070
-; GCN: After Inverting, SchedGroup Mask: 3455
-; GCN: After Inverting, SchedGroup Mask: 3711
-; GCN: After Inverting, SchedGroup Mask: 1151
-; GCN: After Inverting, SchedGroup Mask: 4015
-; GCN: After Inverting, SchedGroup Mask: 4047
-; GCN: After Inverting, SchedGroup Mask: 1807
-; GCN: After Inverting, SchedGroup Mask: 4086
-; GCN: After Inverting, SchedGroup Mask: 4090
-; GCN: After Inverting, SchedGroup Mask: 4092
+
+
+
+; Inverted 3056: 101111110000
; GCN: After Inverting, SchedGroup Mask: 3056
-define amdgpu_kernel void @invert() #0 {
+define amdgpu_kernel void @invert1() #0 {
entry:
call void @llvm.amdgcn.sched.barrier(i32 1) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 4092: 111111111100
+; GCN: After Inverting, SchedGroup Mask: 4092
+define amdgpu_kernel void @invert2() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 2) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 4090: 111111111010
+; GCN: After Inverting, SchedGroup Mask: 4090
+define amdgpu_kernel void @invert4() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 4) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 4086: 111111110110
+; GCN: After Inverting, SchedGroup Mask: 4086
+define amdgpu_kernel void @invert8() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 8) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 1807: 011100001111
+; GCN: After Inverting, SchedGroup Mask: 1807
+define amdgpu_kernel void @invert16() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 16) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 4047: 111111001111
+; GCN: After Inverting, SchedGroup Mask: 4047
+define amdgpu_kernel void @invert32() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 32) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 4015: 111110101111
+; GCN: After Inverting, SchedGroup Mask: 4015
+define amdgpu_kernel void @invert64() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 64) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 3199: 110001111111
+; GCN: After Inverting, SchedGroup Mask: 1151
+define amdgpu_kernel void @invert128() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 128) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 3711: 111001111111
+; GCN: After Inverting, SchedGroup Mask: 3711
+define amdgpu_kernel void @invert256() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 256) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 3455: 110101111111
+; GCN: After Inverting, SchedGroup Mask: 3455
+define amdgpu_kernel void @invert512() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 512) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 3070: 101111111110
+; GCN: After Inverting, SchedGroup Mask: 3070
+define amdgpu_kernel void @invert1024() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 1024) #1
+ call void @llvm.amdgcn.s.nop(i16 0) #1
+ ret void
+}
+
+; Inverted 1903: 011101101111
+; GCN: After Inverting, SchedGroup Mask: 1903
+define amdgpu_kernel void @invert2048() #0 {
+entry:
call void @llvm.amdgcn.sched.barrier(i32 2048) #1
call void @llvm.amdgcn.s.nop(i16 0) #1
ret void
More information about the llvm-commits
mailing list