[llvm] [AMDGPU] Optimize out s_barrier_signal/_wait (PR #116993)

Piotr Sobczak via llvm-commits llvm-commits at lists.llvm.org
Thu Nov 21 04:02:53 PST 2024


https://github.com/piotrAMD updated https://github.com/llvm/llvm-project/pull/116993

>From 3955c2bd13a417a72d333c295e434197bda1eba5 Mon Sep 17 00:00:00 2001
From: Piotr Sobczak <piotr.sobczak at amd.com>
Date: Wed, 20 Nov 2024 16:32:58 +0100
Subject: [PATCH 1/4] [AMDGPU] Add test

---
 .../AMDGPU/barrier-elimination-gfx12.ll       | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll

diff --git a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
new file mode 100644
index 00000000000000..0268f643b5533e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
@@ -0,0 +1,53 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s -global-isel | FileCheck %s
+
+; CHECK-LABEL: {{^}}signal_unknown_wgs:
+; CHECK: s_barrier_signal
+define amdgpu_kernel void @signal_unknown_wgs() {
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+  ret void
+}
+
+; CHECK-LABEL: {{^}}signal_flat_wgs_attr_32_128:
+; CHECK: s_barrier_signal
+define amdgpu_kernel void @signal_flat_wgs_attr_32_128() #1 {
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+  ret void
+}
+
+; CHECK-LABEL: {{^}}signal_flat_wgs_attr_32_64:
+; CHECK: s_barrier_signal
+define amdgpu_kernel void @signal_flat_wgs_attr_32_64() #2 {
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+  ret void
+}
+
+
+; CHECK-LABEL: {{^}}wait_unknown_wgs:
+; CHECK: s_barrier_wait
+define amdgpu_kernel void @wait_unknown_wgs() {
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wait_flat_wgs_attr_32_128:
+; CHECK: s_barrier_wait
+define amdgpu_kernel void @wait_flat_wgs_attr_32_128() #1 {
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wait_flat_wgs_attr_32_64:
+; CHECK: :
+; CHECK-NEXT: s_barrier_wait
+define amdgpu_kernel void @wait_flat_wgs_attr_32_64() #2 {
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+  ret void
+}
+
+declare void @llvm.amdgcn.s.barrier.signal(i32 immarg) #0
+declare void @llvm.amdgcn.s.barrier.wait(i16 immarg) #0
+
+attributes #0 = { convergent nounwind }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="32,128" }
+attributes #2 = { nounwind "amdgpu-flat-work-group-size"="16,32" }

>From f800d648616582b7450b07410a886533d5bcb2b9 Mon Sep 17 00:00:00 2001
From: Piotr Sobczak <piotr.sobczak at amd.com>
Date: Wed, 20 Nov 2024 16:33:38 +0100
Subject: [PATCH 2/4] [AMDGPU] Optimize out s_barrier_signal/_wait

Extend the optimization that converts s_barrier to wave_barrier (nop)
when the number of work items is not larger than wave size.

This handles the "split barrier" form of s_barrier where the barrier
is represented by separate intrinsics (s_barrier_signal/s_barrier_wait).
Note: the version where s_barrier is used in gfx12 (and later split)
has the optimization already, but some front-ends may prefer to use
split intrinsics and this is being addressed by the patch.
---
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp  | 7 +++++--
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp             | 8 +++++---
 llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll | 7 +++++--
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 3522ece24f1c45..f01f57de460c75 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1843,8 +1843,9 @@ bool AMDGPUInstructionSelector::selectSBarrier(MachineInstr &MI) const {
     }
   }
 
-  // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
-  if (STI.hasSplitBarriers()) {
+  Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+  if (STI.hasSplitBarriers() && IntrinsicID == Intrinsic::amdgcn_s_barrier) {
+    // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
     MachineBasicBlock *MBB = MI.getParent();
     const DebugLoc &DL = MI.getDebugLoc();
     BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM))
@@ -2161,6 +2162,8 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
   case Intrinsic::amdgcn_init_whole_wave:
     return selectInitWholeWave(I);
   case Intrinsic::amdgcn_s_barrier:
+  case Intrinsic::amdgcn_s_barrier_signal:
+  case Intrinsic::amdgcn_s_barrier_wait:
     return selectSBarrier(I);
   case Intrinsic::amdgcn_raw_buffer_load_lds:
   case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5b02f9bf80d3fc..4743ff0f65690e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -9605,7 +9605,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
     return SDValue(DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops), 0);
   }
-  case Intrinsic::amdgcn_s_barrier: {
+  case Intrinsic::amdgcn_s_barrier:
+  case Intrinsic::amdgcn_s_barrier_signal:
+  case Intrinsic::amdgcn_s_barrier_wait: {
     const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
     if (getTargetMachine().getOptLevel() > CodeGenOptLevel::None) {
       unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second;
@@ -9615,8 +9617,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                        0);
     }
 
-    // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
-    if (ST.hasSplitBarriers()) {
+    if (ST.hasSplitBarriers() && IntrinsicID == Intrinsic::amdgcn_s_barrier) {
+      // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
       SDValue K =
           DAG.getTargetConstant(AMDGPU::Barrier::WORKGROUP, DL, MVT::i32);
       SDValue BarSignal =
diff --git a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
index 0268f643b5533e..42983b335f497c 100644
--- a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
@@ -16,7 +16,9 @@ define amdgpu_kernel void @signal_flat_wgs_attr_32_128() #1 {
 }
 
 ; CHECK-LABEL: {{^}}signal_flat_wgs_attr_32_64:
-; CHECK: s_barrier_signal
+; CHECK: :
+; CHECK-NEXT: ; wave barrier
+; CHECK-NEXT: s_endpgm
 define amdgpu_kernel void @signal_flat_wgs_attr_32_64() #2 {
   tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
   ret void
@@ -39,7 +41,8 @@ define amdgpu_kernel void @wait_flat_wgs_attr_32_128() #1 {
 
 ; CHECK-LABEL: {{^}}wait_flat_wgs_attr_32_64:
 ; CHECK: :
-; CHECK-NEXT: s_barrier_wait
+; CHECK-NEXT: ; wave barrier
+; CHECK-NEXT: s_endpgm
 define amdgpu_kernel void @wait_flat_wgs_attr_32_64() #2 {
   tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
   ret void

>From 7174202f86920bcbc3b8b1e423c25e35f08483d0 Mon Sep 17 00:00:00 2001
From: Piotr Sobczak <piotr.sobczak at amd.com>
Date: Thu, 21 Nov 2024 10:34:46 +0100
Subject: [PATCH 3/4] Update test

---
 .../AMDGPU/barrier-elimination-gfx12.ll       | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
index 42983b335f497c..24ec9f1e14166b 100644
--- a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
@@ -1,26 +1,26 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s -global-isel | FileCheck %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}signal_unknown_wgs:
 ; CHECK: s_barrier_signal
 define amdgpu_kernel void @signal_unknown_wgs() {
-  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1)
   ret void
 }
 
 ; CHECK-LABEL: {{^}}signal_flat_wgs_attr_32_128:
 ; CHECK: s_barrier_signal
 define amdgpu_kernel void @signal_flat_wgs_attr_32_128() #1 {
-  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1)
   ret void
 }
 
-; CHECK-LABEL: {{^}}signal_flat_wgs_attr_32_64:
+; CHECK-LABEL: {{^}}signal_flat_wgs_attr_16_32:
 ; CHECK: :
 ; CHECK-NEXT: ; wave barrier
 ; CHECK-NEXT: s_endpgm
-define amdgpu_kernel void @signal_flat_wgs_attr_32_64() #2 {
-  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1) #0
+define amdgpu_kernel void @signal_flat_wgs_attr_16_32() #2 {
+  tail call void @llvm.amdgcn.s.barrier.signal(i32 -1)
   ret void
 }
 
@@ -28,23 +28,23 @@ define amdgpu_kernel void @signal_flat_wgs_attr_32_64() #2 {
 ; CHECK-LABEL: {{^}}wait_unknown_wgs:
 ; CHECK: s_barrier_wait
 define amdgpu_kernel void @wait_unknown_wgs() {
-  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1)
   ret void
 }
 
 ; CHECK-LABEL: {{^}}wait_flat_wgs_attr_32_128:
 ; CHECK: s_barrier_wait
 define amdgpu_kernel void @wait_flat_wgs_attr_32_128() #1 {
-  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1)
   ret void
 }
 
-; CHECK-LABEL: {{^}}wait_flat_wgs_attr_32_64:
+; CHECK-LABEL: {{^}}wait_flat_wgs_attr_16_32:
 ; CHECK: :
 ; CHECK-NEXT: ; wave barrier
 ; CHECK-NEXT: s_endpgm
-define amdgpu_kernel void @wait_flat_wgs_attr_32_64() #2 {
-  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1) #0
+define amdgpu_kernel void @wait_flat_wgs_attr_16_32() #2 {
+  tail call void @llvm.amdgcn.s.barrier.wait(i16 -1)
   ret void
 }
 

>From dc157f7ab8e12603b2716b6cf1f6397f19708d79 Mon Sep 17 00:00:00 2001
From: Piotr Sobczak <piotr.sobczak at amd.com>
Date: Thu, 21 Nov 2024 12:40:51 +0100
Subject: [PATCH 4/4] Do not generate wave_barrier for s_barrier_signal

---
 .../Target/AMDGPU/AMDGPUInstructionSelector.cpp    | 13 +++++++++----
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp          | 14 ++++++++++----
 .../CodeGen/AMDGPU/barrier-elimination-gfx12.ll    |  1 -
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index f01f57de460c75..8247c8581dde82 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1832,18 +1832,23 @@ bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
 }
 
 bool AMDGPUInstructionSelector::selectSBarrier(MachineInstr &MI) const {
+  Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
   if (TM.getOptLevel() > CodeGenOptLevel::None) {
     unsigned WGSize = STI.getFlatWorkGroupSizes(MF->getFunction()).second;
     if (WGSize <= STI.getWavefrontSize()) {
-      MachineBasicBlock *MBB = MI.getParent();
-      const DebugLoc &DL = MI.getDebugLoc();
-      BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER));
+      // If the workgroup fits in a wave, remove s_barrier_signal and lower
+      // s_barrier/s_barrier_wait to wave_barrier.
+      if (IntrinsicID == Intrinsic::amdgcn_s_barrier ||
+          IntrinsicID == Intrinsic::amdgcn_s_barrier_wait) {
+        MachineBasicBlock *MBB = MI.getParent();
+        const DebugLoc &DL = MI.getDebugLoc();
+        BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER));
+      }
       MI.eraseFromParent();
       return true;
     }
   }
 
-  Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
   if (STI.hasSplitBarriers() && IntrinsicID == Intrinsic::amdgcn_s_barrier) {
     // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
     MachineBasicBlock *MBB = MI.getParent();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4743ff0f65690e..05442230d52527 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -9611,10 +9611,16 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
     if (getTargetMachine().getOptLevel() > CodeGenOptLevel::None) {
       unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second;
-      if (WGSize <= ST.getWavefrontSize())
-        return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other,
-                                          Op.getOperand(0)),
-                       0);
+      if (WGSize <= ST.getWavefrontSize()) {
+        // If the workgroup fits in a wave, remove s_barrier_signal and lower
+        // s_barrier/s_barrier_wait to wave_barrier.
+        if (IntrinsicID == Intrinsic::amdgcn_s_barrier_signal)
+          return Op.getOperand(0);
+        else
+          return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL,
+                                            MVT::Other, Op.getOperand(0)),
+                         0);
+      }
     }
 
     if (ST.hasSplitBarriers() && IntrinsicID == Intrinsic::amdgcn_s_barrier) {
diff --git a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
index 24ec9f1e14166b..0b852350dad4be 100644
--- a/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/barrier-elimination-gfx12.ll
@@ -17,7 +17,6 @@ define amdgpu_kernel void @signal_flat_wgs_attr_32_128() #1 {
 
 ; CHECK-LABEL: {{^}}signal_flat_wgs_attr_16_32:
 ; CHECK: :
-; CHECK-NEXT: ; wave barrier
 ; CHECK-NEXT: s_endpgm
 define amdgpu_kernel void @signal_flat_wgs_attr_16_32() #2 {
   tail call void @llvm.amdgcn.s.barrier.signal(i32 -1)



More information about the llvm-commits mailing list