[llvm] [AMDGPU] Let LDSDMA ops cross a sched_barrier when the mask allows it (PR #203255)

[via llvm-commits]

llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-amdgpu

Author: Arseniy Obolenskiy (aobolensk)

<details>
<summary>Changes</summary>

A mask allowing LDSDMA (0x800) still kept LDSDMA ops pinned, because canAddMI matched them as VMEM and invertSchedBarrierMask left the DS bit set

---
Full diff: https://github.com/llvm/llvm-project/pull/203255.diff


3 Files Affected:

- (modified) llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp (+5-4) 
- (modified) llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir (+38) 
- (modified) llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll (+4-4) 


``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 6c95dc76f4f6f..04639f55e3935 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2472,11 +2472,11 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
     Result = true;
 
   else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) &&
-           MI.mayLoad() && TII->isVMEM(MI))
+           MI.mayLoad() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
     Result = true;
 
   else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) &&
-           MI.mayStore() && TII->isVMEM(MI))
+           MI.mayStore() && TII->isVMEM(MI) && !TII->isLDSDMA(MI))
     Result = true;
 
   else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) &&
@@ -2679,9 +2679,10 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
   if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
     InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE &
                     ~SchedGroupMask::LDSDMA;
-  // DS_READ, DS_WRITE implies DS.
+  // DS_READ, DS_WRITE, LDSDMA implies DS.
   else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
-           (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE)
+           (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE ||
+           (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE)
     InvertedMask &= ~SchedGroupMask::DS;
 
   LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask
diff --git a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
index 1ab3e349fc0a0..c1924706a0751 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir
@@ -10,6 +10,7 @@
   define amdgpu_kernel void @sched_group_barrier_mask_2048_async_loads() { ret void }
   define amdgpu_kernel void @sched_group_barrier_mask_16_async_loads() { ret void }
   define amdgpu_kernel void @sched_group_barrier_mask_2048_interleave_2load_1valu() { ret void }
+  define amdgpu_kernel void @sched_barrier_mask_2049_allows_ldsdma() { ret void }
 ...
 
 ---
@@ -340,3 +341,40 @@ body: |
     S_ENDPGM 0, implicit %30, implicit %31, implicit %32
 
 ...
+
+# A SCHED_BARRIER whose mask allows LDSDMA (bit 0x800) together with ALU
+# (bit 0x1) must let an LDSDMA op be scheduled across it. The second
+# async load is hoisted above the barrier.
+
+---
+name: sched_barrier_mask_2049_allows_ldsdma
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: sched_barrier_mask_2049_allows_ldsdma
+    ; CHECK: $exec = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF1]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+    ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF2]], [[DEF]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+    ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF3]], [[DEF4]], implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[DEF3]], implicit $exec
+    ; CHECK-NEXT: SCHED_BARRIER 2049
+    ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_1]]
+    $exec = IMPLICIT_DEF
+    %0:vreg_64_align2 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    %4:vgpr_32 = IMPLICIT_DEF
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 %1, %0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+    %10:vgpr_32 = V_ADD_U32_e32 %3, %4, implicit $exec
+    SCHED_BARRIER 2049
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 %2, %0, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt
+    %11:vgpr_32 = V_ADD_U32_e32 %10, %3, implicit $exec
+    S_ENDPGM 0, implicit %11
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
index ca802ac586668..71476ab29029b 100644
--- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
@@ -41,8 +41,8 @@ entry:
   ret void
 }
 
-; Inverted 1935: 011110001111
-; GCN:       After Inverting, SchedGroup Mask: 1935
+; Inverted 1807: 011100001111
+; GCN:       After Inverting, SchedGroup Mask: 1807
 define amdgpu_kernel void @invert16() #0 {
 entry:
   call void @llvm.amdgcn.sched.barrier(i32 16) #1
@@ -104,8 +104,8 @@ entry:
   ret void
 }
 
-; Inverted 2031: 011111101111
-; GCN:       After Inverting, SchedGroup Mask: 2031
+; Inverted 1903: 011101101111
+; GCN:       After Inverting, SchedGroup Mask: 1903
 define amdgpu_kernel void @invert2048() #0 {
 entry:
   call void @llvm.amdgcn.sched.barrier(i32 2048) #1

``````````

</details>


https://github.com/llvm/llvm-project/pull/203255