[llvm] [AMDGPU] Prevent GFX11 VALU Hazard Wait merging into terminators (PR #214935)

[Carl Ritson via llvm-commits]

https://github.com/perlfu created https://github.com/llvm/llvm-project/pull/214935

Fix an issue where a pending wait would be moved into the block terminators causing a validation error.
Flush all pending waits and exit optimization loop when reaching first terminator within a block.

>From f6ea6e98129baa449bd73efc53cd6517f9104d60 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Sat, 8 Aug 2026 14:07:09 +0900
Subject: [PATCH] [AMDGPU] Prevent GFX11 VALU Hazard Wait merging into
 terminators

Fix an issue where a pending wait would be moved into the block
terminators causing a validation error.
Flush all pending waits and exit optimization loop when reaching
first terminator within a block.
---
 .../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp   | 15 ++++--
 .../CodeGen/AMDGPU/valu-mask-write-hazard.mir | 51 +++++++++++++++++++
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index 51860f195cce7..279725e9388be 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -478,9 +478,6 @@ class AMDGPUWaitSGPRHazards {
       };
 
       for (MachineInstr &MI : MBB) {
-        if (MI.isMetaInstruction())
-          continue;
-
         if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
             (MI.getOperand(0).getImm() & ConstantMaskBits) ==
                 ConstantMaskBits) {
@@ -500,12 +497,20 @@ class AMDGPUWaitSGPRHazards {
           continue;
         }
 
-        // Do not optimize over branches
-        if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch())) {
+        // Do not optimize over branches or terminators
+        if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch() ||
+                         MI.isTerminator())) {
           PrevWait->moveBefore(&MI);
           PrevWait = nullptr;
           Changed = true;
         }
+        if (MI.isTerminator())
+          break;
+
+        // Note: test for meta instructions after terminators.
+        // Required to handle terminator meta instruction.
+        if (MI.isMetaInstruction())
+          continue;
 
         const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
         const bool IsSALU = SIInstrInfo::isSALU(MI);
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index d7034abfde61a..0b9754108463d 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -60,6 +60,7 @@
   define amdgpu_gs void @mask_hazard_optimize1() { ret void }
   define amdgpu_gs void @mask_hazard_optimize2() { ret void }
   define amdgpu_gs void @mask_hazard_optimize3() { ret void }
+  define amdgpu_gs void @mask_hazard_optimize_terminators() { ret void }
 ...
 
 ---
@@ -1268,3 +1269,53 @@ body:            |
     $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
     $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
 ...
+
+---
+name:            mask_hazard_optimize_terminators
+body:            |
+  ; GFX11-LABEL: name: mask_hazard_optimize_terminators
+  ; GFX11: bb.0:
+  ; GFX11-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT:   $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX11-NEXT:   V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+  ; GFX11-NEXT:   $sgpr4 = S_MOV_B32 $sgpr1
+  ; GFX11-NEXT:   $sgpr5 = S_MOV_B32 $sgpr2
+  ; GFX11-NEXT:   S_WAITCNT_DEPCTR .VaVcc_0
+  ; GFX11-NEXT:   $sgpr6 = S_MOV_B32_term $sgpr4
+  ; GFX11-NEXT:   $sgpr7 = S_MOV_B32_term $sgpr5
+  ; GFX11-NEXT:   S_BRANCH %bb.1
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT: bb.1:
+  ; GFX11-NEXT:   $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX11-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX12-LABEL: name: mask_hazard_optimize_terminators
+  ; GFX12: bb.0:
+  ; GFX12-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX12-NEXT: {{  $}}
+  ; GFX12-NEXT:   $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX12-NEXT:   V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+  ; GFX12-NEXT:   $sgpr4 = S_MOV_B32 $sgpr1
+  ; GFX12-NEXT:   $sgpr5 = S_MOV_B32 $sgpr2
+  ; GFX12-NEXT:   $sgpr6 = S_MOV_B32_term $sgpr4
+  ; GFX12-NEXT:   $sgpr7 = S_MOV_B32_term $sgpr5
+  ; GFX12-NEXT:   S_BRANCH %bb.1
+  ; GFX12-NEXT: {{  $}}
+  ; GFX12-NEXT: bb.1:
+  ; GFX12-NEXT:   S_WAITCNT_DEPCTR .VaVcc_0
+  ; GFX12-NEXT:   $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX12-NEXT:   S_ENDPGM 0
+  bb.0:
+    $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+    $sgpr4 = S_MOV_B32 $sgpr1
+    $sgpr5 = S_MOV_B32 $sgpr2
+    $sgpr6 = S_MOV_B32_term $sgpr4
+    $sgpr7 = S_MOV_B32_term $sgpr5
+    S_BRANCH %bb.1
+
+  bb.1:
+    $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    S_ENDPGM 0
+...