[llvm] [AMDGPU] Mark `SI_WATERFALL_LOOP` as divergent in `hasDivergentBranch` (PR #218969)

[Igor Wodiany via llvm-commits]

https://github.com/IgWod created https://github.com/llvm/llvm-project/pull/218969

This prevents incorrect sinking out of the waterfall loops.

Assisted-by: Claude Code

>From fce9aafb595672184977ce977dc34c443d2836ae Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Wed, 26 Aug 2026 16:21:22 +0100
Subject: [PATCH] [AMDGPU] Mark `SI_WATERFALL_LOOP` as divergent in
 `hasDivergentBranch`

This prevents incorrect sinking out of the waterfall loop.
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  3 +-
 ...e-sink-temporal-divergence-swdev407790.mir | 51 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index bc419d813f171..009c6a5595148 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -3002,7 +3002,8 @@ SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
 bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
   for (const MachineInstr &MI : MBB->terminators()) {
     if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
-        MI.getOpcode() == AMDGPU::SI_LOOP)
+        MI.getOpcode() == AMDGPU::SI_LOOP ||
+        MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
       return true;
   }
   return false;
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir
index 48789187ec337..c8afebf22e08d 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir
@@ -189,3 +189,54 @@ body: |
     FLAT_STORE_DWORD %3, %11, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32))
     SI_RETURN
 ...
+
+---
+name: machine-sink-temporal-divergence-waterfall-loop
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: machine-sink-temporal-divergence-waterfall-loop
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1_vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+  ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_READFIRSTLANE_B32_]], [[V_READFIRSTLANE_B32_]], 0, implicit $exec
+  ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+  ; CHECK-NEXT:   FLAT_STORE_DWORD [[COPY1]], [[V_ADD_U32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32))
+  ; CHECK-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vreg_64 = COPY $vgpr1_vgpr2
+    %2:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+
+  bb.1:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+
+    %3:sreg_32_xm0 = V_READFIRSTLANE_B32 %0, implicit $exec
+    %4:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 %3, %0, implicit $exec
+    %5:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed %4, implicit-def $exec, implicit-def $scc, implicit $exec
+    %6:vgpr_32 = V_ADD_U32_e64 %3, %3, 0, implicit $exec
+    $exec_lo = S_XOR_B32_term $exec_lo, %5, implicit-def $scc
+    SI_WATERFALL_LOOP %bb.1, implicit $exec
+
+  bb.2:
+    $exec_lo = S_MOV_B32 %2
+    FLAT_STORE_DWORD %1, %6, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32))
+    SI_RETURN
+...