[llvm] AMDGPU: Undo atomicrmw add/sub/xor 0 -> atomicrmw or canonicalization (PR #87533)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 12 13:53:55 PDT 2024
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/87533
>From 7364ea91efd7083d18e3414357a3a68f02988551 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 3 Apr 2024 13:03:15 -0400
Subject: [PATCH 1/2] AMDGPU: Undo atomicrmw add/sub/xor 0 -> atomicrmw or
canonicalization
InstCombine transforms add of 0 to or of 0. For system atomics,
this is problematic because while PCIe supports add, it does
not support the other operations. Undo this for system scope atomics.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 23 +
.../AMDGPU/global_atomics_i32_system.ll | 723 ++++++++++--------
...expand-atomicrmw-integer-ops-0-to-add-0.ll | 130 ++++
3 files changed, 570 insertions(+), 306 deletions(-)
create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 14948ef9ea8d17..6350a6598545a5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16027,6 +16027,19 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
switch (RMW->getOperation()) {
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Or:
+ case AtomicRMWInst::Xor: {
+ // Atomic sub/or/xor do not work over PCI express, but atomic add
+ // does. InstCombine transforms these with 0 to or, so undo that.
+ if (HasSystemScope && AMDGPU::isFlatGlobalAddrSpace(AS)) {
+ if (Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
+ ConstVal && ConstVal->isNullValue())
+ return AtomicExpansionKind::Expand;
+ }
+
+ break;
+ }
case AtomicRMWInst::FAdd: {
Type *Ty = RMW->getType();
@@ -16312,6 +16325,16 @@ bool SITargetLowering::checkForPhysRegDependency(
}
void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
+ if (AI->getOperation() == AtomicRMWInst::Sub ||
+ AI->getOperation() == AtomicRMWInst::Or ||
+ AI->getOperation() == AtomicRMWInst::Xor) {
+ // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
+ assert(cast<Constant>(AI->getValOperand())->isNullValue() &&
+ "this cannot be replaced with add");
+ AI->setOperation(AtomicRMWInst::Add);
+ return;
+ }
+
assert(Subtarget->hasAtomicFaddInsts() &&
"target should have atomic fadd instructions");
assert(AI->getType()->isFloatTy() &&
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index 99d02ffaa523d5..4598d23e088b54 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -1403,6 +1403,43 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1)
ret i32 %result
}
+define i32 @global_atomic_sub_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_sub_0_i32_ret:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s6
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_wbinvl1
+; SI-NEXT: v_mov_b32_e32 v0, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_sub_0_i32_ret:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_sub_0_i32_ret:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst
+ ret i32 %result
+}
+
; ---------------------------------------------------------------------
; atomicrmw and
; ---------------------------------------------------------------------
@@ -1767,7 +1804,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB40_1: ; %atomicrmw.start
+; SI-NEXT: .LBB41_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1782,7 +1819,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB40_1
+; SI-NEXT: s_cbranch_execnz .LBB41_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -1793,7 +1830,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB40_1: ; %atomicrmw.start
+; VI-NEXT: .LBB41_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1805,7 +1842,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB40_1
+; VI-NEXT: s_cbranch_execnz .LBB41_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1815,7 +1852,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1827,7 +1864,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB40_1
+; GFX9-NEXT: s_cbranch_execnz .LBB41_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1845,7 +1882,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB41_1: ; %atomicrmw.start
+; SI-NEXT: .LBB42_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1860,7 +1897,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB41_1
+; SI-NEXT: s_cbranch_execnz .LBB42_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -1873,7 +1910,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB41_1: ; %atomicrmw.start
+; VI-NEXT: .LBB42_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1885,7 +1922,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB41_1
+; VI-NEXT: s_cbranch_execnz .LBB42_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1895,7 +1932,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB41_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v3, v4, v2
@@ -1907,7 +1944,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB41_1
+; GFX9-NEXT: s_cbranch_execnz .LBB42_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1926,7 +1963,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB42_1: ; %atomicrmw.start
+; SI-NEXT: .LBB43_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -1941,7 +1978,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB42_1
+; SI-NEXT: s_cbranch_execnz .LBB43_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -1953,7 +1990,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB42_1: ; %atomicrmw.start
+; VI-NEXT: .LBB43_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
@@ -1965,7 +2002,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB42_1
+; VI-NEXT: s_cbranch_execnz .LBB43_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
@@ -1976,7 +2013,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB42_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -1988,7 +2025,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB42_1
+; GFX9-NEXT: s_cbranch_execnz .LBB43_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -2007,7 +2044,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB43_1: ; %atomicrmw.start
+; SI-NEXT: .LBB44_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -2022,7 +2059,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB43_1
+; SI-NEXT: s_cbranch_execnz .LBB44_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -2036,7 +2073,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v0, v[3:4]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB43_1: ; %atomicrmw.start
+; VI-NEXT: .LBB44_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, v0
@@ -2048,7 +2085,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB43_1
+; VI-NEXT: s_cbranch_execnz .LBB44_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2058,7 +2095,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -2070,7 +2107,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB43_1
+; GFX9-NEXT: s_cbranch_execnz .LBB44_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -2095,7 +2132,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB44_1: ; %atomicrmw.start
+; SI-NEXT: .LBB45_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, s34, v1
@@ -2110,7 +2147,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB44_1
+; SI-NEXT: s_cbranch_execnz .LBB45_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -2128,7 +2165,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB44_1: ; %atomicrmw.start
+; VI-NEXT: .LBB45_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v2, s6, v3
@@ -2140,7 +2177,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB44_1
+; VI-NEXT: s_cbranch_execnz .LBB45_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2151,7 +2188,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, s6, v1
@@ -2163,7 +2200,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB44_1
+; GFX9-NEXT: s_cbranch_execnz .LBB45_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2186,7 +2223,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB45_1: ; %atomicrmw.start
+; SI-NEXT: .LBB46_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, s34, v1
@@ -2201,7 +2238,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB45_1
+; SI-NEXT: s_cbranch_execnz .LBB46_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -2221,7 +2258,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: v_mov_b32_e32 v1, s35
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB45_1: ; %atomicrmw.start
+; VI-NEXT: .LBB46_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v2, s6, v3
@@ -2233,7 +2270,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB45_1
+; VI-NEXT: s_cbranch_execnz .LBB46_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2244,7 +2281,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB45_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, s6, v1
@@ -2256,7 +2293,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB45_1
+; GFX9-NEXT: s_cbranch_execnz .LBB46_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2280,7 +2317,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB46_1: ; %atomicrmw.start
+; SI-NEXT: .LBB47_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -2295,7 +2332,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB46_1
+; SI-NEXT: s_cbranch_execnz .LBB47_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -2315,7 +2352,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB46_1: ; %atomicrmw.start
+; VI-NEXT: .LBB47_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -2327,7 +2364,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB46_1
+; VI-NEXT: s_cbranch_execnz .LBB47_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2338,7 +2375,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -2350,7 +2387,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB46_1
+; GFX9-NEXT: s_cbranch_execnz .LBB47_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2373,7 +2410,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB47_1: ; %atomicrmw.start
+; SI-NEXT: .LBB48_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -2388,7 +2425,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB47_1
+; SI-NEXT: s_cbranch_execnz .LBB48_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -2408,7 +2445,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: flat_load_dword v0, v[1:2]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB47_1: ; %atomicrmw.start
+; VI-NEXT: .LBB48_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -2420,7 +2457,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB47_1
+; VI-NEXT: s_cbranch_execnz .LBB48_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2431,7 +2468,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -2443,7 +2480,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB47_1
+; GFX9-NEXT: s_cbranch_execnz .LBB48_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2802,6 +2839,43 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) i
ret i32 %result
}
+define i32 @global_atomic_or_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_or_0_i32_ret:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s6
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_wbinvl1
+; SI-NEXT: v_mov_b32_e32 v0, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_or_0_i32_ret:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_or_0_i32_ret:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst
+ ret i32 %result
+}
+
; ---------------------------------------------------------------------
; atomicrmw xor
; ---------------------------------------------------------------------
@@ -3152,6 +3226,43 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1)
ret i32 %result
}
+define i32 @global_atomic_xor_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_xor_0_i32_ret:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s6
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_wbinvl1
+; SI-NEXT: v_mov_b32_e32 v0, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_xor_0_i32_ret:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_wbinvl1_vol
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_xor_0_i32_ret:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst
+ ret i32 %result
+}
+
; ---------------------------------------------------------------------
; atomicrmw max
; ---------------------------------------------------------------------
@@ -3166,7 +3277,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB64_1: ; %atomicrmw.start
+; SI-NEXT: .LBB67_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3180,7 +3291,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB64_1
+; SI-NEXT: s_cbranch_execnz .LBB67_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -3191,7 +3302,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB64_1: ; %atomicrmw.start
+; VI-NEXT: .LBB67_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3202,7 +3313,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB64_1
+; VI-NEXT: s_cbranch_execnz .LBB67_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3212,7 +3323,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB64_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3223,7 +3334,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB64_1
+; GFX9-NEXT: s_cbranch_execnz .LBB67_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3241,7 +3352,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB65_1: ; %atomicrmw.start
+; SI-NEXT: .LBB68_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3255,7 +3366,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB65_1
+; SI-NEXT: s_cbranch_execnz .LBB68_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -3268,7 +3379,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB65_1: ; %atomicrmw.start
+; VI-NEXT: .LBB68_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3279,7 +3390,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB65_1
+; VI-NEXT: s_cbranch_execnz .LBB68_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3289,7 +3400,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB65_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i32_e32 v3, v4, v2
@@ -3300,7 +3411,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB65_1
+; GFX9-NEXT: s_cbranch_execnz .LBB68_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3319,7 +3430,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB66_1: ; %atomicrmw.start
+; SI-NEXT: .LBB69_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -3333,7 +3444,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB66_1
+; SI-NEXT: s_cbranch_execnz .LBB69_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -3345,7 +3456,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB66_1: ; %atomicrmw.start
+; VI-NEXT: .LBB69_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
@@ -3356,7 +3467,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB66_1
+; VI-NEXT: s_cbranch_execnz .LBB69_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
@@ -3367,7 +3478,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB66_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -3378,7 +3489,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB66_1
+; GFX9-NEXT: s_cbranch_execnz .LBB69_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -3397,7 +3508,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB67_1: ; %atomicrmw.start
+; SI-NEXT: .LBB70_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -3411,7 +3522,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB67_1
+; SI-NEXT: s_cbranch_execnz .LBB70_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -3425,7 +3536,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v0, v[3:4]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB67_1: ; %atomicrmw.start
+; VI-NEXT: .LBB70_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, v0
@@ -3436,7 +3547,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB67_1
+; VI-NEXT: s_cbranch_execnz .LBB70_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3446,7 +3557,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB67_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -3457,7 +3568,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB67_1
+; GFX9-NEXT: s_cbranch_execnz .LBB70_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -3482,7 +3593,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB68_1: ; %atomicrmw.start
+; SI-NEXT: .LBB71_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_i32_e32 v0, s34, v1
@@ -3496,7 +3607,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB68_1
+; SI-NEXT: s_cbranch_execnz .LBB71_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -3514,7 +3625,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB68_1: ; %atomicrmw.start
+; VI-NEXT: .LBB71_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_i32_e32 v2, s6, v3
@@ -3525,7 +3636,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB68_1
+; VI-NEXT: s_cbranch_execnz .LBB71_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3536,7 +3647,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB68_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i32_e32 v0, s6, v1
@@ -3547,7 +3658,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB68_1
+; GFX9-NEXT: s_cbranch_execnz .LBB71_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3570,7 +3681,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB69_1: ; %atomicrmw.start
+; SI-NEXT: .LBB72_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_i32_e32 v0, s34, v1
@@ -3584,7 +3695,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB69_1
+; SI-NEXT: s_cbranch_execnz .LBB72_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -3604,7 +3715,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v1, s35
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB69_1: ; %atomicrmw.start
+; VI-NEXT: .LBB72_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_i32_e32 v2, s6, v3
@@ -3615,7 +3726,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB69_1
+; VI-NEXT: s_cbranch_execnz .LBB72_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3626,7 +3737,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB69_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i32_e32 v0, s6, v1
@@ -3637,7 +3748,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB69_1
+; GFX9-NEXT: s_cbranch_execnz .LBB72_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3661,7 +3772,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB70_1: ; %atomicrmw.start
+; SI-NEXT: .LBB73_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -3675,7 +3786,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB70_1
+; SI-NEXT: s_cbranch_execnz .LBB73_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -3695,7 +3806,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB70_1: ; %atomicrmw.start
+; VI-NEXT: .LBB73_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -3706,7 +3817,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB70_1
+; VI-NEXT: s_cbranch_execnz .LBB73_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3717,7 +3828,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB70_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -3728,7 +3839,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB70_1
+; GFX9-NEXT: s_cbranch_execnz .LBB73_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3751,7 +3862,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB71_1: ; %atomicrmw.start
+; SI-NEXT: .LBB74_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -3765,7 +3876,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB71_1
+; SI-NEXT: s_cbranch_execnz .LBB74_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -3785,7 +3896,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: flat_load_dword v0, v[1:2]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB71_1: ; %atomicrmw.start
+; VI-NEXT: .LBB74_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -3796,7 +3907,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB71_1
+; VI-NEXT: s_cbranch_execnz .LBB74_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3807,7 +3918,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB71_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -3818,7 +3929,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB71_1
+; GFX9-NEXT: s_cbranch_execnz .LBB74_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3843,7 +3954,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB72_1: ; %atomicrmw.start
+; SI-NEXT: .LBB75_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_i32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -3856,7 +3967,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB72_1
+; SI-NEXT: s_cbranch_execnz .LBB75_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
@@ -3877,7 +3988,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: .LBB72_1: ; %atomicrmw.start
+; VI-NEXT: .LBB75_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_max_i32_e32 v2, s2, v3
; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -3887,7 +3998,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB72_1
+; VI-NEXT: s_cbranch_execnz .LBB75_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
@@ -3905,7 +4016,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: .LBB72_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_i32_e32 v0, s2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -3915,7 +4026,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB72_1
+; GFX9-NEXT: s_cbranch_execnz .LBB75_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
entry:
@@ -3942,7 +4053,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB73_1: ; %atomicrmw.start
+; SI-NEXT: .LBB76_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_i32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -3955,7 +4066,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB73_1
+; SI-NEXT: s_cbranch_execnz .LBB76_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -3983,7 +4094,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_mov_b32_e32 v1, s7
-; VI-NEXT: .LBB73_1: ; %atomicrmw.start
+; VI-NEXT: .LBB76_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_max_i32_e32 v2, s4, v3
@@ -3993,7 +4104,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB73_1
+; VI-NEXT: s_cbranch_execnz .LBB76_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -4016,7 +4127,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB73_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_max_i32_e32 v2, s2, v3
@@ -4026,7 +4137,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB73_1
+; GFX9-NEXT: s_cbranch_execnz .LBB76_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -4056,7 +4167,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB74_1: ; %atomicrmw.start
+; SI-NEXT: .LBB77_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_i32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -4069,7 +4180,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB74_1
+; SI-NEXT: s_cbranch_execnz .LBB77_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
@@ -4088,7 +4199,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: .LBB74_1: ; %atomicrmw.start
+; VI-NEXT: .LBB77_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_max_i32_e32 v2, s2, v3
; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -4098,7 +4209,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB74_1
+; VI-NEXT: s_cbranch_execnz .LBB77_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
@@ -4116,7 +4227,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_i32_e32 v0, s2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
@@ -4126,7 +4237,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB74_1
+; GFX9-NEXT: s_cbranch_execnz .LBB77_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
entry:
@@ -4152,7 +4263,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB75_1: ; %atomicrmw.start
+; SI-NEXT: .LBB78_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_i32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -4165,7 +4276,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB75_1
+; SI-NEXT: s_cbranch_execnz .LBB78_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -4191,7 +4302,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB75_1: ; %atomicrmw.start
+; VI-NEXT: .LBB78_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_max_i32_e32 v2, s4, v3
@@ -4201,7 +4312,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB75_1
+; VI-NEXT: s_cbranch_execnz .LBB78_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -4224,7 +4335,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_max_i32_e32 v2, s2, v3
@@ -4234,7 +4345,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB75_1
+; GFX9-NEXT: s_cbranch_execnz .LBB78_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -4261,7 +4372,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB76_1: ; %atomicrmw.start
+; SI-NEXT: .LBB79_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4275,7 +4386,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB76_1
+; SI-NEXT: s_cbranch_execnz .LBB79_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -4286,7 +4397,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB76_1: ; %atomicrmw.start
+; VI-NEXT: .LBB79_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4297,7 +4408,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB76_1
+; VI-NEXT: s_cbranch_execnz .LBB79_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4307,7 +4418,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB76_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4318,7 +4429,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB76_1
+; GFX9-NEXT: s_cbranch_execnz .LBB79_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4336,7 +4447,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB77_1: ; %atomicrmw.start
+; SI-NEXT: .LBB80_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4350,7 +4461,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB77_1
+; SI-NEXT: s_cbranch_execnz .LBB80_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -4363,7 +4474,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB77_1: ; %atomicrmw.start
+; VI-NEXT: .LBB80_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4374,7 +4485,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB77_1
+; VI-NEXT: s_cbranch_execnz .LBB80_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4384,7 +4495,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB77_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u32_e32 v3, v4, v2
@@ -4395,7 +4506,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB77_1
+; GFX9-NEXT: s_cbranch_execnz .LBB80_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4414,7 +4525,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB78_1: ; %atomicrmw.start
+; SI-NEXT: .LBB81_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -4428,7 +4539,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB78_1
+; SI-NEXT: s_cbranch_execnz .LBB81_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -4440,7 +4551,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB78_1: ; %atomicrmw.start
+; VI-NEXT: .LBB81_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
@@ -4451,7 +4562,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB78_1
+; VI-NEXT: s_cbranch_execnz .LBB81_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
@@ -4462,7 +4573,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB78_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -4473,7 +4584,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB78_1
+; GFX9-NEXT: s_cbranch_execnz .LBB81_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -4492,7 +4603,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB79_1: ; %atomicrmw.start
+; SI-NEXT: .LBB82_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -4506,7 +4617,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB79_1
+; SI-NEXT: s_cbranch_execnz .LBB82_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -4520,7 +4631,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v0, v[3:4]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB79_1: ; %atomicrmw.start
+; VI-NEXT: .LBB82_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, v0
@@ -4531,7 +4642,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB79_1
+; VI-NEXT: s_cbranch_execnz .LBB82_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4541,7 +4652,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB79_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -4552,7 +4663,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB79_1
+; GFX9-NEXT: s_cbranch_execnz .LBB82_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -4577,7 +4688,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB80_1: ; %atomicrmw.start
+; SI-NEXT: .LBB83_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_u32_e32 v0, s34, v1
@@ -4591,7 +4702,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB80_1
+; SI-NEXT: s_cbranch_execnz .LBB83_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -4609,7 +4720,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB80_1: ; %atomicrmw.start
+; VI-NEXT: .LBB83_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_u32_e32 v2, s6, v3
@@ -4620,7 +4731,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB80_1
+; VI-NEXT: s_cbranch_execnz .LBB83_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4631,7 +4742,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u32_e32 v0, s6, v1
@@ -4642,7 +4753,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB80_1
+; GFX9-NEXT: s_cbranch_execnz .LBB83_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4665,7 +4776,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB81_1: ; %atomicrmw.start
+; SI-NEXT: .LBB84_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_max_u32_e32 v0, s34, v1
@@ -4679,7 +4790,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB81_1
+; SI-NEXT: s_cbranch_execnz .LBB84_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -4699,7 +4810,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: v_mov_b32_e32 v1, s35
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB81_1: ; %atomicrmw.start
+; VI-NEXT: .LBB84_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_max_u32_e32 v2, s6, v3
@@ -4710,7 +4821,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB81_1
+; VI-NEXT: s_cbranch_execnz .LBB84_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4721,7 +4832,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u32_e32 v0, s6, v1
@@ -4732,7 +4843,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB81_1
+; GFX9-NEXT: s_cbranch_execnz .LBB84_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4756,7 +4867,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB82_1: ; %atomicrmw.start
+; SI-NEXT: .LBB85_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -4770,7 +4881,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB82_1
+; SI-NEXT: s_cbranch_execnz .LBB85_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -4790,7 +4901,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB82_1: ; %atomicrmw.start
+; VI-NEXT: .LBB85_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -4801,7 +4912,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB82_1
+; VI-NEXT: s_cbranch_execnz .LBB85_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4812,7 +4923,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -4823,7 +4934,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB82_1
+; GFX9-NEXT: s_cbranch_execnz .LBB85_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4846,7 +4957,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB83_1: ; %atomicrmw.start
+; SI-NEXT: .LBB86_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -4860,7 +4971,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB83_1
+; SI-NEXT: s_cbranch_execnz .LBB86_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -4880,7 +4991,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: flat_load_dword v0, v[1:2]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB83_1: ; %atomicrmw.start
+; VI-NEXT: .LBB86_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -4891,7 +5002,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB83_1
+; VI-NEXT: s_cbranch_execnz .LBB86_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4902,7 +5013,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -4913,7 +5024,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB83_1
+; GFX9-NEXT: s_cbranch_execnz .LBB86_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4938,7 +5049,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB84_1: ; %atomicrmw.start
+; SI-NEXT: .LBB87_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_u32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -4951,7 +5062,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB84_1
+; SI-NEXT: s_cbranch_execnz .LBB87_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
@@ -4972,7 +5083,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: .LBB84_1: ; %atomicrmw.start
+; VI-NEXT: .LBB87_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_max_u32_e32 v2, s2, v3
; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -4982,7 +5093,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB84_1
+; VI-NEXT: s_cbranch_execnz .LBB87_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
@@ -5000,7 +5111,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_u32_e32 v0, s2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -5010,7 +5121,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB84_1
+; GFX9-NEXT: s_cbranch_execnz .LBB87_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
entry:
@@ -5037,7 +5148,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB85_1: ; %atomicrmw.start
+; SI-NEXT: .LBB88_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_u32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -5050,7 +5161,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB85_1
+; SI-NEXT: s_cbranch_execnz .LBB88_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -5078,7 +5189,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_mov_b32_e32 v1, s7
-; VI-NEXT: .LBB85_1: ; %atomicrmw.start
+; VI-NEXT: .LBB88_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_max_u32_e32 v2, s4, v3
@@ -5088,7 +5199,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB85_1
+; VI-NEXT: s_cbranch_execnz .LBB88_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -5111,7 +5222,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_max_u32_e32 v2, s2, v3
@@ -5121,7 +5232,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB85_1
+; GFX9-NEXT: s_cbranch_execnz .LBB88_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -5152,7 +5263,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB86_1: ; %atomicrmw.start
+; SI-NEXT: .LBB89_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_max_u32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -5165,7 +5276,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB86_1
+; SI-NEXT: s_cbranch_execnz .LBB89_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -5191,7 +5302,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB86_1: ; %atomicrmw.start
+; VI-NEXT: .LBB89_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_max_u32_e32 v2, s4, v3
@@ -5201,7 +5312,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB86_1
+; VI-NEXT: s_cbranch_execnz .LBB89_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -5224,7 +5335,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_max_u32_e32 v2, s2, v3
@@ -5234,7 +5345,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB86_1
+; GFX9-NEXT: s_cbranch_execnz .LBB89_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -5261,7 +5372,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB87_1: ; %atomicrmw.start
+; SI-NEXT: .LBB90_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5275,7 +5386,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB87_1
+; SI-NEXT: s_cbranch_execnz .LBB90_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -5286,7 +5397,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB87_1: ; %atomicrmw.start
+; VI-NEXT: .LBB90_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5297,7 +5408,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB87_1
+; VI-NEXT: s_cbranch_execnz .LBB90_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5307,7 +5418,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5318,7 +5429,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB87_1
+; GFX9-NEXT: s_cbranch_execnz .LBB90_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5336,7 +5447,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB88_1: ; %atomicrmw.start
+; SI-NEXT: .LBB91_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5350,7 +5461,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB88_1
+; SI-NEXT: s_cbranch_execnz .LBB91_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -5363,7 +5474,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB88_1: ; %atomicrmw.start
+; VI-NEXT: .LBB91_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5374,7 +5485,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB88_1
+; VI-NEXT: s_cbranch_execnz .LBB91_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5384,7 +5495,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_u32_e32 v3, v4, v2
@@ -5395,7 +5506,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB88_1
+; GFX9-NEXT: s_cbranch_execnz .LBB91_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5414,7 +5525,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB89_1: ; %atomicrmw.start
+; SI-NEXT: .LBB92_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -5428,7 +5539,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB89_1
+; SI-NEXT: s_cbranch_execnz .LBB92_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -5440,7 +5551,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB89_1: ; %atomicrmw.start
+; VI-NEXT: .LBB92_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
@@ -5451,7 +5562,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB89_1
+; VI-NEXT: s_cbranch_execnz .LBB92_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
@@ -5462,7 +5573,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB92_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -5473,7 +5584,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB89_1
+; GFX9-NEXT: s_cbranch_execnz .LBB92_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -5492,7 +5603,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB90_1: ; %atomicrmw.start
+; SI-NEXT: .LBB93_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -5506,7 +5617,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB90_1
+; SI-NEXT: s_cbranch_execnz .LBB93_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -5520,7 +5631,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v0, v[3:4]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB90_1: ; %atomicrmw.start
+; VI-NEXT: .LBB93_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, v0
@@ -5531,7 +5642,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB90_1
+; VI-NEXT: s_cbranch_execnz .LBB93_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5541,7 +5652,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB93_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -5552,7 +5663,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB90_1
+; GFX9-NEXT: s_cbranch_execnz .LBB93_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -5577,7 +5688,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB91_1: ; %atomicrmw.start
+; SI-NEXT: .LBB94_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_u32_e32 v0, s34, v1
@@ -5591,7 +5702,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB91_1
+; SI-NEXT: s_cbranch_execnz .LBB94_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -5609,7 +5720,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB91_1: ; %atomicrmw.start
+; VI-NEXT: .LBB94_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_u32_e32 v2, s6, v3
@@ -5620,7 +5731,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB91_1
+; VI-NEXT: s_cbranch_execnz .LBB94_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5631,7 +5742,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_u32_e32 v0, s6, v1
@@ -5642,7 +5753,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB91_1
+; GFX9-NEXT: s_cbranch_execnz .LBB94_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5665,7 +5776,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB92_1: ; %atomicrmw.start
+; SI-NEXT: .LBB95_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_u32_e32 v0, s34, v1
@@ -5679,7 +5790,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB92_1
+; SI-NEXT: s_cbranch_execnz .LBB95_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -5699,7 +5810,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: v_mov_b32_e32 v1, s35
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB92_1: ; %atomicrmw.start
+; VI-NEXT: .LBB95_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_u32_e32 v2, s6, v3
@@ -5710,7 +5821,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB92_1
+; VI-NEXT: s_cbranch_execnz .LBB95_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5721,7 +5832,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB92_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB95_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_u32_e32 v0, s6, v1
@@ -5732,7 +5843,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB92_1
+; GFX9-NEXT: s_cbranch_execnz .LBB95_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5756,7 +5867,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB93_1: ; %atomicrmw.start
+; SI-NEXT: .LBB96_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -5770,7 +5881,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB93_1
+; SI-NEXT: s_cbranch_execnz .LBB96_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -5790,7 +5901,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB93_1: ; %atomicrmw.start
+; VI-NEXT: .LBB96_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -5801,7 +5912,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB93_1
+; VI-NEXT: s_cbranch_execnz .LBB96_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5812,7 +5923,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB93_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB96_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -5823,7 +5934,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB93_1
+; GFX9-NEXT: s_cbranch_execnz .LBB96_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5846,7 +5957,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB94_1: ; %atomicrmw.start
+; SI-NEXT: .LBB97_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -5860,7 +5971,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB94_1
+; SI-NEXT: s_cbranch_execnz .LBB97_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -5880,7 +5991,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: flat_load_dword v0, v[1:2]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB94_1: ; %atomicrmw.start
+; VI-NEXT: .LBB97_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -5891,7 +6002,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB94_1
+; VI-NEXT: s_cbranch_execnz .LBB97_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5902,7 +6013,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB97_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -5913,7 +6024,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB94_1
+; GFX9-NEXT: s_cbranch_execnz .LBB97_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5936,7 +6047,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB95_1: ; %atomicrmw.start
+; SI-NEXT: .LBB98_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_i32_e32 v3, v4, v2
@@ -5950,7 +6061,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB95_1
+; SI-NEXT: s_cbranch_execnz .LBB98_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -5961,7 +6072,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB95_1: ; %atomicrmw.start
+; VI-NEXT: .LBB98_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_i32_e32 v3, v4, v2
@@ -5972,7 +6083,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB95_1
+; VI-NEXT: s_cbranch_execnz .LBB98_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5982,7 +6093,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB95_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_i32_e32 v3, v4, v2
@@ -5993,7 +6104,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB95_1
+; GFX9-NEXT: s_cbranch_execnz .LBB98_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6011,7 +6122,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB96_1: ; %atomicrmw.start
+; SI-NEXT: .LBB99_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_i32_e32 v3, v4, v2
@@ -6025,7 +6136,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, v5
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB96_1
+; SI-NEXT: s_cbranch_execnz .LBB99_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: s_waitcnt expcnt(0)
@@ -6038,7 +6149,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v4, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB96_1: ; %atomicrmw.start
+; VI-NEXT: .LBB99_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_i32_e32 v3, v4, v2
@@ -6049,7 +6160,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB96_1
+; VI-NEXT: s_cbranch_execnz .LBB99_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6059,7 +6170,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB96_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_i32_e32 v3, v4, v2
@@ -6070,7 +6181,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB96_1
+; GFX9-NEXT: s_cbranch_execnz .LBB99_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6089,7 +6200,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB97_1: ; %atomicrmw.start
+; SI-NEXT: .LBB100_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -6103,7 +6214,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB97_1
+; SI-NEXT: s_cbranch_execnz .LBB100_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -6115,7 +6226,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB97_1: ; %atomicrmw.start
+; VI-NEXT: .LBB100_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
@@ -6126,7 +6237,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB97_1
+; VI-NEXT: s_cbranch_execnz .LBB100_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
@@ -6137,7 +6248,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB97_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -6148,7 +6259,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB97_1
+; GFX9-NEXT: s_cbranch_execnz .LBB100_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -6167,7 +6278,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_mov_b32 s5, s6
; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: .LBB98_1: ; %atomicrmw.start
+; SI-NEXT: .LBB101_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, v3
@@ -6181,7 +6292,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT: s_cbranch_execnz .LBB98_1
+; SI-NEXT: s_cbranch_execnz .LBB101_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, v3
@@ -6195,7 +6306,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v0, v[3:4]
; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: .LBB98_1: ; %atomicrmw.start
+; VI-NEXT: .LBB101_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, v0
@@ -6206,7 +6317,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT: s_cbranch_execnz .LBB98_1
+; VI-NEXT: s_cbranch_execnz .LBB101_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6216,7 +6327,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[0:1], off offset:16
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v3
@@ -6227,7 +6338,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB98_1
+; GFX9-NEXT: s_cbranch_execnz .LBB101_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
@@ -6252,7 +6363,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB99_1: ; %atomicrmw.start
+; SI-NEXT: .LBB102_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_i32_e32 v0, s34, v1
@@ -6266,7 +6377,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB99_1
+; SI-NEXT: s_cbranch_execnz .LBB102_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -6284,7 +6395,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB99_1: ; %atomicrmw.start
+; VI-NEXT: .LBB102_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_i32_e32 v2, s6, v3
@@ -6295,7 +6406,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB99_1
+; VI-NEXT: s_cbranch_execnz .LBB102_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6306,7 +6417,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_i32_e32 v0, s6, v1
@@ -6317,7 +6428,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB99_1
+; GFX9-NEXT: s_cbranch_execnz .LBB102_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6340,7 +6451,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB100_1: ; %atomicrmw.start
+; SI-NEXT: .LBB103_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_min_i32_e32 v0, s34, v1
@@ -6354,7 +6465,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB100_1
+; SI-NEXT: s_cbranch_execnz .LBB103_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v4, 1
@@ -6374,7 +6485,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v1, s35
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB100_1: ; %atomicrmw.start
+; VI-NEXT: .LBB103_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_min_i32_e32 v2, s6, v3
@@ -6385,7 +6496,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB100_1
+; VI-NEXT: s_cbranch_execnz .LBB103_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6396,7 +6507,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_min_i32_e32 v0, s6, v1
@@ -6407,7 +6518,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB100_1
+; GFX9-NEXT: s_cbranch_execnz .LBB103_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6431,7 +6542,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB101_1: ; %atomicrmw.start
+; SI-NEXT: .LBB104_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -6445,7 +6556,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB101_1
+; SI-NEXT: s_cbranch_execnz .LBB104_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -6465,7 +6576,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB101_1: ; %atomicrmw.start
+; VI-NEXT: .LBB104_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -6476,7 +6587,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB101_1
+; VI-NEXT: s_cbranch_execnz .LBB104_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6487,7 +6598,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -6498,7 +6609,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB101_1
+; GFX9-NEXT: s_cbranch_execnz .LBB104_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6521,7 +6632,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[36:37], 0
-; SI-NEXT: .LBB102_1: ; %atomicrmw.start
+; SI-NEXT: .LBB105_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, v0
@@ -6535,7 +6646,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT: s_cbranch_execnz .LBB102_1
+; SI-NEXT: s_cbranch_execnz .LBB105_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
; SI-NEXT: v_readlane_b32 s7, v3, 1
@@ -6555,7 +6666,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: flat_load_dword v0, v[1:2]
; VI-NEXT: s_mov_b64 s[34:35], 0
-; VI-NEXT: .LBB102_1: ; %atomicrmw.start
+; VI-NEXT: .LBB105_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v0
@@ -6566,7 +6677,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT: s_cbranch_execnz .LBB102_1
+; VI-NEXT: s_cbranch_execnz .LBB105_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6577,7 +6688,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[34:35], 0
-; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB105_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
@@ -6588,7 +6699,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT: s_cbranch_execnz .LBB102_1
+; GFX9-NEXT: s_cbranch_execnz .LBB105_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6613,7 +6724,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB103_1: ; %atomicrmw.start
+; SI-NEXT: .LBB106_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_min_i32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -6626,7 +6737,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB103_1
+; SI-NEXT: s_cbranch_execnz .LBB106_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
@@ -6647,7 +6758,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: .LBB103_1: ; %atomicrmw.start
+; VI-NEXT: .LBB106_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_min_i32_e32 v2, s2, v3
; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -6657,7 +6768,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB103_1
+; VI-NEXT: s_cbranch_execnz .LBB106_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
@@ -6675,7 +6786,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB106_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_min_i32_e32 v0, s2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -6685,7 +6796,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB103_1
+; GFX9-NEXT: s_cbranch_execnz .LBB106_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
entry:
@@ -6712,7 +6823,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB104_1: ; %atomicrmw.start
+; SI-NEXT: .LBB107_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_min_i32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -6725,7 +6836,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB104_1
+; SI-NEXT: s_cbranch_execnz .LBB107_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -6753,7 +6864,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_mov_b32_e32 v1, s7
-; VI-NEXT: .LBB104_1: ; %atomicrmw.start
+; VI-NEXT: .LBB107_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_min_i32_e32 v2, s4, v3
@@ -6763,7 +6874,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB104_1
+; VI-NEXT: s_cbranch_execnz .LBB107_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -6786,7 +6897,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB107_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_min_i32_e32 v2, s2, v3
@@ -6796,7 +6907,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB104_1
+; GFX9-NEXT: s_cbranch_execnz .LBB107_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -6822,7 +6933,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB105_1: ; %atomicrmw.start
+; SI-NEXT: .LBB108_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_min_i32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -6835,7 +6946,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB105_1
+; SI-NEXT: s_cbranch_execnz .LBB108_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
@@ -6850,7 +6961,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: .LBB105_1: ; %atomicrmw.start
+; VI-NEXT: .LBB108_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_min_i32_e32 v2, s2, v3
; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -6860,7 +6971,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB105_1
+; VI-NEXT: s_cbranch_execnz .LBB108_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
@@ -6874,7 +6985,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: .LBB105_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB108_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_min_i32_e32 v0, s4, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
@@ -6884,7 +6995,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX9-NEXT: s_cbranch_execnz .LBB105_1
+; GFX9-NEXT: s_cbranch_execnz .LBB108_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
entry:
@@ -6909,7 +7020,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: .LBB106_1: ; %atomicrmw.start
+; SI-NEXT: .LBB109_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_min_i32_e32 v0, s8, v1
; SI-NEXT: s_waitcnt expcnt(0)
@@ -6922,7 +7033,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v2
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT: s_cbranch_execnz .LBB106_1
+; SI-NEXT: s_cbranch_execnz .LBB109_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
; SI-NEXT: s_mov_b32 s7, 0xf000
@@ -6948,7 +7059,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: .LBB106_1: ; %atomicrmw.start
+; VI-NEXT: .LBB109_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_min_i32_e32 v2, s4, v3
@@ -6958,7 +7069,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT: s_cbranch_execnz .LBB106_1
+; VI-NEXT: s_cbranch_execnz .LBB109_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
@@ -6981,7 +7092,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: .LBB106_1: ; %atomicrmw.start
+; GFX9-NEXT: .LBB109_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_min_i32_e32 v2, s2, v3
@@ -6991,7 +7102,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execnz .LBB106_1
+; GFX9-NEXT: s_cbranch_execnz .LBB109_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, 0
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
new file mode 100644
index 00000000000000..e32c218e9fe56a
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=atomic-expand %s | FileCheck %s
+
+; Test that system scoped atomicrmw or 0 is transformed to add 0.
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0:![0-9]+]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+ ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_global_one_as(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_one_as(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("one-as") seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 syncscope("one-as") seq_cst
+ ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_flat_system(ptr %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_flat_system(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr %ptr, i32 0 seq_cst
+ ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_as999_system(ptr addrspace(999) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_as999_system(
+; CHECK-SAME: ptr addrspace(999) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(999) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(999) %ptr, i32 0 seq_cst
+ ret i32 %res
+}
+
+; Leave as-is, only system scope should be changed.
+define i32 @test_atomicrmw_or_0_global_agent(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 syncscope("agent") seq_cst
+ ret i32 %res
+}
+
+; Leave as-is, LDS atomics aren't relevant.
+define i32 @test_atomicrmw_or_0_local(ptr addrspace(3) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_local(
+; CHECK-SAME: ptr addrspace(3) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(3) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(3) %ptr, i32 0 seq_cst
+ ret i32 %res
+}
+
+; Leave non-0 values alone.
+define i32 @test_atomicrmw_or_1_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_1_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 1 seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(1) %ptr, i32 1 seq_cst
+ ret i32 %res
+}
+
+define i32 @test_atomicrmw_or_var_global_system(ptr addrspace(1) %ptr, i32 %val) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_var_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw or ptr addrspace(1) %ptr, i32 %val seq_cst
+ ret i32 %res
+}
+
+; Leave as-is
+define i32 @test_atomicrmw_add_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_add_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw add ptr addrspace(1) %ptr, i32 0 seq_cst
+ ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_sub_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_sub_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+ ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_xor_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_xor_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+ ret i32 %res
+}
+
+
+!0 = !{}
+;.
+; CHECK: [[META0]] = !{}
+;.
>From 05fdc0009917d5cd2d5a1117ccf8dce22fd294a6 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Fri, 12 Apr 2024 22:44:42 +0200
Subject: [PATCH 2/2] Formatting
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6350a6598545a5..6fde73119f815e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16325,9 +16325,10 @@ bool SITargetLowering::checkForPhysRegDependency(
}
void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
- if (AI->getOperation() == AtomicRMWInst::Sub ||
- AI->getOperation() == AtomicRMWInst::Or ||
- AI->getOperation() == AtomicRMWInst::Xor) {
+ AtomicRMWInst::BinOp Op = AI->getOperation();
+
+ if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
+ Op == AtomicRMWInst::Xor) {
// atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
assert(cast<Constant>(AI->getValOperand())->isNullValue() &&
"this cannot be replaced with add");
@@ -16341,8 +16342,7 @@ void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
AI->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
"generic atomicrmw expansion only supports FP32 operand in flat "
"address space");
- assert(AI->getOperation() == AtomicRMWInst::FAdd &&
- "only fadd is supported for now");
+ assert(Op == AtomicRMWInst::FAdd && "only fadd is supported for now");
// Given: atomicrmw fadd ptr %addr, float %val ordering
//
More information about the llvm-commits
mailing list