[llvm] [LLVM] Make use of s_flbit_i32_b64 and s_ff1_i32_b64 (PR #75158)
Acim Maravic via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 14 07:20:17 PST 2023
https://github.com/Acim-Maravic updated https://github.com/llvm/llvm-project/pull/75158
>From 38ba4bb76dcc45a2817fee92c57f2bdd641c1fd3 Mon Sep 17 00:00:00 2001
From: Acim Maravic <Acim.Maravic at amd.com>
Date: Thu, 14 Dec 2023 13:18:15 +0100
Subject: [PATCH 1/2] [NFC][LLVM] Make use of s_flbit_i32_b64 and s_ff1_i32_b64
---
llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 788 ++++++++++++++++++
.../CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll | 94 +++
2 files changed, 882 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 354f5b954659a5..d2114257548d23 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -6,6 +6,8 @@
declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone
+declare i16 @llvm.ctlz.i16(i16, i1) nounwind readnone
+
declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) nounwind readnone
@@ -305,6 +307,792 @@ define amdgpu_kernel void @v_ctlz_zero_undef_v4i32(ptr addrspace(1) noalias %out
ret void
}
+define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noalias %out, i8 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i8_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_flbit_i32_b32 s2, s2
+; SI-NEXT: s_sub_i32 s4, s2, 24
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i8_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s2, s2, 0xff
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: s_add_i32 s2, s2, -16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_add_u16_e64 v2, s2, -8
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_byte v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i8_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT T0.W, T0.X,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: -24(nan), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i8_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xff
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0
+; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i8 %val, 0
+ %ret = select i1 %ctlz_ret, i8 %ctlz, i8 32
+ store i8 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i16_with_select(ptr addrspace(1) noalias %out, i16 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i16_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_flbit_i32_b32 s2, s2
+; SI-NEXT: s_add_i32 s4, s2, -16
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i16_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s2, s2, 0xffff
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: s_add_i32 s2, s2, -16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_short v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i16_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT T0.W, T0.X,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: -16(nan), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i16_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0
+; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_short v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i16 @llvm.ctlz.i16(i16 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i16 %val, 0
+ %ret = select i1 %ctlz_ret, i16 %ctlz, i16 32
+ store i16 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i32_with_select(ptr addrspace(1) noalias %out, i32 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i32_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b32 s4, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i32_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i32_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: FFBH_UINT * T1.X, KC0[2].Z,
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i32_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i32 %val, 0
+ %ret = select i1 %ctlz_ret, i32 %ctlz, i32 32
+ store i32 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i64_with_select(ptr addrspace(1) noalias %out, i64 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i64_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b32 s2, s2
+; SI-NEXT: s_flbit_i32_b32 s3, s3
+; SI-NEXT: s_add_i32 s2, s2, 32
+; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i64_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: s_flbit_i32_b32 s3, s3
+; VI-NEXT: s_add_i32 s2, s2, 32
+; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i64_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 7, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: FFBH_UINT * T0.W, KC0[2].W,
+; EG-NEXT: FFBH_UINT T1.W, KC0[3].X,
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, KC0[3].X, PS, PV.W,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i64_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_flbit_i32_b64 s4, s[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i64 %val, 0
+ %ret = select i1 %ctlz_ret, i64 %ctlz, i64 32
+ store i64 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i8_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_ffbh_u32_e32 v1, v0
+; SI-NEXT: v_subrev_i32_e32 v1, vcc, 24, v1
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i8_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT: v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT: v_add_u16_e32 v1, -8, v1
+; VI-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; VI-NEXT: v_cndmask_b32_e32 v2, 32, v1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_byte v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i8_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT * T0.W, T0.X,
+; EG-NEXT: ADD_INT T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: -24(nan), 3(4.203895e-45)
+; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i8_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i8, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i8 %val, 0
+ %ret = select i1 %ctlz_ret, i8 %ctlz, i8 32
+ store i8 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i16_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_ffbh_u32_e32 v1, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, -16, v1
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i16_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 1
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v2
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT: v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; VI-NEXT: v_cndmask_b32_e32 v2, 32, v1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_short v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i16_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT * T0.W, T0.X,
+; EG-NEXT: ADD_INT T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: -16(nan), 3(4.203895e-45)
+; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i16_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i16, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i16 @llvm.ctlz.i16(i16 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i16 %val, 0
+ %ret = select i1 %ctlz_ret, i16 %ctlz, i16 32
+ store i16 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i32_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i32_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_ffbh_u32_e32 v0, v0
+; SI-NEXT: v_min_u32_e32 v0, 32, v0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i32_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 3
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: s_add_u32 s4, s2, 2
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: s_add_u32 s2, s2, 1
+; VI-NEXT: s_addc_u32 s3, s3, 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s2
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: flat_load_ubyte v3, v[4:5]
+; VI-NEXT: flat_load_ubyte v4, v[6:7]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: v_ffbh_u32_e32 v0, v0
+; VI-NEXT: v_min_u32_e32 v2, 32, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i32_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 1 @6
+; EG-NEXT: ALU 6, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 2, #1
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 10:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: LSHL * T0.W, T1.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: OR_INT * T0.W, PV.W, T0.X,
+; EG-NEXT: FFBH_UINT * T1.W, PV.W,
+; EG-NEXT: CNDE_INT T0.X, T0.W, literal.x, PV.W,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 32(4.484155e-44), 2(2.802597e-45)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v2, v3, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i32, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i32 %val, 0
+ %ret = select i1 %ctlz_ret, i32 %ctlz, i32 32
+ store i32 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i64_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s6
+; SI-NEXT: s_mov_b32 s9, s7
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:5
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:2
+; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:4
+; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:6
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v3, v5, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_or_b32_e32 v1, v3, v2
+; SI-NEXT: v_ffbh_u32_e32 v1, v1
+; SI-NEXT: v_ffbh_u32_e32 v0, v0
+; SI-NEXT: v_min_u32_e32 v1, 0xffffffdf, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v1
+; SI-NEXT: v_min3_u32 v0, v1, v0, 64
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i64_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 5
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: s_add_u32 s4, s2, 4
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: s_add_u32 s4, s2, 7
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: s_add_u32 s4, s2, 6
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v7, s5
+; VI-NEXT: v_mov_b32_e32 v6, s4
+; VI-NEXT: s_add_u32 s4, s2, 3
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v9, s5
+; VI-NEXT: v_mov_b32_e32 v8, s4
+; VI-NEXT: s_add_u32 s4, s2, 2
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v11, s5
+; VI-NEXT: v_mov_b32_e32 v10, s4
+; VI-NEXT: s_add_u32 s4, s2, 1
+; VI-NEXT: flat_load_ubyte v12, v[0:1]
+; VI-NEXT: flat_load_ubyte v13, v[2:3]
+; VI-NEXT: flat_load_ubyte v4, v[4:5]
+; VI-NEXT: flat_load_ubyte v5, v[6:7]
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: flat_load_ubyte v6, v[8:9]
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: flat_load_ubyte v7, v[10:11]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt vmcnt(7)
+; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v12
+; VI-NEXT: s_waitcnt vmcnt(6)
+; VI-NEXT: v_or_b32_e32 v3, v3, v13
+; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; VI-NEXT: s_waitcnt vmcnt(4)
+; VI-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_or_b32_e32 v3, v4, v3
+; VI-NEXT: v_ffbh_u32_e32 v3, v3
+; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-NEXT: v_or_b32_e32 v0, v4, v0
+; VI-NEXT: v_ffbh_u32_e32 v0, v0
+; VI-NEXT: v_add_u32_e64 v0, s[2:3], v0, 32 clamp
+; VI-NEXT: v_min3_u32 v0, v0, v3, 64
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i64_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 15, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 2, #1
+; EG-NEXT: VTX_READ_16 T2.X, T0.X, 4, #1
+; EG-NEXT: VTX_READ_16 T3.X, T0.X, 6, #1
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: LSHL * T0.W, T1.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: OR_INT * T0.W, PV.W, T0.X,
+; EG-NEXT: FFBH_UINT T1.W, PV.W,
+; EG-NEXT: LSHL * T2.W, T3.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.W, T0.W, literal.x, PV.W,
+; EG-NEXT: OR_INT * T1.W, PS, T2.X,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: FFBH_UINT T2.W, PS,
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, T1.W, PS, PV.W,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i64_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[2:3] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[2:3] offset:4
+; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[2:3] offset:5
+; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[2:3] offset:6
+; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[2:3] offset:7
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(6)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 8, v0
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(5)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(4)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v3, v0
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v4, v6, 8, v5
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v8, 24, v5
+; GFX9-GISEL-NEXT: v_or3_b32 v3, v0, v4, 0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, 32, v0
+; GFX9-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v4, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 64, v0, vcc
+; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i64 %val, 0
+ %ret = select i1 %ctlz_ret, i64 %ctlz, i64 64
+ store i64 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %valptr) nounwind {
; SI-LABEL: v_ctlz_zero_undef_i8:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
new file mode 100644
index 00000000000000..f52910814f5dea
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+
+declare i64 @llvm.ctlz.i64(i64, i1) nounwind readnone
+declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
+
+define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GCN-LABEL: ctlz_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_clause 0x7
+; GCN-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GCN-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GCN-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GCN-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GCN-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GCN-NEXT: global_load_ubyte v6, v1, s[2:3]
+; GCN-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
+; GCN-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GCN-NEXT: s_waitcnt vmcnt(7)
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GCN-NEXT: s_waitcnt vmcnt(5)
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GCN-NEXT: s_waitcnt vmcnt(4)
+; GCN-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GCN-NEXT: s_waitcnt vmcnt(3)
+; GCN-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, v0, v8
+; GCN-NEXT: v_or_b32_e32 v4, v4, v6
+; GCN-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_e32 v4, v5, v4
+; GCN-NEXT: v_or_b32_e32 v0, v2, v0
+; GCN-NEXT: v_ffbh_u32_e32 v3, v4
+; GCN-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NEXT: v_add_nc_u32_e32 v2, 32, v3
+; GCN-NEXT: v_min_u32_e32 v0, v2, v0
+; GCN-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GCN-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ store i64 %ctlz, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GCN-LABEL: cttz_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_clause 0x7
+; GCN-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GCN-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
+; GCN-NEXT: global_load_ubyte v3, v1, s[2:3] offset:1
+; GCN-NEXT: global_load_ubyte v4, v1, s[2:3] offset:4
+; GCN-NEXT: global_load_ubyte v5, v1, s[2:3] offset:6
+; GCN-NEXT: global_load_ubyte v6, v1, s[2:3] offset:3
+; GCN-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GCN-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GCN-NEXT: s_waitcnt vmcnt(7)
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GCN-NEXT: s_waitcnt vmcnt(6)
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GCN-NEXT: s_waitcnt vmcnt(5)
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GCN-NEXT: s_waitcnt vmcnt(4)
+; GCN-NEXT: v_or_b32_e32 v0, v0, v4
+; GCN-NEXT: s_waitcnt vmcnt(3)
+; GCN-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: s_waitcnt vmcnt(2)
+; GCN-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_or_b32_e32 v3, v3, v7
+; GCN-NEXT: v_or_b32_e32 v0, v2, v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_or_b32_sdwa v2, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_ffbl_b32_e32 v0, v0
+; GCN-NEXT: v_or_b32_e32 v2, v2, v3
+; GCN-NEXT: v_add_nc_u32_e32 v0, 32, v0
+; GCN-NEXT: v_ffbl_b32_e32 v2, v2
+; GCN-NEXT: v_min_u32_e32 v0, v2, v0
+; GCN-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GCN-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
+ store i64 %cttz, ptr addrspace(1) %out, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
>From bd0c467ba49a525da97a13ab1e4366e66b348969 Mon Sep 17 00:00:00 2001
From: Acim Maravic <Acim.Maravic at amd.com>
Date: Thu, 14 Dec 2023 16:19:32 +0100
Subject: [PATCH 2/2] [LLVM] Make use of s_flbit_i32_b64 and s_ff1_i32_b64
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 63 ++++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 +
.../AMDGPU/atomic_optimizations_buffer.ll | 81 ++---
.../atomic_optimizations_global_pointer.ll | 78 ++---
.../atomic_optimizations_local_pointer.ll | 291 +++++-------------
.../AMDGPU/atomic_optimizations_raw_buffer.ll | 54 +---
.../atomic_optimizations_struct_buffer.ll | 54 +---
llvm/test/CodeGen/AMDGPU/ctlz.ll | 76 ++---
llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 39 +--
llvm/test/CodeGen/AMDGPU/cttz.ll | 52 ++--
llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll | 53 ++--
.../AMDGPU/global_atomics_scan_fadd.ll | 105 ++-----
.../AMDGPU/global_atomics_scan_fmax.ll | 51 +--
.../AMDGPU/global_atomics_scan_fmin.ll | 51 +--
.../AMDGPU/global_atomics_scan_fsub.ll | 105 ++-----
llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll | 20 +-
.../CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll | 35 +--
.../CodeGen/AMDGPU/move-to-valu-ctlz-cttz.mir | 208 +++++++++++++
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 25 +-
llvm/test/CodeGen/AMDGPU/srem64.ll | 91 +++---
llvm/test/CodeGen/AMDGPU/udiv64.ll | 74 ++---
llvm/test/CodeGen/AMDGPU/urem64.ll | 76 ++---
23 files changed, 721 insertions(+), 970 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index fcbdf51b03c1fc..960ef1199a015f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3053,17 +3053,21 @@ SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) cons
bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF ||
Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF;
+ bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
- if (Src.getValueType() == MVT::i32) {
+ if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
// (ctlz hi:lo) -> (umin (ffbh src), 32)
// (cttz hi:lo) -> (umin (ffbl src), 32)
// (ctlz_zero_undef src) -> (ffbh src)
// (cttz_zero_undef src) -> (ffbl src)
+ // 64-bit scalar version produce 32-bit result
SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src);
if (!ZeroUndef) {
const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const32);
}
+ if (Is64BitScalar)
+ return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
return NewOpr;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0a06fa88b6b102..1fa2c98af3ea5b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6849,6 +6849,15 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
Inst.eraseFromParent();
return;
+ case AMDGPU::S_FLBIT_I32_B64:
+ splitScalar64BitCountOp(Worklist, Inst, AMDGPU::S_FLBIT_I32_B32);
+ Inst.eraseFromParent();
+ return;
+ case AMDGPU::S_FF1_I32_B64:
+ splitScalar64BitCountOp(Worklist, Inst, AMDGPU::S_FF1_I32_B32);
+ Inst.eraseFromParent();
+ return;
+
case AMDGPU::S_LSHL_B32:
if (ST.hasOnlyRevVALUShifts()) {
NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
@@ -7834,6 +7843,60 @@ void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
}
+void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
+ MachineInstr &Inst, unsigned Opcode,
+ MachineDominatorTree *MDT) const {
+ // (S_FLBIT_I32_B64 hi:lo) -> (umin (S_FLBIT_I32_B32 hi), (uaddsat (S_FLBIT_I32_B32 lo), 32))
+ // (S_FF1_I32_B64 hi:lo) -> (umin (uaddsat (S_FF1_I32_B32 hi), 32) (S_FF1_I32_B32 lo))
+
+ MachineBasicBlock &MBB = *Inst.getParent();
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+ MachineBasicBlock::iterator MII = Inst;
+ const DebugLoc &DL = Inst.getDebugLoc();
+
+ MachineOperand &Dest = Inst.getOperand(0);
+ MachineOperand &Src = Inst.getOperand(1);
+
+ const MCInstrDesc &InstDesc = get(Opcode);
+ bool IsCtlz = Opcode == AMDGPU::S_FLBIT_I32_B32;
+
+ const TargetRegisterClass *SrcRC =
+ Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
+
+ const TargetRegisterClass *SrcSubRC =
+ RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
+
+ MachineOperand SrcRegSub0 =
+ buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);
+
+ MachineOperand SrcRegSub1 =
+ buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);
+
+ Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+
+ MachineInstr *M1 = BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0);
+ MachineInstr *M2 = BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1);
+
+ BuildMI(MBB, MII, DL, get(AMDGPU::S_ADD_I32), MidReg3)
+ .addReg(IsCtlz ? MidReg1 : MidReg2)
+ .addImm(32);
+
+ BuildMI(MBB, MII, DL, get(AMDGPU::S_MIN_U32), MidReg4)
+ .addReg(MidReg3)
+ .addReg(IsCtlz ? MidReg2 : MidReg1);
+
+ MRI.replaceRegWith(Dest.getReg(), MidReg4);
+
+ Worklist.insert(M1);
+ Worklist.insert(M2);
+
+ addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist);
+}
+
void SIInstrInfo::addUsersToMoveToVALUWorklist(
Register DstReg, MachineRegisterInfo &MRI,
SIInstrWorklist &Worklist) const {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 0ce31ac6d54ec5..17e0050f69f5e5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -147,6 +147,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
MachineInstr &Inst) const;
void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
+ void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
+ unsigned Opcode,
+ MachineDominatorTree *MDT = nullptr) const;
void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
MachineInstr &Inst) const;
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index f8f50c7cb23a5a..12574c41f7cd31 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -458,13 +458,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -502,13 +499,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -545,10 +539,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -627,16 +618,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -739,13 +726,10 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB3_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -785,13 +769,10 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -830,10 +811,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB3_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -918,16 +896,12 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB3_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB3_1
@@ -1540,13 +1514,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB7_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1584,13 +1555,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1627,10 +1595,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1709,16 +1674,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB7_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 81fd166e3779f8..d64112e6972de3 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -525,15 +525,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s3
-; GFX8-NEXT: s_ff1_i32_b32 s5, s2
-; GFX8-NEXT: s_add_i32 s4, s4, 32
-; GFX8-NEXT: s_min_u32 s7, s5, s4
-; GFX8-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX8-NEXT: s_mov_b32 m0, s7
+; GFX8-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s4
+; GFX8-NEXT: v_readlane_b32 s7, v0, s4
+; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX8-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8-NEXT: s_add_i32 s6, s6, s8
+; GFX8-NEXT: s_add_i32 s6, s6, s7
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cbranch_scc1 .LBB2_1
@@ -574,15 +571,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s3
-; GFX9-NEXT: s_ff1_i32_b32 s5, s2
-; GFX9-NEXT: s_add_i32 s4, s4, 32
-; GFX9-NEXT: s_min_u32 s7, s5, s4
-; GFX9-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX9-NEXT: s_mov_b32 m0, s7
+; GFX9-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s4
+; GFX9-NEXT: v_readlane_b32 s7, v0, s4
+; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX9-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9-NEXT: s_add_i32 s6, s6, s8
+; GFX9-NEXT: s_add_i32 s6, s6, s7
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cbranch_scc1 .LBB2_1
@@ -623,10 +617,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s2
-; GFX1064-NEXT: s_add_i32 s4, s4, 32
-; GFX1064-NEXT: s_min_u32 s7, s5, s4
+; GFX1064-NEXT: s_ff1_i32_b64 s7, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s7
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1064-NEXT: v_writelane_b32 v1, s6, s7
@@ -721,16 +712,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1164-NEXT: s_add_i32 s4, s4, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s7, s5, s4
+; GFX1164-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s7
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1164-NEXT: v_writelane_b32 v1, s6, s7
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s6, s6, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB2_1
@@ -2038,15 +2025,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB8_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s3
-; GFX8-NEXT: s_ff1_i32_b32 s5, s2
-; GFX8-NEXT: s_add_i32 s4, s4, 32
-; GFX8-NEXT: s_min_u32 s7, s5, s4
-; GFX8-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX8-NEXT: s_mov_b32 m0, s7
+; GFX8-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s4
+; GFX8-NEXT: v_readlane_b32 s7, v0, s4
+; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX8-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8-NEXT: s_add_i32 s6, s6, s8
+; GFX8-NEXT: s_add_i32 s6, s6, s7
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cbranch_scc1 .LBB8_1
@@ -2087,15 +2071,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s3
-; GFX9-NEXT: s_ff1_i32_b32 s5, s2
-; GFX9-NEXT: s_add_i32 s4, s4, 32
-; GFX9-NEXT: s_min_u32 s7, s5, s4
-; GFX9-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX9-NEXT: s_mov_b32 m0, s7
+; GFX9-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s4
+; GFX9-NEXT: v_readlane_b32 s7, v0, s4
+; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX9-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9-NEXT: s_add_i32 s6, s6, s8
+; GFX9-NEXT: s_add_i32 s6, s6, s7
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
@@ -2136,10 +2117,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s2
-; GFX1064-NEXT: s_add_i32 s4, s4, 32
-; GFX1064-NEXT: s_min_u32 s7, s5, s4
+; GFX1064-NEXT: s_ff1_i32_b64 s7, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s7
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1064-NEXT: v_writelane_b32 v1, s6, s7
@@ -2234,16 +2212,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1164-NEXT: s_add_i32 s4, s4, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s7, s5, s4
+; GFX1164-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s7
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1164-NEXT: v_writelane_b32 v1, s6, s7
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s6, s6, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 60c31344452182..22a0730200a0f7 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -500,13 +500,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -545,13 +542,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -589,10 +583,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -677,16 +668,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB2_1
@@ -790,10 +777,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX8-NEXT: s_mov_b32 s2, 0
; GFX8-NEXT: .LBB3_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s3, s1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s0
-; GFX8-NEXT: s_add_i32 s3, s3, 32
-; GFX8-NEXT: s_min_u32 s3, s4, s3
+; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_readlane_b32 s6, v0, s3
; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX8-NEXT: s_add_i32 s2, s2, s6
@@ -823,10 +807,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s0
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s4, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_readlane_b32 s6, v0, s3
; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX9-NEXT: s_add_i32 s2, s2, s6
@@ -855,10 +836,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1064-NEXT: s_mov_b32 s2, 0
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s0
-; GFX1064-NEXT: s_add_i32 s3, s3, 32
-; GFX1064-NEXT: s_min_u32 s3, s4, s3
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s6, v0, s3
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
@@ -919,15 +897,12 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164-NEXT: s_mov_b32 s2, 0
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s0
-; GFX1164-NEXT: s_add_i32 s3, s3, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s3, s4, s3
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s6, v0, s3
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: s_add_i32 s2, s2, s6
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
@@ -2094,13 +2069,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB9_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2139,13 +2111,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2183,10 +2152,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -2271,16 +2237,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
@@ -2384,10 +2346,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX8-NEXT: s_mov_b32 s2, 0
; GFX8-NEXT: .LBB10_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s3, s1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s0
-; GFX8-NEXT: s_add_i32 s3, s3, 32
-; GFX8-NEXT: s_min_u32 s3, s4, s3
+; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_readlane_b32 s6, v0, s3
; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX8-NEXT: s_add_i32 s2, s2, s6
@@ -2417,10 +2376,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s0
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s4, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_readlane_b32 s6, v0, s3
; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX9-NEXT: s_add_i32 s2, s2, s6
@@ -2449,10 +2405,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1064-NEXT: s_mov_b32 s2, 0
; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s0
-; GFX1064-NEXT: s_add_i32 s3, s3, 32
-; GFX1064-NEXT: s_min_u32 s3, s4, s3
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s6, v0, s3
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
@@ -2513,15 +2466,12 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164-NEXT: s_mov_b32 s2, 0
; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s0
-; GFX1164-NEXT: s_add_i32 s3, s3, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s3, s4, s3
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s6, v0, s3
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: s_add_i32 s2, s2, s6
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
@@ -3239,13 +3189,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB14_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_and_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3284,13 +3231,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_and_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3328,10 +3272,7 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -3416,16 +3357,12 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_and_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
@@ -3536,13 +3473,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB15_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_or_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3581,13 +3515,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_or_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3625,10 +3556,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -3713,16 +3641,12 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_or_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
@@ -3833,13 +3757,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB16_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_xor_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3878,13 +3799,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB16_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_xor_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3922,10 +3840,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -4010,16 +3925,12 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_xor_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB16_1
@@ -4130,13 +4041,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB17_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_max_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4175,13 +4083,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_max_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4219,10 +4124,7 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -4307,16 +4209,12 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_max_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
@@ -4680,13 +4578,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB19_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_min_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4725,13 +4620,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB19_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_min_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4769,10 +4661,7 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB19_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -4857,16 +4746,12 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB19_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_min_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB19_1
@@ -5230,13 +5115,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB21_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_max_u32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5275,13 +5157,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB21_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_max_u32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5319,10 +5198,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB21_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -5407,16 +5283,12 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB21_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_max_u32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB21_1
@@ -5775,13 +5647,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB23_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_min_u32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5820,13 +5689,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB23_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_min_u32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5864,10 +5730,7 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -5952,16 +5815,12 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_min_u32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB23_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index baa0c72dbf63e2..854dd7a7d85836 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -457,13 +457,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -501,13 +498,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -544,10 +538,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -626,16 +617,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -1242,13 +1229,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB6_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1286,13 +1270,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1329,10 +1310,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB6_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1411,16 +1389,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB6_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB6_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 37d421b0179794..f0e8e17bb4b179 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -472,13 +472,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -517,13 +514,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -561,10 +555,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -645,16 +636,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -1382,13 +1369,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB7_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1427,13 +1411,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1471,10 +1452,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1555,16 +1533,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB7_1
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 213b6e6e620d30..b3b1880f439f3f 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -607,13 +607,10 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s4, s4
-; SI-NEXT: s_flbit_i32_b32 s5, s5
-; SI-NEXT: s_min_u32 s4, s4, 0xffffffdf
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: s_add_i32 s4, s4, 32
-; SI-NEXT: v_min3_u32 v0, s4, v0, 64
+; SI-NEXT: s_flbit_i32_b64 s4, s[4:5]
+; SI-NEXT: s_min_u32 s4, s4, 32
; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -625,10 +622,9 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s4, s4
-; VI-NEXT: v_add_u32_e64 v0, s[6:7], s4, 32 clamp
-; VI-NEXT: s_flbit_i32_b32 s4, s5
-; VI-NEXT: v_min3_u32 v0, v0, s4, 64
+; VI-NEXT: s_flbit_i32_b64 s4, s[4:5]
+; VI-NEXT: s_min_u32 s4, s4, 32
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
@@ -657,10 +653,9 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_flbit_i32_b32 s0, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s0, 32 clamp
-; GFX10-NEXT: s_flbit_i32_b32 s0, s3
-; GFX10-NEXT: v_min3_u32 v0, v0, s0, 64
+; GFX10-NEXT: s_flbit_i32_b64 s0, s[2:3]
+; GFX10-NEXT: s_min_u32 s0, s0, 32
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -684,14 +679,11 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x4c
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clz_i32_u32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX11-NEXT: s_clz_i32_u32 s2, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_min3_u32 v0, v0, s2, 64
+; GFX11-NEXT: s_clz_i32_u64 s2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_min_u32 s2, s2, 32
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -706,16 +698,13 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; SI-NEXT: s_min_u32 s2, s2, 32
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: s_flbit_i32_b32 s0, s2
-; SI-NEXT: s_min_u32 s0, s0, 0xffffffdf
-; SI-NEXT: s_flbit_i32_b32 s1, s3
-; SI-NEXT: s_add_i32 s0, s0, 32
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: v_min3_u32 v0, s0, v0, 64
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -726,11 +715,10 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: s_flbit_i32_b32 s0, s2
+; VI-NEXT: s_flbit_i32_b64 s0, s[2:3]
+; VI-NEXT: s_min_u32 s0, s0, 32
; VI-NEXT: s_mov_b32 s5, s1
-; VI-NEXT: v_add_u32_e64 v0, s[0:1], s0, 32 clamp
-; VI-NEXT: s_flbit_i32_b32 s0, s3
-; VI-NEXT: v_min3_u32 v0, v0, s0, 64
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -754,13 +742,12 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX10-LABEL: s_ctlz_i64_trunc:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_flbit_i32_b32 s2, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX10-NEXT: s_flbit_i32_b32 s2, s3
-; GFX10-NEXT: v_min3_u32 v0, v0, s2, 64
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; GFX10-NEXT: s_min_u32 s2, s2, 32
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_ctlz_i64_trunc:
@@ -777,15 +764,12 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX11-LABEL: s_ctlz_i64_trunc:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clz_i32_u32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX11-NEXT: s_clz_i32_u32 s2, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_min3_u32 v0, v0, s2, 64
-; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: s_clz_i32_u64 s2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_min_u32 s2, s2, 32
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index d2114257548d23..bce3a6aa8d11d5 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -521,10 +521,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s2, s2
-; SI-NEXT: s_flbit_i32_b32 s3, s3
-; SI-NEXT: s_add_i32 s2, s2, 32
-; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
@@ -537,10 +534,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
-; VI-NEXT: s_flbit_i32_b32 s3, s3
-; VI-NEXT: s_add_i32 s2, s2, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -940,9 +934,9 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: v_or_b32_e32 v1, v3, v2
; SI-NEXT: v_ffbh_u32_e32 v1, v1
; SI-NEXT: v_ffbh_u32_e32 v0, v0
-; SI-NEXT: v_min_u32_e32 v1, 0xffffffdf, v1
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v1
-; SI-NEXT: v_min3_u32 v0, v1, v0, 64
+; SI-NEXT: v_min_u32_e32 v0, v1, v0
+; SI-NEXT: v_min_u32_e32 v0, 32, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1010,9 +1004,10 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: v_or_b32_e32 v0, v4, v0
; VI-NEXT: v_ffbh_u32_e32 v0, v0
-; VI-NEXT: v_add_u32_e64 v0, s[2:3], v0, 32 clamp
-; VI-NEXT: v_min3_u32 v0, v0, v3, 64
+; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT: v_min_u32_e32 v0, v0, v3
; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_min_u32_e32 v0, 32, v0
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
@@ -1191,10 +1186,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64(ptr addrspace(1) noalias %out,
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s4, s4
-; SI-NEXT: s_flbit_i32_b32 s5, s5
-; SI-NEXT: s_add_i32 s4, s4, 32
-; SI-NEXT: s_min_u32 s4, s4, s5
+; SI-NEXT: s_flbit_i32_b64 s4, s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -1206,10 +1198,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64(ptr addrspace(1) noalias %out,
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
-; VI-NEXT: s_flbit_i32_b32 s3, s3
-; VI-NEXT: s_add_i32 s2, s2, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -1255,10 +1244,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s2, s2
-; SI-NEXT: s_flbit_i32_b32 s3, s3
-; SI-NEXT: s_add_i32 s2, s2, 32
-; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
@@ -1270,10 +1256,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
-; VI-NEXT: s_flbit_i32_b32 s3, s3
-; VI-NEXT: s_add_i32 s2, s2, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index e871b80cbe29ea..ff5089a27f1e9c 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -510,13 +510,10 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ff1_i32_b32 s5, s5
-; SI-NEXT: s_min_u32 s5, s5, 0xffffffdf
-; SI-NEXT: s_add_i32 s5, s5, 32
-; SI-NEXT: s_ff1_i32_b32 s4, s4
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: v_min3_u32 v0, s4, v0, 64
+; SI-NEXT: s_ff1_i32_b64 s4, s[4:5]
+; SI-NEXT: s_min_u32 s4, s4, 32
; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -528,10 +525,9 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s5, s5
-; VI-NEXT: v_add_u32_e64 v0, s[6:7], s5, 32 clamp
-; VI-NEXT: s_ff1_i32_b32 s4, s4
-; VI-NEXT: v_min3_u32 v0, s4, v0, 64
+; VI-NEXT: s_ff1_i32_b64 s4, s[4:5]
+; VI-NEXT: s_min_u32 s4, s4, 32
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
@@ -560,10 +556,9 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ff1_i32_b32 s0, s3
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s0, 32 clamp
-; GFX10-NEXT: s_ff1_i32_b32 s0, s2
-; GFX10-NEXT: v_min3_u32 v0, s0, v0, 64
+; GFX10-NEXT: s_ff1_i32_b64 s0, s[2:3]
+; GFX10-NEXT: s_min_u32 s0, s0, 32
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -591,16 +586,13 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_ff1_i32_b64 s2, s[2:3]
+; SI-NEXT: s_min_u32 s2, s2, 32
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: s_ff1_i32_b32 s0, s3
-; SI-NEXT: s_min_u32 s0, s0, 0xffffffdf
-; SI-NEXT: s_add_i32 s0, s0, 32
-; SI-NEXT: s_ff1_i32_b32 s1, s2
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_min3_u32 v0, s1, v0, 64
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -611,11 +603,10 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: s_ff1_i32_b32 s0, s3
+; VI-NEXT: s_ff1_i32_b64 s0, s[2:3]
+; VI-NEXT: s_min_u32 s0, s0, 32
; VI-NEXT: s_mov_b32 s5, s1
-; VI-NEXT: v_add_u32_e64 v0, s[0:1], s0, 32 clamp
-; VI-NEXT: s_ff1_i32_b32 s0, s2
-; VI-NEXT: v_min3_u32 v0, s0, v0, 64
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -639,13 +630,12 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX10-LABEL: s_cttz_i64_trunc:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ff1_i32_b32 s3, s3
-; GFX10-NEXT: s_ff1_i32_b32 s2, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s3, 32 clamp
-; GFX10-NEXT: v_min3_u32 v0, s2, v0, 64
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_ff1_i32_b64 s2, s[2:3]
+; GFX10-NEXT: s_min_u32 s2, s2, 32
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_cttz_i64_trunc:
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 855b5fff11fe55..88789bbb917da7 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -504,10 +504,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ff1_i32_b32 s3, s3
-; SI-NEXT: s_ff1_i32_b32 s2, s2
-; SI-NEXT: s_add_i32 s3, s3, 32
-; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: s_ff1_i32_b64 s2, s[2:3]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
@@ -520,10 +517,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s3, s3
-; VI-NEXT: s_ff1_i32_b32 s2, s2
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_ff1_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -882,39 +876,41 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: s_mov_b32 s10, s2
; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_mov_b32 s8, s6
; SI-NEXT: s_mov_b32 s9, s7
-; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
-; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:1
-; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2
-; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:3
-; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:4
-; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:5
-; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:6
-; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:5
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:2
+; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:4
+; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:6
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_or_b32_e32 v1, v3, v2
-; SI-NEXT: v_or_b32_e32 v2, v5, v4
-; SI-NEXT: v_or_b32_e32 v3, v7, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v3, v5, v4
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_or_b32_e32 v1, v3, v2
; SI-NEXT: v_ffbl_b32_e32 v1, v1
; SI-NEXT: v_ffbl_b32_e32 v0, v0
-; SI-NEXT: v_min_u32_e32 v1, 0xffffffdf, v1
-; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v1
-; SI-NEXT: v_min3_u32 v0, v0, v1, 64
+; SI-NEXT: v_add_i32_e32 v0, vcc, 32, v0
+; SI-NEXT: v_min_u32_e32 v0, v0, v1
+; SI-NEXT: v_min_u32_e32 v0, 32, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -974,7 +970,7 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
; VI-NEXT: v_ffbl_b32_e32 v3, v3
-; VI-NEXT: v_add_u32_e64 v3, s[2:3], v3, 32 clamp
+; VI-NEXT: v_add_u32_e32 v3, vcc, 32, v3
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
@@ -983,8 +979,9 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: v_or_b32_e32 v0, v4, v0
; VI-NEXT: v_ffbl_b32_e32 v0, v0
-; VI-NEXT: v_min3_u32 v0, v0, v3, 64
+; VI-NEXT: v_min_u32_e32 v0, v3, v0
; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_min_u32_e32 v0, 32, v0
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 4cbd5e84871cc7..f9b9f6e0c47dd7 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -417,10 +417,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -487,10 +484,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -613,17 +607,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -1623,10 +1614,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1693,10 +1681,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1819,17 +1804,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -2873,10 +2855,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -2943,10 +2922,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3069,17 +3045,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -3621,10 +3594,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3691,10 +3661,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3817,17 +3784,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -4826,10 +4790,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -4896,10 +4857,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -5022,17 +4980,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index e07514b063ee48..a7c96479466d7c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -436,10 +436,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -510,11 +507,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -644,12 +638,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -1635,10 +1626,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -1709,11 +1697,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -1843,12 +1828,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -2834,10 +2816,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -2908,11 +2887,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -3042,12 +3018,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index ad6edbd2c37a09..b5e9604d93e3a5 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -436,10 +436,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -510,11 +507,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -644,12 +638,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -1635,10 +1626,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -1709,11 +1697,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -1843,12 +1828,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -2834,10 +2816,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -2908,11 +2887,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -3042,12 +3018,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 4a00d7bc71bca8..19fe8dbb82149f 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -469,10 +469,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -539,10 +536,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -665,17 +659,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -1719,10 +1710,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1789,10 +1777,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1915,17 +1900,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -2969,10 +2951,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3039,10 +3018,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3165,17 +3141,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -3761,10 +3734,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3831,10 +3801,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3957,17 +3924,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -5010,10 +4974,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -5080,10 +5041,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -5206,17 +5164,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
index 38e79350c0e181..806f3cd6ed6e45 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
@@ -192,10 +192,7 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB2_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_ff1_i32_b32 s3, s5
-; VI-NEXT: s_ff1_i32_b32 s6, s4
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s3, s6, s3
+; VI-NEXT: s_ff1_i32_b64 s3, s[4:5]
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
@@ -280,10 +277,7 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB2_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s5
-; GFX9-NEXT: s_ff1_i32_b32 s6, s4
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s6, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[4:5]
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
@@ -561,10 +555,7 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB3_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_ff1_i32_b32 s3, s5
-; VI-NEXT: s_ff1_i32_b32 s6, s4
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s3, s6, s3
+; VI-NEXT: s_ff1_i32_b64 s3, s[4:5]
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
@@ -643,10 +634,7 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB3_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s5
-; GFX9-NEXT: s_ff1_i32_b32 s6, s4
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s6, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[4:5]
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
index f52910814f5dea..c45933fd50a26a 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
@@ -32,11 +32,11 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GCN-NEXT: v_or_b32_e32 v4, v4, v6
; GCN-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GCN-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GCN-NEXT: v_or_b32_e32 v4, v5, v4
+; GCN-NEXT: v_or_b32_e32 v3, v5, v4
; GCN-NEXT: v_or_b32_e32 v0, v2, v0
-; GCN-NEXT: v_ffbh_u32_e32 v3, v4
+; GCN-NEXT: v_ffbh_u32_e32 v2, v3
; GCN-NEXT: v_ffbh_u32_e32 v0, v0
-; GCN-NEXT: v_add_nc_u32_e32 v2, 32, v3
+; GCN-NEXT: v_add_nc_u32_e32 v2, 32, v2
; GCN-NEXT: v_min_u32_e32 v0, v2, v0
; GCN-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
; GCN-NEXT: s_endpgm
@@ -55,34 +55,33 @@ define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GCN-NEXT: s_clause 0x7
; GCN-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
; GCN-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
-; GCN-NEXT: global_load_ubyte v3, v1, s[2:3] offset:1
-; GCN-NEXT: global_load_ubyte v4, v1, s[2:3] offset:4
-; GCN-NEXT: global_load_ubyte v5, v1, s[2:3] offset:6
-; GCN-NEXT: global_load_ubyte v6, v1, s[2:3] offset:3
+; GCN-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
+; GCN-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GCN-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GCN-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
; GCN-NEXT: global_load_ubyte v7, v1, s[2:3]
; GCN-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GCN-NEXT: s_waitcnt vmcnt(7)
; GCN-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GCN-NEXT: s_waitcnt vmcnt(6)
; GCN-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GCN-NEXT: s_waitcnt vmcnt(5)
-; GCN-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_or_b32_e32 v0, v0, v4
+; GCN-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; GCN-NEXT: v_or_b32_e32 v0, v0, v6
+; GCN-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_or_b32_e32 v3, v3, v7
-; GCN-NEXT: v_or_b32_e32 v0, v2, v0
+; GCN-NEXT: v_or_b32_e32 v3, v4, v7
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_or_b32_sdwa v2, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_e32 v0, v2, v0
+; GCN-NEXT: v_or_b32_e32 v2, v4, v3
; GCN-NEXT: v_ffbl_b32_e32 v0, v0
-; GCN-NEXT: v_or_b32_e32 v2, v2, v3
-; GCN-NEXT: v_add_nc_u32_e32 v0, 32, v0
; GCN-NEXT: v_ffbl_b32_e32 v2, v2
-; GCN-NEXT: v_min_u32_e32 v0, v2, v0
+; GCN-NEXT: v_add_nc_u32_e32 v0, 32, v0
+; GCN-NEXT: v_min_u32_e32 v0, v0, v2
; GCN-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
; GCN-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.mir b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.mir
new file mode 100644
index 00000000000000..96e7c69115cc2c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.mir
@@ -0,0 +1,208 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
+
+---
+name: ctlz_i64
+tracksRegLiveness: true
+
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1
+
+ ; GCN-LABEL: name: ctlz_i64
+ ; GCN: liveins: $sgpr0_sgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+ ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 44, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 4, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 5, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR1]], implicit $exec
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_]], [[GLOBAL_LOAD_UBYTE_SADDR]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR2:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 6, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR3:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 7, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR3]], implicit $exec
+ ; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_1]], [[GLOBAL_LOAD_UBYTE_SADDR2]], implicit $exec
+ ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_2:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_1]], killed [[V_OR_B32_e64_1]], implicit $exec
+ ; GCN-NEXT: [[V_OR_B32_e64_2:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_2]], killed [[V_OR_B32_e64_]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR4:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR5:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 1, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_3:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR5]], implicit $exec
+ ; GCN-NEXT: [[DEF5:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_3:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_3]], [[GLOBAL_LOAD_UBYTE_SADDR4]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR6:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 2, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR7:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 3, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_4:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR7]], implicit $exec
+ ; GCN-NEXT: [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_4:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_4]], [[GLOBAL_LOAD_UBYTE_SADDR6]], implicit $exec
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_5:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_1]], killed [[V_OR_B32_e64_4]], implicit $exec
+ ; GCN-NEXT: [[V_OR_B32_e64_5:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_5]], killed [[V_OR_B32_e64_3]], implicit $exec
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_OR_B32_e64_5]], %subreg.sub0, killed [[V_OR_B32_e64_2]], %subreg.sub1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GCN-NEXT: [[V_FFBH_U32_e32_:%[0-9]+]]:vgpr_32 = V_FFBH_U32_e32 [[COPY1]], implicit $exec
+ ; GCN-NEXT: [[V_FFBH_U32_e32_1:%[0-9]+]]:vgpr_32 = V_FFBH_U32_e32 [[COPY2]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_FFBH_U32_e32_]], 32, 0, implicit $exec
+ ; GCN-NEXT: [[V_MIN_U32_e64_:%[0-9]+]]:vgpr_32 = V_MIN_U32_e64 [[V_ADD_U32_e64_]], [[V_FFBH_U32_e32_1]], implicit $exec
+ ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[V_MIN_U32_e64_]], implicit $exec
+ ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[V_MOV_B32_e32_1]], %subreg.sub1
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; GCN-NEXT: GLOBAL_STORE_DWORDX2_SADDR [[V_MOV_B32_e32_]], killed [[COPY4]], killed [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+ ; GCN-NEXT: S_ENDPGM 0
+ %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %5:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ %6:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 44, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ %7:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 4, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %8:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 5, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %9:sreg_32 = S_MOV_B32 8
+ %11:sreg_32 = COPY %8
+ %10:sreg_32 = S_LSHL_B32 killed %11, %9, implicit-def dead $scc
+ %13:sreg_32 = COPY %7
+ %12:sreg_32 = S_OR_B32 killed %10, killed %13, implicit-def dead $scc
+ %14:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 6, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %15:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 7, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %17:sreg_32 = COPY %15
+ %16:sreg_32 = S_LSHL_B32 killed %17, %9, implicit-def dead $scc
+ %19:sreg_32 = COPY %14
+ %18:sreg_32 = S_OR_B32 killed %16, killed %19, implicit-def dead $scc
+ %20:sreg_32 = S_MOV_B32 16
+ %21:sreg_32 = S_LSHL_B32 killed %18, %20, implicit-def dead $scc
+ %22:sreg_32 = S_OR_B32 killed %21, killed %12, implicit-def dead $scc
+ %23:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %24:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 1, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %26:sreg_32 = COPY %24
+ %25:sreg_32 = S_LSHL_B32 killed %26, %9, implicit-def dead $scc
+ %28:sreg_32 = COPY %23
+ %27:sreg_32 = S_OR_B32 killed %25, killed %28, implicit-def dead $scc
+ %29:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 2, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %30:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 3, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %32:sreg_32 = COPY %30
+ %31:sreg_32 = S_LSHL_B32 killed %32, %9, implicit-def dead $scc
+ %34:sreg_32 = COPY %29
+ %33:sreg_32 = S_OR_B32 killed %31, killed %34, implicit-def dead $scc
+ %35:sreg_32 = S_LSHL_B32 killed %33, %20, implicit-def dead $scc
+ %36:sreg_32 = S_OR_B32 killed %35, killed %27, implicit-def dead $scc
+ %37:sreg_64 = REG_SEQUENCE killed %36, %subreg.sub0, killed %22, %subreg.sub1
+ %38:sreg_32 = S_FLBIT_I32_B64 killed %37
+ %39:sreg_32 = S_MOV_B32 0
+ %40:sreg_64 = REG_SEQUENCE killed %38, %subreg.sub0, killed %39, %subreg.sub1
+ %41:vreg_64 = COPY %40
+ GLOBAL_STORE_DWORDX2_SADDR %4, killed %41, killed %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+ S_ENDPGM 0
+
+...
+
+---
+name: v_cttz_zero_undef_i64_with_select
+tracksRegLiveness: true
+
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1
+
+ ; GCN-LABEL: name: v_cttz_zero_undef_i64_with_select
+ ; GCN: liveins: $sgpr0_sgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
+ ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 44, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 4, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 5, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR1]], implicit $exec
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_]], [[GLOBAL_LOAD_UBYTE_SADDR]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR2:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 6, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR3:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 7, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR3]], implicit $exec
+ ; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_1]], [[GLOBAL_LOAD_UBYTE_SADDR2]], implicit $exec
+ ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_2:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_1]], killed [[V_OR_B32_e64_1]], implicit $exec
+ ; GCN-NEXT: [[V_OR_B32_e64_2:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_2]], killed [[V_OR_B32_e64_]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR4:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR5:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 1, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_3:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR5]], implicit $exec
+ ; GCN-NEXT: [[DEF5:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_3:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_3]], [[GLOBAL_LOAD_UBYTE_SADDR4]], implicit $exec
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR6:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 2, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR7:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 3, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ ; GCN-NEXT: [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_4:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_]], [[GLOBAL_LOAD_UBYTE_SADDR7]], implicit $exec
+ ; GCN-NEXT: [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[V_OR_B32_e64_4:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_4]], [[GLOBAL_LOAD_UBYTE_SADDR6]], implicit $exec
+ ; GCN-NEXT: [[V_LSHLREV_B32_e64_5:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[S_MOV_B32_1]], killed [[V_OR_B32_e64_4]], implicit $exec
+ ; GCN-NEXT: [[V_OR_B32_e64_5:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_LSHLREV_B32_e64_5]], killed [[V_OR_B32_e64_3]], implicit $exec
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_OR_B32_e64_5]], %subreg.sub0, killed [[V_OR_B32_e64_2]], %subreg.sub1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GCN-NEXT: [[V_FFBL_B32_e32_:%[0-9]+]]:vgpr_32 = V_FFBL_B32_e32 [[COPY1]], implicit $exec
+ ; GCN-NEXT: [[V_FFBL_B32_e32_1:%[0-9]+]]:vgpr_32 = V_FFBL_B32_e32 [[COPY2]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_FFBL_B32_e32_1]], 32, 0, implicit $exec
+ ; GCN-NEXT: [[V_MIN_U32_e64_:%[0-9]+]]:vgpr_32 = V_MIN_U32_e64 [[V_ADD_U32_e64_]], [[V_FFBL_B32_e32_]], implicit $exec
+ ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[V_MIN_U32_e64_]], implicit $exec
+ ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[V_MOV_B32_e32_1]], %subreg.sub1
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; GCN-NEXT: GLOBAL_STORE_DWORDX2_SADDR [[V_MOV_B32_e32_]], killed [[COPY4]], killed [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+ ; GCN-NEXT: S_ENDPGM 0
+ %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %5:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ %6:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 44, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ %7:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 4, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %8:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 5, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %9:sreg_32 = S_MOV_B32 8
+ %11:sreg_32 = COPY %8
+ %10:sreg_32 = S_LSHL_B32 killed %11, %9, implicit-def dead $scc
+ %13:sreg_32 = COPY %7
+ %12:sreg_32 = S_OR_B32 killed %10, killed %13, implicit-def dead $scc
+ %14:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 6, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %15:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 7, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %17:sreg_32 = COPY %15
+ %16:sreg_32 = S_LSHL_B32 killed %17, %9, implicit-def dead $scc
+ %19:sreg_32 = COPY %14
+ %18:sreg_32 = S_OR_B32 killed %16, killed %19, implicit-def dead $scc
+ %20:sreg_32 = S_MOV_B32 16
+ %21:sreg_32 = S_LSHL_B32 killed %18, %20, implicit-def dead $scc
+ %22:sreg_32 = S_OR_B32 killed %21, killed %12, implicit-def dead $scc
+ %23:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %24:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 1, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %26:sreg_32 = COPY %24
+ %25:sreg_32 = S_LSHL_B32 killed %26, %9, implicit-def dead $scc
+ %28:sreg_32 = COPY %23
+ %27:sreg_32 = S_OR_B32 killed %25, killed %28, implicit-def dead $scc
+ %29:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 2, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %30:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR %6, %4, 3, 0, implicit $exec :: ("amdgpu-noclobber" load (s8), addrspace 1)
+ %32:sreg_32 = COPY %30
+ %31:sreg_32 = S_LSHL_B32 killed %32, %9, implicit-def dead $scc
+ %34:sreg_32 = COPY %29
+ %33:sreg_32 = S_OR_B32 killed %31, killed %34, implicit-def dead $scc
+ %35:sreg_32 = S_LSHL_B32 killed %33, %20, implicit-def dead $scc
+ %36:sreg_32 = S_OR_B32 killed %35, killed %27, implicit-def dead $scc
+ %37:sreg_64 = REG_SEQUENCE killed %36, %subreg.sub0, killed %22, %subreg.sub1
+ %38:sreg_32 = S_FF1_I32_B64 killed %37
+ %39:sreg_32 = S_MOV_B32 0
+ %40:sreg_64 = REG_SEQUENCE killed %38, %subreg.sub0, killed %39, %subreg.sub1
+ %41:vreg_64 = COPY %40
+ GLOBAL_STORE_DWORDX2_SADDR %4, killed %41, killed %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index adc4c85aa60dc0..d794968f9830a9 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -156,15 +156,9 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-IR-NEXT: s_subb_u32 s7, s7, s2
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[12:13], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[10:11], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s14, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s12
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT: s_min_u32 s18, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s18, s[12:13]
; GCN-IR-NEXT: s_sub_u32 s16, s14, s18
; GCN-IR-NEXT: s_subb_u32 s17, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
@@ -993,15 +987,9 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
; GCN-IR-NEXT: s_subb_u32 s7, s7, s4
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[12:13], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[8:9], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s14, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s12
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT: s_min_u32 s18, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s18, s[12:13]
; GCN-IR-NEXT: s_sub_u32 s16, s14, s18
; GCN-IR-NEXT: s_subb_u32 s17, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
@@ -1203,10 +1191,7 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GCN-IR-NEXT: s_sub_u32 s2, s2, s4
; GCN-IR-NEXT: s_subb_u32 s3, s3, s4
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[2:3]
; GCN-IR-NEXT: s_add_u32 s12, s10, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s13, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index bfb409438757c7..40d8467ad7e396 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -124,67 +124,61 @@ define amdgpu_kernel void @s_test_srem(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s14, s6, s7
-; GCN-IR-NEXT: s_sub_u32 s12, s10, s14
-; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[14:15], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[14:15]
+; GCN-IR-NEXT: s_sub_u32 s14, s10, s12
+; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[14:15], 63
; GCN-IR-NEXT: s_or_b64 s[16:17], s[8:9], s[16:17]
; GCN-IR-NEXT: s_and_b64 s[8:9], s[16:17], exec
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[16:17], s[16:17], s[18:19]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[16:17]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT: s_add_u32 s16, s12, 1
-; GCN-IR-NEXT: s_addc_u32 s17, s13, 0
+; GCN-IR-NEXT: s_add_u32 s16, s14, 1
+; GCN-IR-NEXT: s_addc_u32 s17, s15, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT: s_sub_i32 s12, 63, s12
+; GCN-IR-NEXT: s_sub_i32 s13, 63, s14
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s12
+; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s13
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_4
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: s_lshr_b64 s[12:13], s[2:3], s16
+; GCN-IR-NEXT: s_lshr_b64 s[14:15], s[2:3], s16
; GCN-IR-NEXT: s_add_u32 s16, s4, -1
; GCN-IR-NEXT: s_addc_u32 s17, s5, -1
; GCN-IR-NEXT: s_not_b64 s[6:7], s[10:11]
-; GCN-IR-NEXT: s_add_u32 s10, s6, s14
+; GCN-IR-NEXT: s_add_u32 s10, s6, s12
; GCN-IR-NEXT: s_addc_u32 s11, s7, 0
-; GCN-IR-NEXT: s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT: s_mov_b64 s[12:13], 0
; GCN-IR-NEXT: s_mov_b32 s7, 0
; GCN-IR-NEXT: .LBB0_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT: s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT: s_lshl_b64 s[14:15], s[14:15], 1
; GCN-IR-NEXT: s_lshr_b32 s6, s9, 31
; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[8:9], 1
-; GCN-IR-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[14:15], s[8:9]
-; GCN-IR-NEXT: s_sub_u32 s6, s16, s12
-; GCN-IR-NEXT: s_subb_u32 s6, s17, s13
-; GCN-IR-NEXT: s_ashr_i32 s14, s6, 31
-; GCN-IR-NEXT: s_mov_b32 s15, s14
-; GCN-IR-NEXT: s_and_b32 s6, s14, 1
-; GCN-IR-NEXT: s_and_b64 s[14:15], s[14:15], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s12, s12, s14
-; GCN-IR-NEXT: s_subb_u32 s13, s13, s15
+; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[6:7]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[12:13], s[8:9]
+; GCN-IR-NEXT: s_sub_u32 s6, s16, s14
+; GCN-IR-NEXT: s_subb_u32 s6, s17, s15
+; GCN-IR-NEXT: s_ashr_i32 s12, s6, 31
+; GCN-IR-NEXT: s_mov_b32 s13, s12
+; GCN-IR-NEXT: s_and_b32 s6, s12, 1
+; GCN-IR-NEXT: s_and_b64 s[12:13], s[12:13], s[4:5]
+; GCN-IR-NEXT: s_sub_u32 s14, s14, s12
+; GCN-IR-NEXT: s_subb_u32 s15, s15, s13
; GCN-IR-NEXT: s_add_u32 s10, s10, 1
; GCN-IR-NEXT: s_addc_u32 s11, s11, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[10:11], 0
-; GCN-IR-NEXT: s_mov_b64 s[14:15], s[6:7]
+; GCN-IR-NEXT: s_mov_b64 s[12:13], s[6:7]
; GCN-IR-NEXT: s_and_b64 vcc, exec, s[18:19]
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_3
; GCN-IR-NEXT: .LBB0_4: ; %Flow7
@@ -1029,15 +1023,9 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_subb_u32 s9, s7, s10
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[8:9], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[8:9]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[6:7], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s8
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s9
-; GCN-IR-NEXT: s_min_u32 s12, s6, s7
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s16, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s16, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s14, s12, s16
; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
@@ -1180,15 +1168,9 @@ define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %
; GCN-IR-NEXT: s_subb_u32 s7, s7, s10
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[4:5], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[8:9], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s12, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s4
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s5
-; GCN-IR-NEXT: s_min_u32 s16, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s16, s[4:5]
; GCN-IR-NEXT: s_sub_u32 s14, s12, s16
; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
@@ -1393,10 +1375,7 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_xor_b64 s[2:3], s[2:3], s[8:9]
; GCN-IR-NEXT: s_sub_u32 s4, s2, s8
; GCN-IR-NEXT: s_subb_u32 s5, s3, s8
-; GCN-IR-NEXT: s_flbit_i32_b32 s2, s4
-; GCN-IR-NEXT: s_add_i32 s2, s2, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s3, s5
-; GCN-IR-NEXT: s_min_u32 s8, s2, s3
+; GCN-IR-NEXT: s_flbit_i32_b64 s8, s[4:5]
; GCN-IR-NEXT: s_add_u32 s2, s8, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s3, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[4:5], 0
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 6102d652402a21..9ff30a0e361051 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -125,63 +125,57 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s14, s6, s7
-; GCN-IR-NEXT: s_sub_u32 s12, s10, s14
-; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[14:15], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[14:15]
+; GCN-IR-NEXT: s_sub_u32 s14, s10, s12
+; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[14:15], 63
; GCN-IR-NEXT: s_or_b64 s[16:17], s[8:9], s[16:17]
; GCN-IR-NEXT: s_and_b64 s[8:9], s[16:17], exec
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[16:17], s[16:17], s[18:19]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[16:17]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT: s_add_u32 s16, s12, 1
-; GCN-IR-NEXT: s_addc_u32 s17, s13, 0
+; GCN-IR-NEXT: s_add_u32 s16, s14, 1
+; GCN-IR-NEXT: s_addc_u32 s17, s15, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT: s_sub_i32 s12, 63, s12
+; GCN-IR-NEXT: s_sub_i32 s13, 63, s14
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s12
+; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s13
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_4
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: s_lshr_b64 s[12:13], s[2:3], s16
-; GCN-IR-NEXT: s_add_u32 s15, s4, -1
+; GCN-IR-NEXT: s_lshr_b64 s[14:15], s[2:3], s16
+; GCN-IR-NEXT: s_add_u32 s13, s4, -1
; GCN-IR-NEXT: s_addc_u32 s16, s5, -1
; GCN-IR-NEXT: s_not_b64 s[2:3], s[10:11]
-; GCN-IR-NEXT: s_add_u32 s2, s2, s14
+; GCN-IR-NEXT: s_add_u32 s2, s2, s12
; GCN-IR-NEXT: s_addc_u32 s3, s3, 0
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: s_mov_b32 s7, 0
; GCN-IR-NEXT: .LBB0_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT: s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT: s_lshl_b64 s[14:15], s[14:15], 1
; GCN-IR-NEXT: s_lshr_b32 s6, s9, 31
; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[8:9], 1
-; GCN-IR-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
+; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[6:7]
; GCN-IR-NEXT: s_or_b64 s[8:9], s[10:11], s[8:9]
-; GCN-IR-NEXT: s_sub_u32 s6, s15, s12
-; GCN-IR-NEXT: s_subb_u32 s6, s16, s13
+; GCN-IR-NEXT: s_sub_u32 s6, s13, s14
+; GCN-IR-NEXT: s_subb_u32 s6, s16, s15
; GCN-IR-NEXT: s_ashr_i32 s10, s6, 31
; GCN-IR-NEXT: s_mov_b32 s11, s10
; GCN-IR-NEXT: s_and_b32 s6, s10, 1
; GCN-IR-NEXT: s_and_b64 s[10:11], s[10:11], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s12, s12, s10
-; GCN-IR-NEXT: s_subb_u32 s13, s13, s11
+; GCN-IR-NEXT: s_sub_u32 s14, s14, s10
+; GCN-IR-NEXT: s_subb_u32 s15, s15, s11
; GCN-IR-NEXT: s_add_u32 s2, s2, 1
; GCN-IR-NEXT: s_addc_u32 s3, s3, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[2:3], 0
@@ -796,15 +790,9 @@ define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48
; GCN-IR-NEXT: s_and_b32 s3, s3, 0xffff
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[8:9], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[2:3]
; GCN-IR-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GCN-IR-NEXT: s_flbit_i32_b32 s4, s2
-; GCN-IR-NEXT: s_add_i32 s4, s4, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s5, s3
-; GCN-IR-NEXT: s_min_u32 s10, s4, s5
-; GCN-IR-NEXT: s_flbit_i32_b32 s4, s8
-; GCN-IR-NEXT: s_add_i32 s4, s4, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s5, s9
-; GCN-IR-NEXT: s_min_u32 s14, s4, s5
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[8:9]
; GCN-IR-NEXT: s_sub_u32 s12, s10, s14
; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
@@ -986,10 +974,7 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_mov_b64 s[4:5], 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s3
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_min_u32 s8, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s8, s[2:3]
; GCN-IR-NEXT: s_add_u32 s10, s8, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s11, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
@@ -1406,10 +1391,7 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_min_u32 s10, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s8, 59, s10
; GCN-IR-NEXT: s_subb_u32 s9, 0, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 7e8661b3ff32dd..b781ccb23cfb49 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -124,67 +124,61 @@ define amdgpu_kernel void @s_test_urem_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s14, s6, s7
-; GCN-IR-NEXT: s_sub_u32 s12, s10, s14
-; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[14:15], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[14:15]
+; GCN-IR-NEXT: s_sub_u32 s14, s10, s12
+; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[14:15], 63
; GCN-IR-NEXT: s_or_b64 s[16:17], s[8:9], s[16:17]
; GCN-IR-NEXT: s_and_b64 s[8:9], s[16:17], exec
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[16:17], s[16:17], s[18:19]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[16:17]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT: s_add_u32 s16, s12, 1
-; GCN-IR-NEXT: s_addc_u32 s17, s13, 0
+; GCN-IR-NEXT: s_add_u32 s16, s14, 1
+; GCN-IR-NEXT: s_addc_u32 s17, s15, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT: s_sub_i32 s12, 63, s12
+; GCN-IR-NEXT: s_sub_i32 s13, 63, s14
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s12
+; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[2:3], s13
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_4
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: s_lshr_b64 s[12:13], s[2:3], s16
+; GCN-IR-NEXT: s_lshr_b64 s[14:15], s[2:3], s16
; GCN-IR-NEXT: s_add_u32 s16, s4, -1
; GCN-IR-NEXT: s_addc_u32 s17, s5, -1
; GCN-IR-NEXT: s_not_b64 s[6:7], s[10:11]
-; GCN-IR-NEXT: s_add_u32 s10, s6, s14
+; GCN-IR-NEXT: s_add_u32 s10, s6, s12
; GCN-IR-NEXT: s_addc_u32 s11, s7, 0
-; GCN-IR-NEXT: s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT: s_mov_b64 s[12:13], 0
; GCN-IR-NEXT: s_mov_b32 s7, 0
; GCN-IR-NEXT: .LBB0_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT: s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT: s_lshl_b64 s[14:15], s[14:15], 1
; GCN-IR-NEXT: s_lshr_b32 s6, s9, 31
; GCN-IR-NEXT: s_lshl_b64 s[8:9], s[8:9], 1
-; GCN-IR-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[14:15], s[8:9]
-; GCN-IR-NEXT: s_sub_u32 s6, s16, s12
-; GCN-IR-NEXT: s_subb_u32 s6, s17, s13
-; GCN-IR-NEXT: s_ashr_i32 s14, s6, 31
-; GCN-IR-NEXT: s_mov_b32 s15, s14
-; GCN-IR-NEXT: s_and_b32 s6, s14, 1
-; GCN-IR-NEXT: s_and_b64 s[14:15], s[14:15], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s12, s12, s14
-; GCN-IR-NEXT: s_subb_u32 s13, s13, s15
+; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[6:7]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[12:13], s[8:9]
+; GCN-IR-NEXT: s_sub_u32 s6, s16, s14
+; GCN-IR-NEXT: s_subb_u32 s6, s17, s15
+; GCN-IR-NEXT: s_ashr_i32 s12, s6, 31
+; GCN-IR-NEXT: s_mov_b32 s13, s12
+; GCN-IR-NEXT: s_and_b32 s6, s12, 1
+; GCN-IR-NEXT: s_and_b64 s[12:13], s[12:13], s[4:5]
+; GCN-IR-NEXT: s_sub_u32 s14, s14, s12
+; GCN-IR-NEXT: s_subb_u32 s15, s15, s13
; GCN-IR-NEXT: s_add_u32 s10, s10, 1
; GCN-IR-NEXT: s_addc_u32 s11, s11, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[18:19], s[10:11], 0
-; GCN-IR-NEXT: s_mov_b64 s[14:15], s[6:7]
+; GCN-IR-NEXT: s_mov_b64 s[12:13], s[6:7]
; GCN-IR-NEXT: s_and_b64 vcc, exec, s[18:19]
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_3
; GCN-IR-NEXT: .LBB0_4: ; %Flow7
@@ -814,10 +808,7 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_mov_b64 s[4:5], 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s3
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_min_u32 s8, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s8, s[2:3]
; GCN-IR-NEXT: s_add_u32 s10, s8, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s11, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
@@ -973,10 +964,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_min_u32 s8, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s8, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s10, 59, s8
; GCN-IR-NEXT: s_subb_u32 s11, 0, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
More information about the llvm-commits
mailing list