[llvm] 48f36c6 - [LLVM] Make use of s_flbit_i32_b64 and s_ff1_i32_b64 (#75158)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 25 02:55:25 PST 2023
Author: Acim Maravic
Date: 2023-12-25T11:55:20+01:00
New Revision: 48f36c6e742e743e33f931536c653bf4e23568fb
URL: https://github.com/llvm/llvm-project/commit/48f36c6e742e743e33f931536c653bf4e23568fb
DIFF: https://github.com/llvm/llvm-project/commit/48f36c6e742e743e33f931536c653bf4e23568fb.diff
LOG: [LLVM] Make use of s_flbit_i32_b64 and s_ff1_i32_b64 (#75158)
Update DAG ISel to support 64bit versions S_FF1_I32_B64 and
S_FLBIT_I32_B664
---------
Co-authored-by: Acim Maravic <Acim.Maravic at amd.com>
Added:
llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/lib/Target/AMDGPU/SIInstrInfo.h
llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
llvm/test/CodeGen/AMDGPU/ctlz.ll
llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
llvm/test/CodeGen/AMDGPU/cttz.ll
llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
llvm/test/CodeGen/AMDGPU/sdiv64.ll
llvm/test/CodeGen/AMDGPU/srem64.ll
llvm/test/CodeGen/AMDGPU/udiv64.ll
llvm/test/CodeGen/AMDGPU/urem64.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 4bf4707553e5fe..541a5b62450ddf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3070,18 +3070,26 @@ SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) cons
bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF ||
Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF;
+ bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
- if (Src.getValueType() == MVT::i32) {
+ if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
// (ctlz hi:lo) -> (umin (ffbh src), 32)
// (cttz hi:lo) -> (umin (ffbl src), 32)
// (ctlz_zero_undef src) -> (ffbh src)
// (cttz_zero_undef src) -> (ffbl src)
+
+ // 64-bit scalar version produce 32-bit result
+ // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
+ // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
+ // (ctlz_zero_undef src) -> (S_FLBIT_I32_B64 src)
+ // (cttz_zero_undef src) -> (S_FF1_I32_B64 src)
SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src);
if (!ZeroUndef) {
- const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
- NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const32);
+ const SDValue ConstVal = DAG.getConstant(
+ Op.getValueType().getScalarSizeInBits(), SL, MVT::i32);
+ NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal);
}
- return NewOpr;
+ return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr);
}
SDValue Lo, Hi;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 29ac08b6895e55..ebe23a5eac57b5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6912,6 +6912,15 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
Inst.eraseFromParent();
return;
+ case AMDGPU::S_FLBIT_I32_B64:
+ splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);
+ Inst.eraseFromParent();
+ return;
+ case AMDGPU::S_FF1_I32_B64:
+ splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);
+ Inst.eraseFromParent();
+ return;
+
case AMDGPU::S_LSHL_B32:
if (ST.hasOnlyRevVALUShifts()) {
NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
@@ -7845,6 +7854,61 @@ void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
}
+void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
+ MachineInstr &Inst, unsigned Opcode,
+ MachineDominatorTree *MDT) const {
+ // (S_FLBIT_I32_B64 hi:lo) ->
+ // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32))
+ // (S_FF1_I32_B64 hi:lo) ->
+ // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
+
+ MachineBasicBlock &MBB = *Inst.getParent();
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+ MachineBasicBlock::iterator MII = Inst;
+ const DebugLoc &DL = Inst.getDebugLoc();
+
+ MachineOperand &Dest = Inst.getOperand(0);
+ MachineOperand &Src = Inst.getOperand(1);
+
+ const MCInstrDesc &InstDesc = get(Opcode);
+
+ bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
+ unsigned OpcodeAdd =
+ ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
+
+ const TargetRegisterClass *SrcRC =
+ Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
+ const TargetRegisterClass *SrcSubRC =
+ RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
+
+ MachineOperand SrcRegSub0 =
+ buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);
+ MachineOperand SrcRegSub1 =
+ buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);
+
+ Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0);
+
+ BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1);
+
+ BuildMI(MBB, MII, DL, get(OpcodeAdd), MidReg3)
+ .addReg(IsCtlz ? MidReg1 : MidReg2)
+ .addImm(32)
+ .addImm(1); // enable clamp
+
+ BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4)
+ .addReg(MidReg3)
+ .addReg(IsCtlz ? MidReg2 : MidReg1);
+
+ MRI.replaceRegWith(Dest.getReg(), MidReg4);
+
+ addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist);
+}
+
void SIInstrInfo::addUsersToMoveToVALUWorklist(
Register DstReg, MachineRegisterInfo &MRI,
SIInstrWorklist &Worklist) const {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index affe5204675209..46eee6fae0a52a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -144,6 +144,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
MachineInstr &Inst) const;
void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
+ void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
+ unsigned Opcode,
+ MachineDominatorTree *MDT = nullptr) const;
void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
MachineInstr &Inst) const;
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index db33ed8fa55660..e3d2ecefbda30d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -589,13 +589,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -633,13 +630,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -676,10 +670,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -758,16 +749,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -849,16 +836,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -961,13 +944,10 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB3_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1007,13 +987,10 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1052,10 +1029,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB3_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1140,16 +1114,12 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB3_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB3_1
@@ -1237,16 +1207,12 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB3_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB3_1
@@ -2005,13 +1971,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB7_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2049,13 +2012,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2092,10 +2052,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -2174,16 +2131,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB7_1
@@ -2266,16 +2219,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB7_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 6a664f26d470b7..9f97f1f4bace5d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -657,15 +657,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s3
-; GFX8-NEXT: s_ff1_i32_b32 s5, s2
-; GFX8-NEXT: s_add_i32 s4, s4, 32
-; GFX8-NEXT: s_min_u32 s7, s5, s4
-; GFX8-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX8-NEXT: s_mov_b32 m0, s7
+; GFX8-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s4
+; GFX8-NEXT: v_readlane_b32 s7, v0, s4
+; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX8-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8-NEXT: s_add_i32 s6, s6, s8
+; GFX8-NEXT: s_add_i32 s6, s6, s7
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cbranch_scc1 .LBB2_1
@@ -705,15 +702,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s3
-; GFX9-NEXT: s_ff1_i32_b32 s5, s2
-; GFX9-NEXT: s_add_i32 s4, s4, 32
-; GFX9-NEXT: s_min_u32 s7, s5, s4
-; GFX9-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX9-NEXT: s_mov_b32 m0, s7
+; GFX9-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s4
+; GFX9-NEXT: v_readlane_b32 s7, v0, s4
+; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX9-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9-NEXT: s_add_i32 s6, s6, s8
+; GFX9-NEXT: s_add_i32 s6, s6, s7
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cbranch_scc1 .LBB2_1
@@ -753,10 +747,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s2
-; GFX1064-NEXT: s_add_i32 s4, s4, 32
-; GFX1064-NEXT: s_min_u32 s7, s5, s4
+; GFX1064-NEXT: s_ff1_i32_b64 s7, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s7
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1064-NEXT: v_writelane_b32 v1, s6, s7
@@ -847,16 +838,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1164-NEXT: s_add_i32 s4, s4, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s7, s5, s4
+; GFX1164-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s7
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1164-NEXT: v_writelane_b32 v1, s6, s7
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s6, s6, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB2_1
@@ -951,16 +938,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: ; implicit-def: $vgpr1
; GFX1264-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1264-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1264-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1264-NEXT: s_add_co_i32 s4, s4, 32
-; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-NEXT: s_min_u32 s7, s5, s4
+; GFX1264-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1264-NEXT: v_readlane_b32 s8, v0, s7
; GFX1264-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1264-NEXT: v_writelane_b32 v1, s6, s7
; GFX1264-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1264-NEXT: s_add_co_i32 s6, s6, s8
; GFX1264-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1264-NEXT: s_cbranch_scc1 .LBB2_1
@@ -2557,15 +2540,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB8_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s3
-; GFX8-NEXT: s_ff1_i32_b32 s5, s2
-; GFX8-NEXT: s_add_i32 s4, s4, 32
-; GFX8-NEXT: s_min_u32 s7, s5, s4
-; GFX8-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX8-NEXT: s_mov_b32 m0, s7
+; GFX8-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s4
+; GFX8-NEXT: v_readlane_b32 s7, v0, s4
+; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX8-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8-NEXT: s_add_i32 s6, s6, s8
+; GFX8-NEXT: s_add_i32 s6, s6, s7
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cbranch_scc1 .LBB8_1
@@ -2605,15 +2585,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s3
-; GFX9-NEXT: s_ff1_i32_b32 s5, s2
-; GFX9-NEXT: s_add_i32 s4, s4, 32
-; GFX9-NEXT: s_min_u32 s7, s5, s4
-; GFX9-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s7
-; GFX9-NEXT: s_mov_b32 m0, s7
+; GFX9-NEXT: s_ff1_i32_b64 s4, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s4
+; GFX9-NEXT: v_readlane_b32 s7, v0, s4
+; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX9-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9-NEXT: s_add_i32 s6, s6, s8
+; GFX9-NEXT: s_add_i32 s6, s6, s7
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
@@ -2653,10 +2630,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s2
-; GFX1064-NEXT: s_add_i32 s4, s4, 32
-; GFX1064-NEXT: s_min_u32 s7, s5, s4
+; GFX1064-NEXT: s_ff1_i32_b64 s7, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s7
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1064-NEXT: v_writelane_b32 v1, s6, s7
@@ -2747,16 +2721,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1164-NEXT: s_add_i32 s4, s4, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s7, s5, s4
+; GFX1164-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s7
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1164-NEXT: v_writelane_b32 v1, s6, s7
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s6, s6, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
@@ -2851,16 +2821,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: ; implicit-def: $vgpr1
; GFX1264-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1264-NEXT: s_ctz_i32_b32 s4, s3
-; GFX1264-NEXT: s_ctz_i32_b32 s5, s2
-; GFX1264-NEXT: s_add_co_i32 s4, s4, 32
-; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-NEXT: s_min_u32 s7, s5, s4
+; GFX1264-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1264-NEXT: v_readlane_b32 s8, v0, s7
; GFX1264-NEXT: s_lshl_b64 s[4:5], 1, s7
; GFX1264-NEXT: v_writelane_b32 v1, s6, s7
; GFX1264-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1264-NEXT: s_add_co_i32 s6, s6, s8
; GFX1264-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1264-NEXT: s_cbranch_scc1 .LBB8_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index cf2afeb7b01bc9..34499043ce6bb0 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -478,13 +478,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -522,13 +519,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -565,10 +559,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -649,16 +640,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB2_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB2_1
@@ -757,10 +744,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX8-NEXT: s_mov_b32 s2, 0
; GFX8-NEXT: .LBB3_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s3, s1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s0
-; GFX8-NEXT: s_add_i32 s3, s3, 32
-; GFX8-NEXT: s_min_u32 s3, s4, s3
+; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_readlane_b32 s6, v0, s3
; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX8-NEXT: s_add_i32 s2, s2, s6
@@ -789,10 +773,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s0
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s4, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_readlane_b32 s6, v0, s3
; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX9-NEXT: s_add_i32 s2, s2, s6
@@ -820,10 +801,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1064-NEXT: s_mov_b32 s2, 0
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s0
-; GFX1064-NEXT: s_add_i32 s3, s3, 32
-; GFX1064-NEXT: s_min_u32 s3, s4, s3
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s6, v0, s3
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
@@ -880,15 +858,12 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164-NEXT: s_mov_b32 s2, 0
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s0
-; GFX1164-NEXT: s_add_i32 s3, s3, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s3, s4, s3
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s6, v0, s3
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: s_add_i32 s2, s2, s6
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
@@ -2005,13 +1980,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB9_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2049,13 +2021,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -2092,10 +2061,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -2176,16 +2142,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_add_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
@@ -2284,10 +2246,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX8-NEXT: s_mov_b32 s2, 0
; GFX8-NEXT: .LBB10_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s3, s1
-; GFX8-NEXT: s_ff1_i32_b32 s4, s0
-; GFX8-NEXT: s_add_i32 s3, s3, 32
-; GFX8-NEXT: s_min_u32 s3, s4, s3
+; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_readlane_b32 s6, v0, s3
; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX8-NEXT: s_add_i32 s2, s2, s6
@@ -2316,10 +2275,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s1
-; GFX9-NEXT: s_ff1_i32_b32 s4, s0
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s4, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_readlane_b32 s6, v0, s3
; GFX9-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX9-NEXT: s_add_i32 s2, s2, s6
@@ -2347,10 +2303,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1064-NEXT: s_mov_b32 s2, 0
; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s4, s0
-; GFX1064-NEXT: s_add_i32 s3, s3, 32
-; GFX1064-NEXT: s_min_u32 s3, s4, s3
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s6, v0, s3
; GFX1064-NEXT: s_lshl_b64 s[4:5], 1, s3
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
@@ -2407,15 +2360,12 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164-NEXT: s_mov_b32 s2, 0
; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s4, s0
-; GFX1164-NEXT: s_add_i32 s3, s3, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s3, s4, s3
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s6, v0, s3
; GFX1164-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: s_add_i32 s2, s2, s6
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
@@ -3105,13 +3055,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB14_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_and_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3149,13 +3096,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_and_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3192,10 +3136,7 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -3276,16 +3217,12 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_and_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
@@ -3392,13 +3329,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB15_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_or_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3436,13 +3370,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_or_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3479,10 +3410,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -3563,16 +3491,12 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_or_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
@@ -3679,13 +3603,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB16_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_xor_b32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3723,13 +3644,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB16_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_xor_b32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -3766,10 +3684,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -3850,16 +3765,12 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_xor_b32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB16_1
@@ -3966,13 +3877,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB17_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_max_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4010,13 +3918,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_max_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4053,10 +3958,7 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -4137,16 +4039,12 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_max_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
@@ -4495,13 +4393,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB19_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_min_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4539,13 +4434,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB19_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_min_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -4582,10 +4474,7 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB19_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -4666,16 +4555,12 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB19_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_min_i32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB19_1
@@ -5024,13 +4909,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB21_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_max_u32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5068,13 +4950,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB21_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_max_u32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5111,10 +4990,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB21_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -5195,16 +5071,12 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB21_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_max_u32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB21_1
@@ -5548,13 +5420,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB23_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_min_u32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5592,13 +5461,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB23_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_min_u32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -5635,10 +5501,7 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s5, s3
-; GFX1064-NEXT: s_ff1_i32_b32 s6, s2
-; GFX1064-NEXT: s_add_i32 s5, s5, 32
-; GFX1064-NEXT: s_min_u32 s5, s6, s5
+; GFX1064-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX1064-NEXT: v_readlane_b32 s8, v0, s5
; GFX1064-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1064-NEXT: v_writelane_b32 v1, s4, s5
@@ -5719,16 +5582,12 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_ctz_i32_b32 s5, s3
-; GFX1164-NEXT: s_ctz_i32_b32 s6, s2
-; GFX1164-NEXT: s_add_i32 s5, s5, 32
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_min_u32 s5, s6, s5
+; GFX1164-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_readlane_b32 s8, v0, s5
; GFX1164-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX1164-NEXT: v_writelane_b32 v1, s4, s5
; GFX1164-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-NEXT: s_min_u32 s4, s4, s8
; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164-NEXT: s_cbranch_scc1 .LBB23_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 43068a28812ed1..79f8b3a1d5d84c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -588,13 +588,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -632,13 +629,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -675,10 +669,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -757,16 +748,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -848,16 +835,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -1610,13 +1593,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB6_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1654,13 +1634,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1697,10 +1674,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB6_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1779,16 +1753,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB6_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB6_1
@@ -1871,16 +1841,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB6_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB6_1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 5ac8ed8df456d9..edf6fbadf1a60a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -605,13 +605,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB2_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -650,13 +647,10 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB2_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -694,10 +688,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -778,16 +769,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -871,16 +858,12 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB2_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB2_1
@@ -1785,13 +1768,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: .LBB7_1: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b32 s5, s3
-; GFX8-NEXT: s_ff1_i32_b32 s6, s2
-; GFX8-NEXT: s_add_i32 s5, s5, 32
-; GFX8-NEXT: s_min_u32 s5, s6, s5
+; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_readlane_b32 s8, v0, s5
; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX8-NEXT: s_mov_b32 m0, s5
; GFX8-NEXT: v_writelane_b32 v1, s4, m0
; GFX8-NEXT: s_add_i32 s4, s4, s8
; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1830,13 +1810,10 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s5, s3
-; GFX9-NEXT: s_ff1_i32_b32 s6, s2
-; GFX9-NEXT: s_add_i32 s5, s5, 32
-; GFX9-NEXT: s_min_u32 s5, s6, s5
+; GFX9-NEXT: s_ff1_i32_b64 s5, s[2:3]
+; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_readlane_b32 s8, v0, s5
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s5
-; GFX9-NEXT: s_mov_b32 m0, s5
; GFX9-NEXT: v_writelane_b32 v1, s4, m0
; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
@@ -1874,10 +1851,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX10W64-NEXT: ; implicit-def: $vgpr1
; GFX10W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10W64-NEXT: s_ff1_i32_b32 s5, s3
-; GFX10W64-NEXT: s_ff1_i32_b32 s6, s2
-; GFX10W64-NEXT: s_add_i32 s5, s5, 32
-; GFX10W64-NEXT: s_min_u32 s5, s6, s5
+; GFX10W64-NEXT: s_ff1_i32_b64 s5, s[2:3]
; GFX10W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX10W64-NEXT: v_writelane_b32 v1, s4, s5
@@ -1958,16 +1932,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX11W64-NEXT: ; implicit-def: $vgpr1
; GFX11W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX11W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX11W64-NEXT: s_add_i32 s5, s5, 32
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11W64-NEXT: s_min_u32 s5, s6, s5
+; GFX11W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX11W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX11W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11W64-NEXT: s_add_i32 s4, s4, s8
; GFX11W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX11W64-NEXT: s_cbranch_scc1 .LBB7_1
@@ -2052,16 +2022,12 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
; GFX12W64-NEXT: ; implicit-def: $vgpr1
; GFX12W64-NEXT: .LBB7_1: ; %ComputeLoop
; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12W64-NEXT: s_ctz_i32_b32 s5, s3
-; GFX12W64-NEXT: s_ctz_i32_b32 s6, s2
-; GFX12W64-NEXT: s_add_co_i32 s5, s5, 32
-; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12W64-NEXT: s_min_u32 s5, s6, s5
+; GFX12W64-NEXT: s_ctz_i32_b64 s5, s[2:3]
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12W64-NEXT: v_readlane_b32 s8, v0, s5
; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s5
; GFX12W64-NEXT: v_writelane_b32 v1, s4, s5
; GFX12W64-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12W64-NEXT: s_add_co_i32 s4, s4, s8
; GFX12W64-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX12W64-NEXT: s_cbranch_scc1 .LBB7_1
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 213b6e6e620d30..3d69655111da63 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -607,13 +607,10 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s4, s4
-; SI-NEXT: s_flbit_i32_b32 s5, s5
-; SI-NEXT: s_min_u32 s4, s4, 0xffffffdf
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: s_add_i32 s4, s4, 32
-; SI-NEXT: v_min3_u32 v0, s4, v0, 64
+; SI-NEXT: s_flbit_i32_b64 s4, s[4:5]
+; SI-NEXT: s_min_u32 s4, s4, 64
; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -625,10 +622,9 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s4, s4
-; VI-NEXT: v_add_u32_e64 v0, s[6:7], s4, 32 clamp
-; VI-NEXT: s_flbit_i32_b32 s4, s5
-; VI-NEXT: v_min3_u32 v0, v0, s4, 64
+; VI-NEXT: s_flbit_i32_b64 s4, s[4:5]
+; VI-NEXT: s_min_u32 s4, s4, 64
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
@@ -657,10 +653,9 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_flbit_i32_b32 s0, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s0, 32 clamp
-; GFX10-NEXT: s_flbit_i32_b32 s0, s3
-; GFX10-NEXT: v_min3_u32 v0, v0, s0, 64
+; GFX10-NEXT: s_flbit_i32_b64 s0, s[2:3]
+; GFX10-NEXT: s_min_u32 s0, s0, 64
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -684,14 +679,11 @@ define amdgpu_kernel void @s_ctlz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x4c
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clz_i32_u32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX11-NEXT: s_clz_i32_u32 s2, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_min3_u32 v0, v0, s2, 64
+; GFX11-NEXT: s_clz_i32_u64 s2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_min_u32 s2, s2, 64
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -706,16 +698,13 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; SI-NEXT: s_min_u32 s2, s2, 64
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: s_flbit_i32_b32 s0, s2
-; SI-NEXT: s_min_u32 s0, s0, 0xffffffdf
-; SI-NEXT: s_flbit_i32_b32 s1, s3
-; SI-NEXT: s_add_i32 s0, s0, 32
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: v_min3_u32 v0, s0, v0, 64
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -726,11 +715,10 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: s_flbit_i32_b32 s0, s2
+; VI-NEXT: s_flbit_i32_b64 s0, s[2:3]
+; VI-NEXT: s_min_u32 s0, s0, 64
; VI-NEXT: s_mov_b32 s5, s1
-; VI-NEXT: v_add_u32_e64 v0, s[0:1], s0, 32 clamp
-; VI-NEXT: s_flbit_i32_b32 s0, s3
-; VI-NEXT: v_min3_u32 v0, v0, s0, 64
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -754,13 +742,12 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX10-LABEL: s_ctlz_i64_trunc:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_flbit_i32_b32 s2, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX10-NEXT: s_flbit_i32_b32 s2, s3
-; GFX10-NEXT: v_min3_u32 v0, v0, s2, 64
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; GFX10-NEXT: s_min_u32 s2, s2, 64
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_ctlz_i64_trunc:
@@ -777,15 +764,12 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX11-LABEL: s_ctlz_i64_trunc:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clz_i32_u32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e64 v0, s2, 32 clamp
-; GFX11-NEXT: s_clz_i32_u32 s2, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_min3_u32 v0, v0, s2, 64
-; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: s_clz_i32_u64 s2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_min_u32 s2, s2, 64
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 354f5b954659a5..03f3d04cf8a68f 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -6,6 +6,8 @@
declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone
+declare i16 @llvm.ctlz.i16(i16, i1) nounwind readnone
+
declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) nounwind readnone
@@ -305,6 +307,787 @@ define amdgpu_kernel void @v_ctlz_zero_undef_v4i32(ptr addrspace(1) noalias %out
ret void
}
+define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noalias %out, i8 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i8_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_flbit_i32_b32 s2, s2
+; SI-NEXT: s_sub_i32 s4, s2, 24
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i8_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s2, s2, 0xff
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: s_add_i32 s2, s2, -16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_add_u16_e64 v2, s2, -8
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_byte v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i8_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT T0.W, T0.X,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: -24(nan), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i8_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xff
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0
+; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i8 %val, 0
+ %ret = select i1 %ctlz_ret, i8 %ctlz, i8 32
+ store i8 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i16_with_select(ptr addrspace(1) noalias %out, i16 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i16_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_flbit_i32_b32 s2, s2
+; SI-NEXT: s_add_i32 s4, s2, -16
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i16_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s2, s2, 0xffff
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: s_add_i32 s2, s2, -16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_short v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i16_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT T0.W, T0.X,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: -16(nan), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i16_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0
+; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_short v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i16 @llvm.ctlz.i16(i16 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i16 %val, 0
+ %ret = select i1 %ctlz_ret, i16 %ctlz, i16 32
+ store i16 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i32_with_select(ptr addrspace(1) noalias %out, i32 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i32_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[0:1], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b32 s4, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i32_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_flbit_i32_b32 s2, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i32_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: FFBH_UINT * T1.X, KC0[2].Z,
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i32_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i32 %val, 0
+ %ret = select i1 %ctlz_ret, i32 %ctlz, i32 32
+ store i32 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i64_with_select(ptr addrspace(1) noalias %out, i64 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i64_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i64_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i64_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 7, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: FFBH_UINT * T0.W, KC0[2].W,
+; EG-NEXT: FFBH_UINT T1.W, KC0[3].X,
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, KC0[3].X, PS, PV.W,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i64_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_flbit_i32_b64 s4, s[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i64 %val, 0
+ %ret = select i1 %ctlz_ret, i64 %ctlz, i64 32
+ store i64 %ctlz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i8_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_ffbh_u32_e32 v1, v0
+; SI-NEXT: v_subrev_i32_e32 v1, vcc, 24, v1
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i8_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT: v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT: v_add_u16_e32 v1, -8, v1
+; VI-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; VI-NEXT: v_cndmask_b32_e32 v2, 32, v1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_byte v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i8_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT * T0.W, T0.X,
+; EG-NEXT: ADD_INT T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: -24(nan), 3(4.203895e-45)
+; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i8_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i8, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i8 %val, 0
+ %ret = select i1 %ctlz_ret, i8 %ctlz, i8 32
+ store i8 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i16_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_ffbh_u32_e32 v1, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, -16, v1
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i16_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 1
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b16_e32 v1, 8, v2
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT: v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; VI-NEXT: v_cndmask_b32_e32 v2, 32, v1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_short v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i16_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: FFBH_UINT * T0.W, T0.X,
+; EG-NEXT: ADD_INT T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: -16(nan), 3(4.203895e-45)
+; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i16_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i16, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i16 @llvm.ctlz.i16(i16 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i16 %val, 0
+ %ret = select i1 %ctlz_ret, i16 %ctlz, i16 32
+ store i16 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i32_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i32_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_ffbh_u32_e32 v0, v0
+; SI-NEXT: v_min_u32_e32 v0, 32, v0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i32_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 3
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: s_add_u32 s4, s2, 2
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: s_add_u32 s2, s2, 1
+; VI-NEXT: s_addc_u32 s3, s3, 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_mov_b32_e32 v6, s2
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: flat_load_ubyte v3, v[4:5]
+; VI-NEXT: flat_load_ubyte v4, v[6:7]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: v_ffbh_u32_e32 v0, v0
+; VI-NEXT: v_min_u32_e32 v2, 32, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i32_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 1 @6
+; EG-NEXT: ALU 6, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 2, #1
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 10:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: LSHL * T0.W, T1.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: OR_INT * T0.W, PV.W, T0.X,
+; EG-NEXT: FFBH_UINT * T1.W, PV.W,
+; EG-NEXT: CNDE_INT T0.X, T0.W, literal.x, PV.W,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 32(4.484155e-44), 2(2.802597e-45)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v2, v3, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i32, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i32 %val, 0
+ %ret = select i1 %ctlz_ret, i32 %ctlz, i32 32
+ store i32 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; SI-LABEL: v_ctlz_zero_undef_i64_with_select:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s6
+; SI-NEXT: s_mov_b32 s9, s7
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:5
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:2
+; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:4
+; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:6
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v3, v5, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_or_b32_e32 v1, v3, v2
+; SI-NEXT: v_ffbh_u32_e32 v1, v1
+; SI-NEXT: v_ffbh_u32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v1
+; SI-NEXT: v_min_u32_e32 v0, v1, v0
+; SI-NEXT: v_min_u32_e32 v0, 64, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_ctlz_zero_undef_i64_with_select:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s4, s2, 5
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: s_add_u32 s4, s2, 4
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: s_add_u32 s4, s2, 7
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: s_add_u32 s4, s2, 6
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v7, s5
+; VI-NEXT: v_mov_b32_e32 v6, s4
+; VI-NEXT: s_add_u32 s4, s2, 3
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v9, s5
+; VI-NEXT: v_mov_b32_e32 v8, s4
+; VI-NEXT: s_add_u32 s4, s2, 2
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v11, s5
+; VI-NEXT: v_mov_b32_e32 v10, s4
+; VI-NEXT: s_add_u32 s4, s2, 1
+; VI-NEXT: flat_load_ubyte v12, v[0:1]
+; VI-NEXT: flat_load_ubyte v13, v[2:3]
+; VI-NEXT: flat_load_ubyte v4, v[4:5]
+; VI-NEXT: flat_load_ubyte v5, v[6:7]
+; VI-NEXT: s_addc_u32 s5, s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: flat_load_ubyte v6, v[8:9]
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: flat_load_ubyte v7, v[10:11]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt vmcnt(7)
+; VI-NEXT: v_lshlrev_b32_e32 v3, 8, v12
+; VI-NEXT: s_waitcnt vmcnt(6)
+; VI-NEXT: v_or_b32_e32 v3, v3, v13
+; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; VI-NEXT: s_waitcnt vmcnt(4)
+; VI-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_or_b32_e32 v3, v4, v3
+; VI-NEXT: v_ffbh_u32_e32 v3, v3
+; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-NEXT: v_or_b32_e32 v0, v4, v0
+; VI-NEXT: v_ffbh_u32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT: v_min_u32_e32 v0, v0, v3
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_min_u32_e32 v0, 64, v0
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: v_ctlz_zero_undef_i64_with_select:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 15, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 2, #1
+; EG-NEXT: VTX_READ_16 T2.X, T0.X, 4, #1
+; EG-NEXT: VTX_READ_16 T3.X, T0.X, 6, #1
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: LSHL * T0.W, T1.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: OR_INT * T0.W, PV.W, T0.X,
+; EG-NEXT: FFBH_UINT T1.W, PV.W,
+; EG-NEXT: LSHL * T2.W, T3.X, literal.x,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.W, T0.W, literal.x, PV.W,
+; EG-NEXT: OR_INT * T1.W, PS, T2.X,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: FFBH_UINT T2.W, PS,
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, T1.W, PS, PV.W,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i64_with_select:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[2:3] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[2:3] offset:4
+; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[2:3] offset:5
+; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[2:3] offset:6
+; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[2:3] offset:7
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(6)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 8, v0
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(5)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(4)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v3, v0
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v4, v6, 8, v5
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v8, 24, v5
+; GFX9-GISEL-NEXT: v_or3_b32 v3, v0, v4, 0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, 32, v0
+; GFX9-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v4, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 64, v0, vcc
+; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ %ctlz_ret = icmp ne i64 %val, 0
+ %ret = select i1 %ctlz_ret, i64 %ctlz, i64 64
+ store i64 %ret, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %valptr) nounwind {
; SI-LABEL: v_ctlz_zero_undef_i8:
; SI: ; %bb.0:
@@ -403,10 +1186,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64(ptr addrspace(1) noalias %out,
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s4, s4
-; SI-NEXT: s_flbit_i32_b32 s5, s5
-; SI-NEXT: s_add_i32 s4, s4, 32
-; SI-NEXT: s_min_u32 s4, s4, s5
+; SI-NEXT: s_flbit_i32_b64 s4, s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -418,10 +1198,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64(ptr addrspace(1) noalias %out,
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
-; VI-NEXT: s_flbit_i32_b32 s3, s3
-; VI-NEXT: s_add_i32 s2, s2, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -467,10 +1244,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_flbit_i32_b32 s2, s2
-; SI-NEXT: s_flbit_i32_b32 s3, s3
-; SI-NEXT: s_add_i32 s2, s2, 32
-; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
@@ -482,10 +1256,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
-; VI-NEXT: s_flbit_i32_b32 s3, s3
-; VI-NEXT: s_add_i32 s2, s2, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_flbit_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index e871b80cbe29ea..db91554b2ff39b 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -510,13 +510,10 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ff1_i32_b32 s5, s5
-; SI-NEXT: s_min_u32 s5, s5, 0xffffffdf
-; SI-NEXT: s_add_i32 s5, s5, 32
-; SI-NEXT: s_ff1_i32_b32 s4, s4
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: v_min3_u32 v0, s4, v0, 64
+; SI-NEXT: s_ff1_i32_b64 s4, s[4:5]
+; SI-NEXT: s_min_u32 s4, s4, 64
; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -528,10 +525,9 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s5, s5
-; VI-NEXT: v_add_u32_e64 v0, s[6:7], s5, 32 clamp
-; VI-NEXT: s_ff1_i32_b32 s4, s4
-; VI-NEXT: v_min3_u32 v0, s4, v0, 64
+; VI-NEXT: s_ff1_i32_b64 s4, s[4:5]
+; VI-NEXT: s_min_u32 s4, s4, 64
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
@@ -560,10 +556,9 @@ define amdgpu_kernel void @s_cttz_i64(ptr addrspace(1) noalias %out, [8 x i32],
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ff1_i32_b32 s0, s3
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s0, 32 clamp
-; GFX10-NEXT: s_ff1_i32_b32 s0, s2
-; GFX10-NEXT: v_min3_u32 v0, s0, v0, 64
+; GFX10-NEXT: s_ff1_i32_b64 s0, s[2:3]
+; GFX10-NEXT: s_min_u32 s0, s0, 64
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -591,16 +586,13 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_ff1_i32_b64 s2, s[2:3]
+; SI-NEXT: s_min_u32 s2, s2, 64
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: s_ff1_i32_b32 s0, s3
-; SI-NEXT: s_min_u32 s0, s0, 0xffffffdf
-; SI-NEXT: s_add_i32 s0, s0, 32
-; SI-NEXT: s_ff1_i32_b32 s1, s2
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_min3_u32 v0, s1, v0, 64
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -611,11 +603,10 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: s_ff1_i32_b32 s0, s3
+; VI-NEXT: s_ff1_i32_b64 s0, s[2:3]
+; VI-NEXT: s_min_u32 s0, s0, 64
; VI-NEXT: s_mov_b32 s5, s1
-; VI-NEXT: v_add_u32_e64 v0, s[0:1], s0, 32 clamp
-; VI-NEXT: s_ff1_i32_b32 s0, s2
-; VI-NEXT: v_min3_u32 v0, s0, v0, 64
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -639,13 +630,12 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
; GFX10-LABEL: s_cttz_i64_trunc:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ff1_i32_b32 s3, s3
-; GFX10-NEXT: s_ff1_i32_b32 s2, s2
-; GFX10-NEXT: v_add_nc_u32_e64 v0, s3, 32 clamp
-; GFX10-NEXT: v_min3_u32 v0, s2, v0, 64
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_ff1_i32_b64 s2, s[2:3]
+; GFX10-NEXT: s_min_u32 s2, s2, 64
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_cttz_i64_trunc:
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 7e8c28fa447509..5985a235680c0c 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -500,10 +500,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ff1_i32_b32 s3, s3
-; SI-NEXT: s_ff1_i32_b32 s2, s2
-; SI-NEXT: s_add_i32 s3, s3, 32
-; SI-NEXT: s_min_u32 s2, s2, s3
+; SI-NEXT: s_ff1_i32_b64 s2, s[2:3]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
@@ -516,10 +513,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s3, s3
-; VI-NEXT: s_ff1_i32_b32 s2, s2
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s2, s2, s3
+; VI-NEXT: s_ff1_i32_b64 s2, s[2:3]
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -878,39 +872,41 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; SI-NEXT: s_mov_b32 s10, s2
; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_mov_b32 s8, s6
; SI-NEXT: s_mov_b32 s9, s7
-; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
-; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:1
-; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2
-; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:3
-; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:4
-; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:5
-; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:6
-; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:5
+; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:7
+; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0
+; SI-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:1
+; SI-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:2
+; SI-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:4
+; SI-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:6
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_or_b32_e32 v1, v3, v2
-; SI-NEXT: v_or_b32_e32 v2, v5, v4
-; SI-NEXT: v_or_b32_e32 v3, v7, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v3, v5, v4
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_or_b32_e32 v1, v3, v2
; SI-NEXT: v_ffbl_b32_e32 v1, v1
; SI-NEXT: v_ffbl_b32_e32 v0, v0
-; SI-NEXT: v_min_u32_e32 v1, 0xffffffdf, v1
-; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v1
-; SI-NEXT: v_min3_u32 v0, v0, v1, 64
+; SI-NEXT: v_add_i32_e32 v0, vcc, 32, v0
+; SI-NEXT: v_min_u32_e32 v0, v0, v1
+; SI-NEXT: v_min_u32_e32 v0, 64, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -970,7 +966,7 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
; VI-NEXT: v_ffbl_b32_e32 v3, v3
-; VI-NEXT: v_add_u32_e64 v3, s[2:3], v3, 32 clamp
+; VI-NEXT: v_add_u32_e32 v3, vcc, 32, v3
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: s_waitcnt vmcnt(1)
@@ -979,8 +975,9 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: v_or_b32_e32 v0, v4, v0
; VI-NEXT: v_ffbl_b32_e32 v0, v0
-; VI-NEXT: v_min3_u32 v0, v0, v3, 64
+; VI-NEXT: v_min_u32_e32 v0, v3, v0
; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_min_u32_e32 v0, 64, v0
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 064831f0986678..5ebd3eef69f257 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -415,10 +415,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -484,10 +481,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -608,17 +602,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -1610,10 +1601,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1679,10 +1667,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1803,17 +1788,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -2849,10 +2831,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -2918,10 +2897,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3042,17 +3018,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -3586,10 +3559,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3655,10 +3625,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3779,17 +3746,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v1, s4, v1
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -4780,10 +4744,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -4849,10 +4810,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -4973,17 +4931,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 66c89de1789ee6..ce1654b38d4b81 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -434,10 +434,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -507,11 +504,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -639,12 +633,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -1622,10 +1613,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -1695,11 +1683,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -1827,12 +1812,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -2810,10 +2792,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff800000
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -2883,11 +2862,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -3015,12 +2991,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 17533e22ce2afc..7379fd5a5422f7 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -434,10 +434,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -507,11 +504,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -639,12 +633,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -1622,10 +1613,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -1695,11 +1683,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -1827,12 +1812,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
@@ -2810,10 +2792,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: v_max_f32_e32 v1, v2, v2
@@ -2883,11 +2862,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
@@ -3015,12 +2991,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index a8b83edfa74389..b2c749c131f603 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -467,10 +467,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -536,10 +533,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -660,17 +654,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -1706,10 +1697,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1775,10 +1763,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -1899,17 +1884,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -2945,10 +2927,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3014,10 +2993,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3138,17 +3114,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -3726,10 +3699,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3795,10 +3765,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -3919,17 +3886,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
@@ -4964,10 +4928,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s2, s1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s0
-; GFX9-NEXT: s_add_i32 s2, s2, 32
-; GFX9-NEXT: s_min_u32 s2, s3, s2
+; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s4, v0, s2
; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -5033,10 +4994,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b32 s2, s1
-; GFX1064-NEXT: s_ff1_i32_b32 s3, s0
-; GFX1064-NEXT: s_add_i32 s2, s2, 32
-; GFX1064-NEXT: s_min_u32 s2, s3, s2
+; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s4, v0, s2
; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
@@ -5157,17 +5115,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1164-NEXT: s_ctz_i32_b32 s3, s0
-; GFX1164-NEXT: s_add_i32 s2, s2, 32
-; GFX1164-NEXT: s_min_u32 s2, s3, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164-NEXT: v_readlane_b32 s4, v0, s2
; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_add_f32_e32 v2, s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
index b90c92bf9be52a..5296ef1f886789 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
@@ -182,10 +182,7 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB2_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_ff1_i32_b32 s3, s5
-; VI-NEXT: s_ff1_i32_b32 s6, s4
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s3, s6, s3
+; VI-NEXT: s_ff1_i32_b64 s3, s[4:5]
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
@@ -268,10 +265,7 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB2_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s5
-; GFX9-NEXT: s_ff1_i32_b32 s6, s4
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s6, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[4:5]
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
@@ -543,10 +537,7 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB3_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_ff1_i32_b32 s3, s5
-; VI-NEXT: s_ff1_i32_b32 s6, s4
-; VI-NEXT: s_add_i32 s3, s3, 32
-; VI-NEXT: s_min_u32 s3, s6, s3
+; VI-NEXT: s_ff1_i32_b64 s3, s[4:5]
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
@@ -625,10 +616,7 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB3_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b32 s3, s5
-; GFX9-NEXT: s_ff1_i32_b32 s6, s4
-; GFX9-NEXT: s_add_i32 s3, s3, 32
-; GFX9-NEXT: s_min_u32 s3, s6, s3
+; GFX9-NEXT: s_ff1_i32_b64 s3, s[4:5]
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
new file mode 100644
index 00000000000000..91ba353390f3c7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
@@ -0,0 +1,332 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+
+declare i64 @llvm.ctlz.i64(i64, i1) nounwind readnone
+declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
+
+define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GFX9-LABEL: ctlz_i64_poison:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX9-NEXT: v_ffbh_u32_e32 v2, v2
+; GFX9-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e64 v2, v2, 32 clamp
+; GFX9-NEXT: v_min_u32_e32 v0, v2, v0
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: ctlz_i64_poison:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x7
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
+; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
+; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: s_waitcnt vmcnt(7)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: s_waitcnt vmcnt(5)
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_ffbh_u32_e32 v2, v3
+; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX10-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
+; GFX10-NEXT: v_min_u32_e32 v0, v2, v0
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
+ store i64 %ctlz, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GFX9-LABEL: ctlz_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX9-NEXT: v_ffbh_u32_e32 v2, v2
+; GFX9-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e64 v2, v2, 32 clamp
+; GFX9-NEXT: v_min_u32_e32 v0, v2, v0
+; GFX9-NEXT: v_min_u32_e32 v0, 64, v0
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: ctlz_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x7
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
+; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
+; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: s_waitcnt vmcnt(7)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: s_waitcnt vmcnt(5)
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_ffbh_u32_e32 v2, v3
+; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX10-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
+; GFX10-NEXT: v_min_u32_e32 v0, v2, v0
+; GFX10-NEXT: v_min_u32_e32 v0, 64, v0
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 false) nounwind readnone
+ store i64 %ctlz, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GFX9-LABEL: cttz_i64_poison:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX9-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX9-NEXT: v_ffbl_b32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e64 v0, v0, 32 clamp
+; GFX9-NEXT: v_min_u32_e32 v0, v0, v2
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: cttz_i64_poison:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x7
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
+; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX10-NEXT: s_waitcnt vmcnt(7)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
+; GFX10-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
+; GFX10-NEXT: v_min_u32_e32 v0, v0, v2
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
+ store i64 %cttz, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
+; GFX9-LABEL: cttz_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX9-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX9-NEXT: v_ffbl_b32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e64 v0, v0, 32 clamp
+; GFX9-NEXT: v_min_u32_e32 v0, v0, v2
+; GFX9-NEXT: v_min_u32_e32 v0, 64, v0
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: cttz_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x7
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
+; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
+; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX10-NEXT: s_waitcnt vmcnt(7)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_or_b32_e32 v3, v4, v7
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
+; GFX10-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
+; GFX10-NEXT: v_min_u32_e32 v0, v0, v2
+; GFX10-NEXT: v_min_u32_e32 v0, 64, v0
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
+ %val = load i64, ptr addrspace(1) %arrayidx, align 1
+ %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 false) nounwind readnone
+ store i64 %cttz, ptr addrspace(1) %out, align 8
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index c48370a9c6c75b..7f84d21fbbc448 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -156,15 +156,9 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-IR-NEXT: s_subb_u32 s7, s7, s2
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[12:13], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[10:11], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s14, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s12
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT: s_min_u32 s20, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s20, s[12:13]
; GCN-IR-NEXT: s_sub_u32 s16, s14, s20
; GCN-IR-NEXT: s_subb_u32 s17, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[18:19], s[16:17], 63
@@ -993,15 +987,9 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
; GCN-IR-NEXT: s_subb_u32 s7, s7, s4
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[12:13], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[8:9], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s14, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s12
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT: s_min_u32 s20, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s20, s[12:13]
; GCN-IR-NEXT: s_sub_u32 s16, s14, s20
; GCN-IR-NEXT: s_subb_u32 s17, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[18:19], s[16:17], 63
@@ -1203,10 +1191,7 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GCN-IR-NEXT: s_sub_u32 s2, s2, s4
; GCN-IR-NEXT: s_subb_u32 s3, s3, s4
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT: s_min_u32 s14, s10, s11
+; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[2:3]
; GCN-IR-NEXT: s_add_u32 s10, s14, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s11, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index ac212d22e9cfa7..70e75116e180ad 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -124,18 +124,14 @@ define amdgpu_kernel void @s_test_srem(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s18, s6, s7
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s18, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
; GCN-IR-NEXT: s_sub_u32 s12, s10, s18
; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
@@ -145,9 +141,7 @@ define amdgpu_kernel void @s_test_srem(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[14:15]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
; GCN-IR-NEXT: s_add_u32 s14, s12, 1
@@ -1029,15 +1023,9 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_subb_u32 s9, s7, s10
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[8:9], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[8:9]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[6:7], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s8
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s9
-; GCN-IR-NEXT: s_min_u32 s12, s6, s7
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s20, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s20, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s14, s12, s20
; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
@@ -1180,15 +1168,9 @@ define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %
; GCN-IR-NEXT: s_subb_u32 s7, s7, s10
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[4:5], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[6:7]
; GCN-IR-NEXT: s_or_b64 s[10:11], s[8:9], s[10:11]
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT: s_min_u32 s12, s8, s9
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s4
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s5
-; GCN-IR-NEXT: s_min_u32 s20, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s20, s[4:5]
; GCN-IR-NEXT: s_sub_u32 s14, s12, s20
; GCN-IR-NEXT: s_subb_u32 s15, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
@@ -1393,10 +1375,7 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_xor_b64 s[2:3], s[2:3], s[8:9]
; GCN-IR-NEXT: s_sub_u32 s4, s2, s8
; GCN-IR-NEXT: s_subb_u32 s5, s3, s8
-; GCN-IR-NEXT: s_flbit_i32_b32 s2, s4
-; GCN-IR-NEXT: s_add_i32 s2, s2, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s3, s5
-; GCN-IR-NEXT: s_min_u32 s12, s2, s3
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[4:5]
; GCN-IR-NEXT: s_add_u32 s2, s12, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s3, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 9301170c034d89..7a8d19200a72e4 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -125,18 +125,14 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s16, s6, s7
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s16, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
; GCN-IR-NEXT: s_sub_u32 s12, s10, s16
; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
@@ -146,9 +142,7 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[18:19]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[14:15]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
; GCN-IR-NEXT: s_add_u32 s14, s12, 1
@@ -796,15 +790,9 @@ define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48
; GCN-IR-NEXT: s_and_b32 s3, s3, 0xffff
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[8:9], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[2:3]
; GCN-IR-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GCN-IR-NEXT: s_flbit_i32_b32 s4, s2
-; GCN-IR-NEXT: s_add_i32 s4, s4, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s5, s3
-; GCN-IR-NEXT: s_min_u32 s10, s4, s5
-; GCN-IR-NEXT: s_flbit_i32_b32 s4, s8
-; GCN-IR-NEXT: s_add_i32 s4, s4, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s5, s9
-; GCN-IR-NEXT: s_min_u32 s16, s4, s5
+; GCN-IR-NEXT: s_flbit_i32_b64 s16, s[8:9]
; GCN-IR-NEXT: s_sub_u32 s12, s10, s16
; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
@@ -986,10 +974,7 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_mov_b64 s[4:5], 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s3
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_min_u32 s12, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
; GCN-IR-NEXT: s_add_u32 s8, s12, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s9, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
@@ -1406,10 +1391,7 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_min_u32 s12, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s8, 59, s12
; GCN-IR-NEXT: s_subb_u32 s9, 0, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 784993ccd3bd1a..3ec51b01c7a3c0 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -124,18 +124,14 @@ define amdgpu_kernel void @s_test_urem_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[4:5], 0
-; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT: s_flbit_i32_b32 s10, s4
-; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s11, s5
-; GCN-IR-NEXT: s_add_i32 s10, s10, 32
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_min_u32 s10, s10, s11
-; GCN-IR-NEXT: s_min_u32 s18, s6, s7
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT: s_flbit_i32_b64 s10, s[4:5]
+; GCN-IR-NEXT: s_flbit_i32_b64 s18, s[2:3]
+; GCN-IR-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
; GCN-IR-NEXT: s_sub_u32 s12, s10, s18
; GCN-IR-NEXT: s_subb_u32 s13, 0, 0
; GCN-IR-NEXT: v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
@@ -145,9 +141,7 @@ define amdgpu_kernel void @s_test_urem_i64(ptr addrspace(1) %out, i64 %x, i64 %y
; GCN-IR-NEXT: s_cselect_b32 s9, 0, s3
; GCN-IR-NEXT: s_cselect_b32 s8, 0, s2
; GCN-IR-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
-; GCN-IR-NEXT: s_mov_b64 s[6:7], 0
; GCN-IR-NEXT: s_andn2_b64 vcc, exec, s[14:15]
-; GCN-IR-NEXT: s_mov_b32 s11, 0
; GCN-IR-NEXT: s_cbranch_vccz .LBB0_5
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
; GCN-IR-NEXT: s_add_u32 s14, s12, 1
@@ -814,10 +808,7 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_mov_b64 s[4:5], 0
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s8, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s9, s3
-; GCN-IR-NEXT: s_add_i32 s8, s8, 32
-; GCN-IR-NEXT: s_min_u32 s12, s8, s9
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
; GCN-IR-NEXT: s_add_u32 s8, s12, 0xffffffc5
; GCN-IR-NEXT: s_addc_u32 s9, 0, -1
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
@@ -973,10 +964,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT: s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT: s_add_i32 s6, s6, 32
-; GCN-IR-NEXT: s_min_u32 s12, s6, s7
+; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[2:3]
; GCN-IR-NEXT: s_sub_u32 s8, 59, s12
; GCN-IR-NEXT: s_subb_u32 s9, 0, 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
More information about the llvm-commits
mailing list