[llvm] [AMDGPU] Analyze REG_SEQUENCE To Remove Redundant CMP Instructions (PR #167364)
Patrick Simmons via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 17:04:19 PDT 2026
https://github.com/linuxrocks123 updated https://github.com/llvm/llvm-project/pull/167364
>From 8c488f61a804f52948ca9d2ed63edca222f920f2 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 12:50:36 -0500
Subject: [PATCH 01/22] revised initial commit
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 24 ++++++++++++++++++++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +++-
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6e95f418b882a..8ad1f34ea53f4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1313,6 +1313,27 @@ Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
return Reg;
}
+MachineInstr *
+SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
+ const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+ int64_t SubRegValues[2];
+ bool SubRegIsConst[2];
+ MachineInstr *RealDefs[2];
+ for (unsigned I : {2, 4}) {
+ unsigned ArrayIdx = MI.getOperand(I).getImm() == AMDGPU::sub0 ? 0 : 1;
+ Register Subreg = MI.getOperand(I - 1).getReg();
+ RealDefs[ArrayIdx] = MRI.getUniqueVRegDef(Subreg);
+ SubRegIsConst[ArrayIdx] = getConstValDefinedInReg(
+ *RealDefs[ArrayIdx], Subreg, SubRegValues[ArrayIdx]);
+ }
+
+ for (unsigned I : {0, 1})
+ if (SubRegIsConst[I] && !SubRegValues[I])
+ return RealDefs[(I + 1) % 2];
+
+ return nullptr;
+}
+
bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
const Register Reg,
int64_t &ImmVal) const {
@@ -10997,6 +11018,9 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!Def)
return false;
+ if (MachineInstr *RegSequenceDef = pierceThroughRegSequence(*Def))
+ Def = RegSequenceDef;
+
// For S_OP that set SCC = DST!=0, do the transformation
//
// s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 5bc9f9674e567..dd4c519192110 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -730,7 +730,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
}
- static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
+ MachineInstr *pierceThroughRegSequence(const MachineInstr &MI) const;
+
+ static bool setsSCCifResultIsNonZero(const MachineInstr &MI) {
switch (MI.getOpcode()) {
case AMDGPU::S_ABSDIFF_I32:
case AMDGPU::S_ABS_I32:
>From 216ee0fb8cea14fc6ddf8075fda315eba682cffd Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 13:04:15 -0500
Subject: [PATCH 02/22] Fix
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 8ad1f34ea53f4..57be1cc004af2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1315,6 +1315,9 @@ Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
MachineInstr *
SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
+ if (MI.getOpcode() != AMDGPU::REG_SEQUENCE)
+ return nullptr;
+
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
int64_t SubRegValues[2];
bool SubRegIsConst[2];
>From 9f2e709c08677bde17e8f149f75fdbeeaf8c624f Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 13:06:06 -0500
Subject: [PATCH 03/22] Add testcase passing on main
---
.../AMDGPU/redundant-cmp-reg-sequence.ll | 23 +++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
diff --git a/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
new file mode 100644
index 0000000000000..00e479fe2eccc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s
+define amdgpu_ps i64 @ordertest(i64 inreg %val0) {
+; CHECK-LABEL: ordertest:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_lshr_b32 s0, s1, 2
+; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
+; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, s[0:1]
+; CHECK-NEXT: v_xor_b32_e32 v0, v2, v0
+; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: ; return to shader part epilog
+ %shl = lshr i64 %val0, 34
+ %result = and i64 %shl, 4294967295
+ %cmp = icmp ne i64 %result, 0
+ %zext = zext i1 %cmp to i64
+ %param0 = lshr i64 %shl, %zext
+ %param = and i64 %param0, 4294967295
+ %xory = xor i64 %zext, %param
+ ret i64 %xory
+}
>From 600f9263176bf2604861bf7421efaae50c999aff Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 13:06:34 -0500
Subject: [PATCH 04/22] Add testcase passing on branch
---
llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
index 00e479fe2eccc..750a9027d47a4 100644
--- a/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
+++ b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
@@ -4,9 +4,8 @@ define amdgpu_ps i64 @ordertest(i64 inreg %val0) {
; CHECK-LABEL: ordertest:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_lshr_b32 s0, s1, 2
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, s[0:1]
; CHECK-NEXT: v_xor_b32_e32 v0, v2, v0
>From b55c0456b0fba309e8dbb82a6b3f04c7f8801541 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 13:29:14 -0500
Subject: [PATCH 05/22] Update testcases
---
.../test/CodeGen/AMDGPU/carryout-selection.ll | 346 +++++++++---------
llvm/test/CodeGen/AMDGPU/s_cmp_0.ll | 4 -
llvm/test/CodeGen/AMDGPU/srem.ll | 98 ++---
llvm/test/CodeGen/AMDGPU/wave32.ll | 115 +++---
4 files changed, 263 insertions(+), 300 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 163d7ff9c61fc..356a69b6e8aee 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -2132,8 +2132,6 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
-; VI-NEXT: s_mov_b32 s6, 0
-; VI-NEXT: s_cmp_lg_u64 s[6:7], 0
; VI-NEXT: s_cbranch_scc0 .LBB16_3
; VI-NEXT: ; %bb.1:
; VI-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2284,8 +2282,6 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_or_b64 s[4:5], s[2:3], s[6:7]
-; GFX9-NEXT: s_mov_b32 s4, 0
-; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cbranch_scc0 .LBB16_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
@@ -2434,10 +2430,9 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_clause 0x1
; GFX1010-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1010-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX1010-NEXT: s_mov_b32 s8, 0
; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-NEXT: s_or_b64 s[4:5], s[2:3], s[6:7]
-; GFX1010-NEXT: s_mov_b32 s4, 0
-; GFX1010-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1010-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1010-NEXT: ; %bb.1:
; GFX1010-NEXT: v_cvt_f32_u32_e32 v0, s6
@@ -2452,71 +2447,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
; GFX1010-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1010-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1010-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010-NEXT: s_mul_i32 s11, s9, s5
-; GFX1010-NEXT: s_mul_hi_u32 s13, s9, s8
-; GFX1010-NEXT: s_mul_i32 s12, s10, s8
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1010-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1010-NEXT: s_mul_i32 s11, s9, s4
+; GFX1010-NEXT: s_mul_hi_u32 s13, s9, s5
+; GFX1010-NEXT: s_mul_i32 s12, s10, s5
; GFX1010-NEXT: s_add_i32 s11, s13, s11
-; GFX1010-NEXT: s_mul_i32 s14, s9, s8
+; GFX1010-NEXT: s_mul_i32 s14, s9, s5
; GFX1010-NEXT: s_add_i32 s11, s11, s12
-; GFX1010-NEXT: s_mul_hi_u32 s13, s8, s14
-; GFX1010-NEXT: s_mul_i32 s16, s8, s11
-; GFX1010-NEXT: s_mul_hi_u32 s15, s5, s14
-; GFX1010-NEXT: s_mul_i32 s12, s5, s14
-; GFX1010-NEXT: s_mul_hi_u32 s14, s8, s11
+; GFX1010-NEXT: s_mul_hi_u32 s13, s5, s14
+; GFX1010-NEXT: s_mul_i32 s16, s5, s11
+; GFX1010-NEXT: s_mul_hi_u32 s15, s4, s14
+; GFX1010-NEXT: s_mul_i32 s12, s4, s14
+; GFX1010-NEXT: s_mul_hi_u32 s14, s5, s11
; GFX1010-NEXT: s_add_u32 s13, s13, s16
; GFX1010-NEXT: s_addc_u32 s14, 0, s14
-; GFX1010-NEXT: s_mul_hi_u32 s17, s5, s11
+; GFX1010-NEXT: s_mul_hi_u32 s17, s4, s11
; GFX1010-NEXT: s_add_u32 s12, s13, s12
-; GFX1010-NEXT: s_mul_i32 s11, s5, s11
+; GFX1010-NEXT: s_mul_i32 s11, s4, s11
; GFX1010-NEXT: s_addc_u32 s12, s14, s15
; GFX1010-NEXT: s_addc_u32 s13, s17, 0
; GFX1010-NEXT: s_add_u32 s11, s12, s11
; GFX1010-NEXT: s_addc_u32 s12, 0, s13
-; GFX1010-NEXT: s_add_u32 s8, s8, s11
-; GFX1010-NEXT: s_addc_u32 s5, s5, s12
-; GFX1010-NEXT: s_mul_hi_u32 s11, s9, s8
-; GFX1010-NEXT: s_mul_i32 s12, s9, s8
-; GFX1010-NEXT: s_mul_i32 s9, s9, s5
-; GFX1010-NEXT: s_mul_i32 s10, s10, s8
+; GFX1010-NEXT: s_add_u32 s5, s5, s11
+; GFX1010-NEXT: s_addc_u32 s4, s4, s12
+; GFX1010-NEXT: s_mul_hi_u32 s11, s9, s5
+; GFX1010-NEXT: s_mul_i32 s12, s9, s5
+; GFX1010-NEXT: s_mul_i32 s9, s9, s4
+; GFX1010-NEXT: s_mul_i32 s10, s10, s5
; GFX1010-NEXT: s_add_i32 s9, s11, s9
-; GFX1010-NEXT: s_mul_i32 s11, s5, s12
+; GFX1010-NEXT: s_mul_i32 s11, s4, s12
; GFX1010-NEXT: s_add_i32 s9, s9, s10
-; GFX1010-NEXT: s_mul_hi_u32 s10, s8, s12
-; GFX1010-NEXT: s_mul_i32 s15, s8, s9
-; GFX1010-NEXT: s_mul_hi_u32 s14, s8, s9
+; GFX1010-NEXT: s_mul_hi_u32 s10, s5, s12
+; GFX1010-NEXT: s_mul_i32 s15, s5, s9
+; GFX1010-NEXT: s_mul_hi_u32 s14, s5, s9
; GFX1010-NEXT: s_add_u32 s10, s10, s15
-; GFX1010-NEXT: s_mul_hi_u32 s13, s5, s12
+; GFX1010-NEXT: s_mul_hi_u32 s13, s4, s12
; GFX1010-NEXT: s_addc_u32 s14, 0, s14
-; GFX1010-NEXT: s_mul_hi_u32 s12, s5, s9
+; GFX1010-NEXT: s_mul_hi_u32 s12, s4, s9
; GFX1010-NEXT: s_add_u32 s10, s10, s11
-; GFX1010-NEXT: s_mul_i32 s9, s5, s9
+; GFX1010-NEXT: s_mul_i32 s9, s4, s9
; GFX1010-NEXT: s_addc_u32 s10, s14, s13
; GFX1010-NEXT: s_addc_u32 s11, s12, 0
; GFX1010-NEXT: s_add_u32 s9, s10, s9
; GFX1010-NEXT: s_addc_u32 s10, 0, s11
-; GFX1010-NEXT: s_add_u32 s8, s8, s9
-; GFX1010-NEXT: s_addc_u32 s5, s5, s10
-; GFX1010-NEXT: s_mul_hi_u32 s9, s2, s8
-; GFX1010-NEXT: s_mul_i32 s12, s2, s5
-; GFX1010-NEXT: s_mul_hi_u32 s11, s2, s5
-; GFX1010-NEXT: s_mul_hi_u32 s10, s3, s8
-; GFX1010-NEXT: s_mul_i32 s8, s3, s8
+; GFX1010-NEXT: s_add_u32 s5, s5, s9
+; GFX1010-NEXT: s_addc_u32 s4, s4, s10
+; GFX1010-NEXT: s_mul_hi_u32 s9, s2, s5
+; GFX1010-NEXT: s_mul_i32 s12, s2, s4
+; GFX1010-NEXT: s_mul_hi_u32 s11, s2, s4
+; GFX1010-NEXT: s_mul_hi_u32 s10, s3, s5
+; GFX1010-NEXT: s_mul_i32 s5, s3, s5
; GFX1010-NEXT: s_add_u32 s9, s9, s12
; GFX1010-NEXT: s_addc_u32 s11, 0, s11
-; GFX1010-NEXT: s_mul_hi_u32 s13, s3, s5
-; GFX1010-NEXT: s_add_u32 s8, s9, s8
-; GFX1010-NEXT: s_mul_i32 s5, s3, s5
-; GFX1010-NEXT: s_addc_u32 s8, s11, s10
+; GFX1010-NEXT: s_mul_hi_u32 s13, s3, s4
+; GFX1010-NEXT: s_add_u32 s5, s9, s5
+; GFX1010-NEXT: s_mul_i32 s4, s3, s4
+; GFX1010-NEXT: s_addc_u32 s5, s11, s10
; GFX1010-NEXT: s_addc_u32 s9, s13, 0
-; GFX1010-NEXT: s_add_u32 s5, s8, s5
-; GFX1010-NEXT: s_addc_u32 s8, 0, s9
-; GFX1010-NEXT: s_mul_hi_u32 s9, s6, s5
-; GFX1010-NEXT: s_mul_i32 s10, s6, s8
-; GFX1010-NEXT: s_mul_i32 s11, s7, s5
-; GFX1010-NEXT: s_add_i32 s9, s9, s10
+; GFX1010-NEXT: s_add_u32 s4, s5, s4
+; GFX1010-NEXT: s_addc_u32 s5, 0, s9
+; GFX1010-NEXT: s_mul_hi_u32 s9, s6, s4
; GFX1010-NEXT: s_mul_i32 s10, s6, s5
+; GFX1010-NEXT: s_mul_i32 s11, s7, s4
+; GFX1010-NEXT: s_add_i32 s9, s9, s10
+; GFX1010-NEXT: s_mul_i32 s10, s6, s4
; GFX1010-NEXT: s_add_i32 s9, s9, s11
; GFX1010-NEXT: s_sub_i32 s11, s3, s9
; GFX1010-NEXT: s_sub_u32 s10, s2, s10
@@ -2530,10 +2525,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cselect_b32 s13, -1, 0
; GFX1010-NEXT: s_cmp_eq_u32 s11, s7
; GFX1010-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1010-NEXT: s_add_u32 s13, s5, 1
-; GFX1010-NEXT: s_addc_u32 s14, s8, 0
-; GFX1010-NEXT: s_add_u32 s15, s5, 2
-; GFX1010-NEXT: s_addc_u32 s16, s8, 0
+; GFX1010-NEXT: s_add_u32 s13, s4, 1
+; GFX1010-NEXT: s_addc_u32 s14, s5, 0
+; GFX1010-NEXT: s_add_u32 s15, s4, 2
+; GFX1010-NEXT: s_addc_u32 s16, s5, 0
; GFX1010-NEXT: s_cmp_lg_u32 s11, 0
; GFX1010-NEXT: s_cselect_b32 s11, s15, s13
; GFX1010-NEXT: s_cselect_b32 s13, s16, s14
@@ -2546,14 +2541,13 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cmp_eq_u32 s3, s7
; GFX1010-NEXT: s_cselect_b32 s3, s10, s9
; GFX1010-NEXT: s_cmp_lg_u32 s3, 0
-; GFX1010-NEXT: s_cselect_b32 s9, s13, s8
-; GFX1010-NEXT: s_cselect_b32 s8, s11, s5
-; GFX1010-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
+; GFX1010-NEXT: s_cselect_b32 s5, s13, s5
+; GFX1010-NEXT: s_cselect_b32 s4, s11, s4
+; GFX1010-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
; GFX1010-NEXT: s_cbranch_vccnz .LBB16_3
; GFX1010-NEXT: .LBB16_2:
; GFX1010-NEXT: v_cvt_f32_u32_e32 v0, s6
; GFX1010-NEXT: s_sub_i32 s4, 0, s6
-; GFX1010-NEXT: s_mov_b32 s9, 0
; GFX1010-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1010-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1010-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -2571,15 +2565,16 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cselect_b32 s2, s5, s2
; GFX1010-NEXT: s_add_i32 s4, s3, 1
; GFX1010-NEXT: s_cmp_ge_u32 s2, s6
-; GFX1010-NEXT: s_cselect_b32 s8, s4, s3
+; GFX1010-NEXT: s_mov_b32 s5, 0
+; GFX1010-NEXT: s_cselect_b32 s4, s4, s3
; GFX1010-NEXT: .LBB16_3:
-; GFX1010-NEXT: v_mov_b32_e32 v0, s8
+; GFX1010-NEXT: v_mov_b32_e32 v0, s4
; GFX1010-NEXT: v_mov_b32_e32 v2, 0
-; GFX1010-NEXT: v_mov_b32_e32 v1, s9
+; GFX1010-NEXT: v_mov_b32_e32 v1, s5
; GFX1010-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-NEXT: s_endpgm
; GFX1010-NEXT: .LBB16_4:
-; GFX1010-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX1010-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX1010-NEXT: s_branch .LBB16_2
;
; GFX1030W32-LABEL: sudiv64:
@@ -2587,10 +2582,9 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_clause 0x1
; GFX1030W32-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1030W32-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX1030W32-NEXT: s_mov_b32 s8, 0
; GFX1030W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX1030W32-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
-; GFX1030W32-NEXT: s_mov_b32 s6, 0
-; GFX1030W32-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX1030W32-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1030W32-NEXT: ; %bb.1:
; GFX1030W32-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2605,71 +2599,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: v_fmamk_f32 v0, v1, 0xcf800000, v0
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1030W32-NEXT: v_readfirstlane_b32 s7, v1
-; GFX1030W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1030W32-NEXT: s_mul_i32 s11, s9, s7
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s9, s8
-; GFX1030W32-NEXT: s_mul_i32 s12, s10, s8
+; GFX1030W32-NEXT: v_readfirstlane_b32 s6, v1
+; GFX1030W32-NEXT: v_readfirstlane_b32 s7, v0
+; GFX1030W32-NEXT: s_mul_i32 s11, s9, s6
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s12, s10, s7
; GFX1030W32-NEXT: s_add_i32 s11, s13, s11
-; GFX1030W32-NEXT: s_mul_i32 s14, s9, s8
+; GFX1030W32-NEXT: s_mul_i32 s14, s9, s7
; GFX1030W32-NEXT: s_add_i32 s11, s11, s12
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s8, s14
-; GFX1030W32-NEXT: s_mul_i32 s16, s8, s11
-; GFX1030W32-NEXT: s_mul_hi_u32 s15, s7, s14
-; GFX1030W32-NEXT: s_mul_i32 s12, s7, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s14, s8, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s7, s14
+; GFX1030W32-NEXT: s_mul_i32 s16, s7, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s15, s6, s14
+; GFX1030W32-NEXT: s_mul_i32 s12, s6, s14
+; GFX1030W32-NEXT: s_mul_hi_u32 s14, s7, s11
; GFX1030W32-NEXT: s_add_u32 s13, s13, s16
; GFX1030W32-NEXT: s_addc_u32 s14, 0, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s17, s7, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s17, s6, s11
; GFX1030W32-NEXT: s_add_u32 s12, s13, s12
-; GFX1030W32-NEXT: s_mul_i32 s11, s7, s11
+; GFX1030W32-NEXT: s_mul_i32 s11, s6, s11
; GFX1030W32-NEXT: s_addc_u32 s12, s14, s15
; GFX1030W32-NEXT: s_addc_u32 s13, s17, 0
; GFX1030W32-NEXT: s_add_u32 s11, s12, s11
; GFX1030W32-NEXT: s_addc_u32 s12, 0, s13
-; GFX1030W32-NEXT: s_add_u32 s8, s8, s11
-; GFX1030W32-NEXT: s_addc_u32 s7, s7, s12
-; GFX1030W32-NEXT: s_mul_hi_u32 s11, s9, s8
-; GFX1030W32-NEXT: s_mul_i32 s12, s9, s8
-; GFX1030W32-NEXT: s_mul_i32 s9, s9, s7
-; GFX1030W32-NEXT: s_mul_i32 s10, s10, s8
+; GFX1030W32-NEXT: s_add_u32 s7, s7, s11
+; GFX1030W32-NEXT: s_addc_u32 s6, s6, s12
+; GFX1030W32-NEXT: s_mul_hi_u32 s11, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s12, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s9, s9, s6
+; GFX1030W32-NEXT: s_mul_i32 s10, s10, s7
; GFX1030W32-NEXT: s_add_i32 s9, s11, s9
-; GFX1030W32-NEXT: s_mul_i32 s11, s7, s12
+; GFX1030W32-NEXT: s_mul_i32 s11, s6, s12
; GFX1030W32-NEXT: s_add_i32 s9, s9, s10
-; GFX1030W32-NEXT: s_mul_hi_u32 s10, s8, s12
-; GFX1030W32-NEXT: s_mul_i32 s15, s8, s9
-; GFX1030W32-NEXT: s_mul_hi_u32 s14, s8, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s10, s7, s12
+; GFX1030W32-NEXT: s_mul_i32 s15, s7, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s14, s7, s9
; GFX1030W32-NEXT: s_add_u32 s10, s10, s15
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s7, s12
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s6, s12
; GFX1030W32-NEXT: s_addc_u32 s14, 0, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s12, s7, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s12, s6, s9
; GFX1030W32-NEXT: s_add_u32 s10, s10, s11
-; GFX1030W32-NEXT: s_mul_i32 s9, s7, s9
+; GFX1030W32-NEXT: s_mul_i32 s9, s6, s9
; GFX1030W32-NEXT: s_addc_u32 s10, s14, s13
; GFX1030W32-NEXT: s_addc_u32 s11, s12, 0
; GFX1030W32-NEXT: s_add_u32 s9, s10, s9
; GFX1030W32-NEXT: s_addc_u32 s10, 0, s11
-; GFX1030W32-NEXT: s_add_u32 s8, s8, s9
-; GFX1030W32-NEXT: s_addc_u32 s7, s7, s10
-; GFX1030W32-NEXT: s_mul_hi_u32 s9, s2, s8
-; GFX1030W32-NEXT: s_mul_i32 s12, s2, s7
-; GFX1030W32-NEXT: s_mul_hi_u32 s11, s2, s7
-; GFX1030W32-NEXT: s_mul_hi_u32 s10, s3, s8
-; GFX1030W32-NEXT: s_mul_i32 s8, s3, s8
+; GFX1030W32-NEXT: s_add_u32 s7, s7, s9
+; GFX1030W32-NEXT: s_addc_u32 s6, s6, s10
+; GFX1030W32-NEXT: s_mul_hi_u32 s9, s2, s7
+; GFX1030W32-NEXT: s_mul_i32 s12, s2, s6
+; GFX1030W32-NEXT: s_mul_hi_u32 s11, s2, s6
+; GFX1030W32-NEXT: s_mul_hi_u32 s10, s3, s7
+; GFX1030W32-NEXT: s_mul_i32 s7, s3, s7
; GFX1030W32-NEXT: s_add_u32 s9, s9, s12
; GFX1030W32-NEXT: s_addc_u32 s11, 0, s11
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s3, s7
-; GFX1030W32-NEXT: s_add_u32 s8, s9, s8
-; GFX1030W32-NEXT: s_mul_i32 s7, s3, s7
-; GFX1030W32-NEXT: s_addc_u32 s8, s11, s10
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s3, s6
+; GFX1030W32-NEXT: s_add_u32 s7, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s6, s3, s6
+; GFX1030W32-NEXT: s_addc_u32 s7, s11, s10
; GFX1030W32-NEXT: s_addc_u32 s9, s13, 0
-; GFX1030W32-NEXT: s_add_u32 s7, s8, s7
-; GFX1030W32-NEXT: s_addc_u32 s8, 0, s9
-; GFX1030W32-NEXT: s_mul_hi_u32 s9, s4, s7
-; GFX1030W32-NEXT: s_mul_i32 s10, s4, s8
-; GFX1030W32-NEXT: s_mul_i32 s11, s5, s7
-; GFX1030W32-NEXT: s_add_i32 s9, s9, s10
+; GFX1030W32-NEXT: s_add_u32 s6, s7, s6
+; GFX1030W32-NEXT: s_addc_u32 s7, 0, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s9, s4, s6
; GFX1030W32-NEXT: s_mul_i32 s10, s4, s7
+; GFX1030W32-NEXT: s_mul_i32 s11, s5, s6
+; GFX1030W32-NEXT: s_add_i32 s9, s9, s10
+; GFX1030W32-NEXT: s_mul_i32 s10, s4, s6
; GFX1030W32-NEXT: s_add_i32 s9, s9, s11
; GFX1030W32-NEXT: s_sub_i32 s11, s3, s9
; GFX1030W32-NEXT: s_sub_u32 s10, s2, s10
@@ -2683,10 +2677,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cselect_b32 s13, -1, 0
; GFX1030W32-NEXT: s_cmp_eq_u32 s11, s5
; GFX1030W32-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1030W32-NEXT: s_add_u32 s13, s7, 1
-; GFX1030W32-NEXT: s_addc_u32 s14, s8, 0
-; GFX1030W32-NEXT: s_add_u32 s15, s7, 2
-; GFX1030W32-NEXT: s_addc_u32 s16, s8, 0
+; GFX1030W32-NEXT: s_add_u32 s13, s6, 1
+; GFX1030W32-NEXT: s_addc_u32 s14, s7, 0
+; GFX1030W32-NEXT: s_add_u32 s15, s6, 2
+; GFX1030W32-NEXT: s_addc_u32 s16, s7, 0
; GFX1030W32-NEXT: s_cmp_lg_u32 s11, 0
; GFX1030W32-NEXT: s_cselect_b32 s11, s15, s13
; GFX1030W32-NEXT: s_cselect_b32 s13, s16, s14
@@ -2699,14 +2693,14 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cmp_eq_u32 s3, s5
; GFX1030W32-NEXT: s_cselect_b32 s3, s10, s9
; GFX1030W32-NEXT: s_cmp_lg_u32 s3, 0
-; GFX1030W32-NEXT: s_cselect_b32 s9, s13, s8
-; GFX1030W32-NEXT: s_cselect_b32 s8, s11, s7
-; GFX1030W32-NEXT: s_andn2_b32 vcc_lo, exec_lo, s6
+; GFX1030W32-NEXT: s_cselect_b32 s7, s13, s7
+; GFX1030W32-NEXT: s_cselect_b32 s6, s11, s6
+; GFX1030W32-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
; GFX1030W32-NEXT: s_cbranch_vccnz .LBB16_3
; GFX1030W32-NEXT: .LBB16_2:
; GFX1030W32-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX1030W32-NEXT: s_sub_i32 s5, 0, s4
-; GFX1030W32-NEXT: s_mov_b32 s9, 0
+; GFX1030W32-NEXT: s_mov_b32 s7, 0
; GFX1030W32-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1030W32-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -2724,15 +2718,15 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cselect_b32 s2, s6, s2
; GFX1030W32-NEXT: s_add_i32 s5, s3, 1
; GFX1030W32-NEXT: s_cmp_ge_u32 s2, s4
-; GFX1030W32-NEXT: s_cselect_b32 s8, s5, s3
+; GFX1030W32-NEXT: s_cselect_b32 s6, s5, s3
; GFX1030W32-NEXT: .LBB16_3:
-; GFX1030W32-NEXT: v_mov_b32_e32 v0, s8
+; GFX1030W32-NEXT: v_mov_b32_e32 v0, s6
; GFX1030W32-NEXT: v_mov_b32_e32 v2, 0
-; GFX1030W32-NEXT: v_mov_b32_e32 v1, s9
+; GFX1030W32-NEXT: v_mov_b32_e32 v1, s7
; GFX1030W32-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1030W32-NEXT: s_endpgm
; GFX1030W32-NEXT: .LBB16_4:
-; GFX1030W32-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX1030W32-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX1030W32-NEXT: s_branch .LBB16_2
;
; GFX1030W64-LABEL: sudiv64:
@@ -2742,8 +2736,6 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W64-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX1030W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX1030W64-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
-; GFX1030W64-NEXT: s_mov_b32 s6, 0
-; GFX1030W64-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX1030W64-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1030W64-NEXT: ; %bb.1:
; GFX1030W64-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2892,11 +2884,9 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT: s_mov_b32 s8, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
-; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX11-NEXT: s_cbranch_scc0 .LBB16_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2916,71 +2906,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s7, v1
-; GFX11-NEXT: v_readfirstlane_b32 s8, v0
-; GFX11-NEXT: s_mul_i32 s11, s9, s7
-; GFX11-NEXT: s_mul_hi_u32 s13, s9, s8
-; GFX11-NEXT: s_mul_i32 s12, s10, s8
+; GFX11-NEXT: v_readfirstlane_b32 s6, v1
+; GFX11-NEXT: v_readfirstlane_b32 s7, v0
+; GFX11-NEXT: s_mul_i32 s11, s9, s6
+; GFX11-NEXT: s_mul_hi_u32 s13, s9, s7
+; GFX11-NEXT: s_mul_i32 s12, s10, s7
; GFX11-NEXT: s_add_i32 s11, s13, s11
-; GFX11-NEXT: s_mul_i32 s14, s9, s8
+; GFX11-NEXT: s_mul_i32 s14, s9, s7
; GFX11-NEXT: s_add_i32 s11, s11, s12
-; GFX11-NEXT: s_mul_hi_u32 s13, s8, s14
-; GFX11-NEXT: s_mul_i32 s16, s8, s11
-; GFX11-NEXT: s_mul_hi_u32 s15, s7, s14
-; GFX11-NEXT: s_mul_i32 s12, s7, s14
-; GFX11-NEXT: s_mul_hi_u32 s14, s8, s11
+; GFX11-NEXT: s_mul_hi_u32 s13, s7, s14
+; GFX11-NEXT: s_mul_i32 s16, s7, s11
+; GFX11-NEXT: s_mul_hi_u32 s15, s6, s14
+; GFX11-NEXT: s_mul_i32 s12, s6, s14
+; GFX11-NEXT: s_mul_hi_u32 s14, s7, s11
; GFX11-NEXT: s_add_u32 s13, s13, s16
; GFX11-NEXT: s_addc_u32 s14, 0, s14
-; GFX11-NEXT: s_mul_hi_u32 s17, s7, s11
+; GFX11-NEXT: s_mul_hi_u32 s17, s6, s11
; GFX11-NEXT: s_add_u32 s12, s13, s12
-; GFX11-NEXT: s_mul_i32 s11, s7, s11
+; GFX11-NEXT: s_mul_i32 s11, s6, s11
; GFX11-NEXT: s_addc_u32 s12, s14, s15
; GFX11-NEXT: s_addc_u32 s13, s17, 0
; GFX11-NEXT: s_add_u32 s11, s12, s11
; GFX11-NEXT: s_addc_u32 s12, 0, s13
-; GFX11-NEXT: s_add_u32 s8, s8, s11
-; GFX11-NEXT: s_addc_u32 s7, s7, s12
-; GFX11-NEXT: s_mul_hi_u32 s11, s9, s8
-; GFX11-NEXT: s_mul_i32 s12, s9, s8
-; GFX11-NEXT: s_mul_i32 s9, s9, s7
-; GFX11-NEXT: s_mul_i32 s10, s10, s8
+; GFX11-NEXT: s_add_u32 s7, s7, s11
+; GFX11-NEXT: s_addc_u32 s6, s6, s12
+; GFX11-NEXT: s_mul_hi_u32 s11, s9, s7
+; GFX11-NEXT: s_mul_i32 s12, s9, s7
+; GFX11-NEXT: s_mul_i32 s9, s9, s6
+; GFX11-NEXT: s_mul_i32 s10, s10, s7
; GFX11-NEXT: s_add_i32 s9, s11, s9
-; GFX11-NEXT: s_mul_i32 s11, s7, s12
+; GFX11-NEXT: s_mul_i32 s11, s6, s12
; GFX11-NEXT: s_add_i32 s9, s9, s10
-; GFX11-NEXT: s_mul_hi_u32 s10, s8, s12
-; GFX11-NEXT: s_mul_i32 s15, s8, s9
-; GFX11-NEXT: s_mul_hi_u32 s14, s8, s9
+; GFX11-NEXT: s_mul_hi_u32 s10, s7, s12
+; GFX11-NEXT: s_mul_i32 s15, s7, s9
+; GFX11-NEXT: s_mul_hi_u32 s14, s7, s9
; GFX11-NEXT: s_add_u32 s10, s10, s15
-; GFX11-NEXT: s_mul_hi_u32 s13, s7, s12
+; GFX11-NEXT: s_mul_hi_u32 s13, s6, s12
; GFX11-NEXT: s_addc_u32 s14, 0, s14
-; GFX11-NEXT: s_mul_hi_u32 s12, s7, s9
+; GFX11-NEXT: s_mul_hi_u32 s12, s6, s9
; GFX11-NEXT: s_add_u32 s10, s10, s11
-; GFX11-NEXT: s_mul_i32 s9, s7, s9
+; GFX11-NEXT: s_mul_i32 s9, s6, s9
; GFX11-NEXT: s_addc_u32 s10, s14, s13
; GFX11-NEXT: s_addc_u32 s11, s12, 0
; GFX11-NEXT: s_add_u32 s9, s10, s9
; GFX11-NEXT: s_addc_u32 s10, 0, s11
-; GFX11-NEXT: s_add_u32 s8, s8, s9
-; GFX11-NEXT: s_addc_u32 s7, s7, s10
-; GFX11-NEXT: s_mul_hi_u32 s9, s2, s8
-; GFX11-NEXT: s_mul_i32 s12, s2, s7
-; GFX11-NEXT: s_mul_hi_u32 s11, s2, s7
-; GFX11-NEXT: s_mul_hi_u32 s10, s3, s8
-; GFX11-NEXT: s_mul_i32 s8, s3, s8
+; GFX11-NEXT: s_add_u32 s7, s7, s9
+; GFX11-NEXT: s_addc_u32 s6, s6, s10
+; GFX11-NEXT: s_mul_hi_u32 s9, s2, s7
+; GFX11-NEXT: s_mul_i32 s12, s2, s6
+; GFX11-NEXT: s_mul_hi_u32 s11, s2, s6
+; GFX11-NEXT: s_mul_hi_u32 s10, s3, s7
+; GFX11-NEXT: s_mul_i32 s7, s3, s7
; GFX11-NEXT: s_add_u32 s9, s9, s12
; GFX11-NEXT: s_addc_u32 s11, 0, s11
-; GFX11-NEXT: s_mul_hi_u32 s13, s3, s7
-; GFX11-NEXT: s_add_u32 s8, s9, s8
-; GFX11-NEXT: s_mul_i32 s7, s3, s7
-; GFX11-NEXT: s_addc_u32 s8, s11, s10
+; GFX11-NEXT: s_mul_hi_u32 s13, s3, s6
+; GFX11-NEXT: s_add_u32 s7, s9, s7
+; GFX11-NEXT: s_mul_i32 s6, s3, s6
+; GFX11-NEXT: s_addc_u32 s7, s11, s10
; GFX11-NEXT: s_addc_u32 s9, s13, 0
-; GFX11-NEXT: s_add_u32 s7, s8, s7
-; GFX11-NEXT: s_addc_u32 s8, 0, s9
-; GFX11-NEXT: s_mul_hi_u32 s9, s4, s7
-; GFX11-NEXT: s_mul_i32 s10, s4, s8
-; GFX11-NEXT: s_mul_i32 s11, s5, s7
-; GFX11-NEXT: s_add_i32 s9, s9, s10
+; GFX11-NEXT: s_add_u32 s6, s7, s6
+; GFX11-NEXT: s_addc_u32 s7, 0, s9
+; GFX11-NEXT: s_mul_hi_u32 s9, s4, s6
; GFX11-NEXT: s_mul_i32 s10, s4, s7
+; GFX11-NEXT: s_mul_i32 s11, s5, s6
+; GFX11-NEXT: s_add_i32 s9, s9, s10
+; GFX11-NEXT: s_mul_i32 s10, s4, s6
; GFX11-NEXT: s_add_i32 s9, s9, s11
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_sub_i32 s11, s3, s9
@@ -2996,10 +2986,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s13, -1, 0
; GFX11-NEXT: s_cmp_eq_u32 s11, s5
; GFX11-NEXT: s_cselect_b32 s11, s13, s14
-; GFX11-NEXT: s_add_u32 s13, s7, 1
-; GFX11-NEXT: s_addc_u32 s14, s8, 0
-; GFX11-NEXT: s_add_u32 s15, s7, 2
-; GFX11-NEXT: s_addc_u32 s16, s8, 0
+; GFX11-NEXT: s_add_u32 s13, s6, 1
+; GFX11-NEXT: s_addc_u32 s14, s7, 0
+; GFX11-NEXT: s_add_u32 s15, s6, 2
+; GFX11-NEXT: s_addc_u32 s16, s7, 0
; GFX11-NEXT: s_cmp_lg_u32 s11, 0
; GFX11-NEXT: s_cselect_b32 s11, s15, s13
; GFX11-NEXT: s_cselect_b32 s13, s16, s14
@@ -3014,14 +3004,14 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s3, s10, s9
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s9, s13, s8
-; GFX11-NEXT: s_cselect_b32 s8, s11, s7
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
+; GFX11-NEXT: s_cselect_b32 s7, s13, s7
+; GFX11-NEXT: s_cselect_b32 s6, s11, s6
+; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
; GFX11-NEXT: s_cbranch_vccnz .LBB16_3
; GFX11-NEXT: .LBB16_2:
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX11-NEXT: s_sub_i32 s5, 0, s4
-; GFX11-NEXT: s_mov_b32 s9, 0
+; GFX11-NEXT: s_mov_b32 s7, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -3044,15 +3034,15 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s2, s6, s2
; GFX11-NEXT: s_add_i32 s5, s3, 1
; GFX11-NEXT: s_cmp_ge_u32 s2, s4
-; GFX11-NEXT: s_cselect_b32 s8, s5, s3
+; GFX11-NEXT: s_cselect_b32 s6, s5, s3
; GFX11-NEXT: .LBB16_3:
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s8
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_mov_b32_e32 v0, s6
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
; GFX11-NEXT: .LBB16_4:
-; GFX11-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr6_sgpr7
; GFX11-NEXT: s_branch .LBB16_2
;
; GFX1250-LABEL: sudiv64:
diff --git a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
index d66b9029dbf99..cacf3d3c38ee3 100644
--- a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
@@ -489,8 +489,6 @@ define amdgpu_ps i32 @bfe_i64(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_bfe_i64 s[2:3], s[0:1], 0x80000
; CHECK-NEXT: s_and_b32 s0, s0, 0xff
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
@@ -526,7 +524,6 @@ define amdgpu_ps i32 @bfe_u64(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_and_b32 s0, s0, 0xff
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; use s[0:1]
; CHECK-NEXT: ;;#ASMEND
@@ -608,7 +605,6 @@ define amdgpu_ps i32 @bcnt164(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; use s[0:1]
; CHECK-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/srem.ll b/llvm/test/CodeGen/AMDGPU/srem.ll
index e12e31b14e97d..3c3d634c96410 100644
--- a/llvm/test/CodeGen/AMDGPU/srem.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem.ll
@@ -1501,8 +1501,6 @@ define amdgpu_kernel void @srem_i64(ptr addrspace(1) %out, ptr addrspace(1) %in)
; GCN-NEXT: v_readfirstlane_b32 s3, v3
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: s_or_b64 s[6:7], s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
; GCN-NEXT: s_cbranch_scc0 .LBB8_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_ashr_i32 s6, s3, 31
@@ -1832,8 +1830,6 @@ define amdgpu_kernel void @srem_i64(ptr addrspace(1) %out, ptr addrspace(1) %in)
; TONGA-NEXT: v_readfirstlane_b32 s3, v3
; TONGA-NEXT: v_readfirstlane_b32 s2, v2
; TONGA-NEXT: s_or_b64 s[6:7], s[4:5], s[2:3]
-; TONGA-NEXT: s_mov_b32 s6, 0
-; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB8_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s3, 31
@@ -2701,12 +2697,10 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s11, v5
; GCN-NEXT: v_readfirstlane_b32 s10, v4
-; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[8:9]
-; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_readfirstlane_b32 s3, v3
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: v_readfirstlane_b32 s5, v7
-; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
+; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[8:9]
; GCN-NEXT: v_readfirstlane_b32 s4, v6
; GCN-NEXT: s_cbranch_scc0 .LBB10_6
; GCN-NEXT: ; %bb.1:
@@ -2855,8 +2849,6 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_cselect_b32 s6, s9, s6
; GCN-NEXT: .LBB10_3:
; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s8, 0
-; GCN-NEXT: s_cmp_lg_u64 s[8:9], 0
; GCN-NEXT: s_cbranch_scc0 .LBB10_7
; GCN-NEXT: ; %bb.4:
; GCN-NEXT: s_ashr_i32 s8, s3, 31
@@ -3344,8 +3336,6 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: v_readfirstlane_b32 s3, v5
; TONGA-NEXT: v_readfirstlane_b32 s2, v4
; TONGA-NEXT: s_or_b64 s[6:7], s[2:3], s[0:1]
-; TONGA-NEXT: s_mov_b32 s6, 0
-; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB10_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s1, 31
@@ -4878,8 +4868,6 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s19, v13
; GCN-NEXT: v_readfirstlane_b32 s18, v12
-; GCN-NEXT: s_or_b64 s[6:7], s[18:19], s[16:17]
-; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: v_readfirstlane_b32 s9, v1
; GCN-NEXT: v_readfirstlane_b32 s8, v0
@@ -4890,9 +4878,9 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_readfirstlane_b32 s11, v9
; GCN-NEXT: v_readfirstlane_b32 s10, v8
; GCN-NEXT: v_readfirstlane_b32 s15, v15
-; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
+; GCN-NEXT: s_or_b64 s[6:7], s[18:19], s[16:17]
; GCN-NEXT: v_readfirstlane_b32 s14, v14
-; GCN-NEXT: s_cbranch_scc0 .LBB12_6
+; GCN-NEXT: s_cbranch_scc0 .LBB12_10
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_ashr_i32 s6, s17, 31
; GCN-NEXT: s_add_u32 s20, s16, s6
@@ -5039,9 +5027,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_cselect_b32 s6, s17, s6
; GCN-NEXT: .LBB12_3:
; GCN-NEXT: s_or_b64 s[16:17], s[14:15], s[12:13]
-; GCN-NEXT: s_mov_b32 s16, 0
-; GCN-NEXT: s_cmp_lg_u64 s[16:17], 0
-; GCN-NEXT: s_cbranch_scc0 .LBB12_7
+; GCN-NEXT: s_cbranch_scc0 .LBB12_11
; GCN-NEXT: ; %bb.4:
; GCN-NEXT: s_ashr_i32 s16, s13, 31
; GCN-NEXT: s_add_u32 s18, s12, s16
@@ -5165,7 +5151,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[18:19], s[18:19], s[20:21]
; GCN-NEXT: s_sub_u32 s18, s18, s20
; GCN-NEXT: s_subb_u32 s19, s19, s20
-; GCN-NEXT: s_cbranch_execnz .LBB12_8
+; GCN-NEXT: s_cbranch_execnz .LBB12_12
; GCN-NEXT: .LBB12_5:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s12
; GCN-NEXT: s_sub_i32 s13, 0, s12
@@ -5185,22 +5171,35 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s12, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s12, v0
; GCN-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-NEXT: s_branch .LBB12_9
+; GCN-NEXT: s_or_b64 s[12:13], s[10:11], s[8:9]
+; GCN-NEXT: s_cbranch_scc1 .LBB12_13
; GCN-NEXT: .LBB12_6:
+; GCN-NEXT: ; implicit-def: $sgpr14_sgpr15
+; GCN-NEXT: s_branch .LBB12_14
+; GCN-NEXT: .LBB12_7:
+; GCN-NEXT: v_mov_b32_e32 v4, s14
+; GCN-NEXT: v_mov_b32_e32 v5, s15
+; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
+; GCN-NEXT: s_cbranch_scc1 .LBB12_15
+; GCN-NEXT: .LBB12_8:
+; GCN-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GCN-NEXT: s_branch .LBB12_16
+; GCN-NEXT: .LBB12_9:
+; GCN-NEXT: v_mov_b32_e32 v6, s10
+; GCN-NEXT: v_mov_b32_e32 v7, s11
+; GCN-NEXT: s_branch .LBB12_17
+; GCN-NEXT: .LBB12_10:
; GCN-NEXT: ; implicit-def: $sgpr6_sgpr7
; GCN-NEXT: s_branch .LBB12_2
-; GCN-NEXT: .LBB12_7:
+; GCN-NEXT: .LBB12_11:
; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19
; GCN-NEXT: s_branch .LBB12_5
-; GCN-NEXT: .LBB12_8:
+; GCN-NEXT: .LBB12_12:
; GCN-NEXT: v_mov_b32_e32 v2, s18
; GCN-NEXT: v_mov_b32_e32 v3, s19
-; GCN-NEXT: .LBB12_9:
; GCN-NEXT: s_or_b64 s[12:13], s[10:11], s[8:9]
-; GCN-NEXT: s_mov_b32 s12, 0
-; GCN-NEXT: s_cmp_lg_u64 s[12:13], 0
-; GCN-NEXT: s_cbranch_scc0 .LBB12_12
-; GCN-NEXT: ; %bb.10:
+; GCN-NEXT: s_cbranch_scc0 .LBB12_6
+; GCN-NEXT: .LBB12_13:
; GCN-NEXT: s_ashr_i32 s12, s9, 31
; GCN-NEXT: s_add_u32 s14, s8, s12
; GCN-NEXT: s_mov_b32 s13, s12
@@ -5323,8 +5322,8 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[14:15], s[14:15], s[16:17]
; GCN-NEXT: s_sub_u32 s14, s14, s16
; GCN-NEXT: s_subb_u32 s15, s15, s16
-; GCN-NEXT: s_cbranch_execnz .LBB12_13
-; GCN-NEXT: .LBB12_11:
+; GCN-NEXT: s_cbranch_execnz .LBB12_7
+; GCN-NEXT: .LBB12_14:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8
; GCN-NEXT: s_sub_i32 s9, 0, s8
; GCN-NEXT: v_mov_b32_e32 v5, 0
@@ -5343,19 +5342,9 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s8, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s8, v0
; GCN-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; GCN-NEXT: s_branch .LBB12_14
-; GCN-NEXT: .LBB12_12:
-; GCN-NEXT: ; implicit-def: $sgpr14_sgpr15
-; GCN-NEXT: s_branch .LBB12_11
-; GCN-NEXT: .LBB12_13:
-; GCN-NEXT: v_mov_b32_e32 v4, s14
-; GCN-NEXT: v_mov_b32_e32 v5, s15
-; GCN-NEXT: .LBB12_14:
; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s8, 0
-; GCN-NEXT: s_cmp_lg_u64 s[8:9], 0
-; GCN-NEXT: s_cbranch_scc0 .LBB12_17
-; GCN-NEXT: ; %bb.15:
+; GCN-NEXT: s_cbranch_scc0 .LBB12_8
+; GCN-NEXT: .LBB12_15:
; GCN-NEXT: s_ashr_i32 s8, s3, 31
; GCN-NEXT: s_add_u32 s10, s2, s8
; GCN-NEXT: s_mov_b32 s9, s8
@@ -5478,7 +5467,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[10:11], s[10:11], s[12:13]
; GCN-NEXT: s_sub_u32 s10, s10, s12
; GCN-NEXT: s_subb_u32 s11, s11, s12
-; GCN-NEXT: s_cbranch_execnz .LBB12_18
+; GCN-NEXT: s_cbranch_execnz .LBB12_9
; GCN-NEXT: .LBB12_16:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2
; GCN-NEXT: s_sub_i32 s3, 0, s2
@@ -5498,14 +5487,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s2, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0
; GCN-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
-; GCN-NEXT: s_branch .LBB12_19
; GCN-NEXT: .LBB12_17:
-; GCN-NEXT: ; implicit-def: $sgpr10_sgpr11
-; GCN-NEXT: s_branch .LBB12_16
-; GCN-NEXT: .LBB12_18:
-; GCN-NEXT: v_mov_b32_e32 v6, s10
-; GCN-NEXT: v_mov_b32_e32 v7, s11
-; GCN-NEXT: .LBB12_19:
; GCN-NEXT: v_mov_b32_e32 v8, 0
; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: v_mov_b32_e32 v1, s7
@@ -6119,23 +6101,23 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA: ; %bb.0:
; TONGA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: s_add_u32 s0, s6, 48
; TONGA-NEXT: v_mov_b32_e32 v0, s6
-; TONGA-NEXT: s_addc_u32 s1, s7, 0
+; TONGA-NEXT: s_add_u32 s0, s6, 48
; TONGA-NEXT: v_mov_b32_e32 v1, s7
-; TONGA-NEXT: s_add_u32 s2, s6, 32
+; TONGA-NEXT: s_addc_u32 s1, s7, 0
; TONGA-NEXT: flat_load_dwordx4 v[14:17], v[0:1]
-; TONGA-NEXT: s_addc_u32 s3, s7, 0
-; TONGA-NEXT: v_mov_b32_e32 v0, s2
-; TONGA-NEXT: v_mov_b32_e32 v1, s3
-; TONGA-NEXT: flat_load_dwordx4 v[10:13], v[0:1]
; TONGA-NEXT: v_mov_b32_e32 v0, s0
; TONGA-NEXT: v_mov_b32_e32 v1, s1
+; TONGA-NEXT: s_add_u32 s0, s6, 32
+; TONGA-NEXT: s_addc_u32 s1, s7, 0
+; TONGA-NEXT: v_mov_b32_e32 v3, s1
+; TONGA-NEXT: v_mov_b32_e32 v2, s0
; TONGA-NEXT: s_add_u32 s0, s6, 16
; TONGA-NEXT: s_addc_u32 s1, s7, 0
; TONGA-NEXT: v_mov_b32_e32 v5, s1
-; TONGA-NEXT: v_mov_b32_e32 v4, s0
+; TONGA-NEXT: flat_load_dwordx4 v[10:13], v[2:3]
; TONGA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; TONGA-NEXT: v_mov_b32_e32 v4, s0
; TONGA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; TONGA-NEXT: s_waitcnt vmcnt(3)
; TONGA-NEXT: v_readfirstlane_b32 s3, v15
@@ -6144,8 +6126,6 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: v_readfirstlane_b32 s1, v11
; TONGA-NEXT: v_readfirstlane_b32 s0, v10
; TONGA-NEXT: s_or_b64 s[6:7], s[2:3], s[0:1]
-; TONGA-NEXT: s_mov_b32 s6, 0
-; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB12_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s1, 31
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 8e7dc4072c012..5f82a5e4b2fa1 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -731,12 +731,11 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-LABEL: test_udiv64:
; GFX1032: ; %bb.0: ; %bb
; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX1032-NEXT: s_mov_b32 s8, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_or_b64 s[4:5], s[2:3], s[0:1]
-; GFX1032-NEXT: s_mov_b32 s4, 0
-; GFX1032-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1032-NEXT: s_cbranch_scc0 .LBB15_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: v_cvt_f32_u32_e32 v0, s0
@@ -751,71 +750,71 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
; GFX1032-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1032-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1032-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1032-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1032-NEXT: s_mul_i32 s11, s9, s5
-; GFX1032-NEXT: s_mul_hi_u32 s13, s9, s8
-; GFX1032-NEXT: s_mul_i32 s12, s10, s8
+; GFX1032-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1032-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1032-NEXT: s_mul_i32 s11, s9, s4
+; GFX1032-NEXT: s_mul_hi_u32 s13, s9, s5
+; GFX1032-NEXT: s_mul_i32 s12, s10, s5
; GFX1032-NEXT: s_add_i32 s11, s13, s11
-; GFX1032-NEXT: s_mul_i32 s14, s9, s8
+; GFX1032-NEXT: s_mul_i32 s14, s9, s5
; GFX1032-NEXT: s_add_i32 s11, s11, s12
-; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s14
-; GFX1032-NEXT: s_mul_i32 s16, s8, s11
-; GFX1032-NEXT: s_mul_hi_u32 s15, s5, s14
-; GFX1032-NEXT: s_mul_i32 s12, s5, s14
-; GFX1032-NEXT: s_mul_hi_u32 s14, s8, s11
+; GFX1032-NEXT: s_mul_hi_u32 s13, s5, s14
+; GFX1032-NEXT: s_mul_i32 s16, s5, s11
+; GFX1032-NEXT: s_mul_hi_u32 s15, s4, s14
+; GFX1032-NEXT: s_mul_i32 s12, s4, s14
+; GFX1032-NEXT: s_mul_hi_u32 s14, s5, s11
; GFX1032-NEXT: s_add_u32 s13, s13, s16
; GFX1032-NEXT: s_addc_u32 s14, 0, s14
-; GFX1032-NEXT: s_mul_hi_u32 s17, s5, s11
+; GFX1032-NEXT: s_mul_hi_u32 s17, s4, s11
; GFX1032-NEXT: s_add_u32 s12, s13, s12
-; GFX1032-NEXT: s_mul_i32 s11, s5, s11
+; GFX1032-NEXT: s_mul_i32 s11, s4, s11
; GFX1032-NEXT: s_addc_u32 s12, s14, s15
; GFX1032-NEXT: s_addc_u32 s13, s17, 0
; GFX1032-NEXT: s_add_u32 s11, s12, s11
; GFX1032-NEXT: s_addc_u32 s12, 0, s13
-; GFX1032-NEXT: s_add_u32 s8, s8, s11
-; GFX1032-NEXT: s_addc_u32 s5, s5, s12
-; GFX1032-NEXT: s_mul_hi_u32 s11, s9, s8
-; GFX1032-NEXT: s_mul_i32 s12, s9, s8
-; GFX1032-NEXT: s_mul_i32 s9, s9, s5
-; GFX1032-NEXT: s_mul_i32 s10, s10, s8
+; GFX1032-NEXT: s_add_u32 s5, s5, s11
+; GFX1032-NEXT: s_addc_u32 s4, s4, s12
+; GFX1032-NEXT: s_mul_hi_u32 s11, s9, s5
+; GFX1032-NEXT: s_mul_i32 s12, s9, s5
+; GFX1032-NEXT: s_mul_i32 s9, s9, s4
+; GFX1032-NEXT: s_mul_i32 s10, s10, s5
; GFX1032-NEXT: s_add_i32 s9, s11, s9
-; GFX1032-NEXT: s_mul_i32 s11, s5, s12
+; GFX1032-NEXT: s_mul_i32 s11, s4, s12
; GFX1032-NEXT: s_add_i32 s9, s9, s10
-; GFX1032-NEXT: s_mul_hi_u32 s10, s8, s12
-; GFX1032-NEXT: s_mul_i32 s15, s8, s9
-; GFX1032-NEXT: s_mul_hi_u32 s14, s8, s9
+; GFX1032-NEXT: s_mul_hi_u32 s10, s5, s12
+; GFX1032-NEXT: s_mul_i32 s15, s5, s9
+; GFX1032-NEXT: s_mul_hi_u32 s14, s5, s9
; GFX1032-NEXT: s_add_u32 s10, s10, s15
-; GFX1032-NEXT: s_mul_hi_u32 s13, s5, s12
+; GFX1032-NEXT: s_mul_hi_u32 s13, s4, s12
; GFX1032-NEXT: s_addc_u32 s14, 0, s14
-; GFX1032-NEXT: s_mul_hi_u32 s12, s5, s9
+; GFX1032-NEXT: s_mul_hi_u32 s12, s4, s9
; GFX1032-NEXT: s_add_u32 s10, s10, s11
-; GFX1032-NEXT: s_mul_i32 s9, s5, s9
+; GFX1032-NEXT: s_mul_i32 s9, s4, s9
; GFX1032-NEXT: s_addc_u32 s10, s14, s13
; GFX1032-NEXT: s_addc_u32 s11, s12, 0
; GFX1032-NEXT: s_add_u32 s9, s10, s9
; GFX1032-NEXT: s_addc_u32 s10, 0, s11
-; GFX1032-NEXT: s_add_u32 s8, s8, s9
-; GFX1032-NEXT: s_addc_u32 s5, s5, s10
-; GFX1032-NEXT: s_mul_hi_u32 s9, s2, s8
-; GFX1032-NEXT: s_mul_i32 s12, s2, s5
-; GFX1032-NEXT: s_mul_hi_u32 s11, s2, s5
-; GFX1032-NEXT: s_mul_hi_u32 s10, s3, s8
-; GFX1032-NEXT: s_mul_i32 s8, s3, s8
+; GFX1032-NEXT: s_add_u32 s5, s5, s9
+; GFX1032-NEXT: s_addc_u32 s4, s4, s10
+; GFX1032-NEXT: s_mul_hi_u32 s9, s2, s5
+; GFX1032-NEXT: s_mul_i32 s12, s2, s4
+; GFX1032-NEXT: s_mul_hi_u32 s11, s2, s4
+; GFX1032-NEXT: s_mul_hi_u32 s10, s3, s5
+; GFX1032-NEXT: s_mul_i32 s5, s3, s5
; GFX1032-NEXT: s_add_u32 s9, s9, s12
; GFX1032-NEXT: s_addc_u32 s11, 0, s11
-; GFX1032-NEXT: s_mul_hi_u32 s13, s3, s5
-; GFX1032-NEXT: s_add_u32 s8, s9, s8
-; GFX1032-NEXT: s_mul_i32 s5, s3, s5
-; GFX1032-NEXT: s_addc_u32 s8, s11, s10
+; GFX1032-NEXT: s_mul_hi_u32 s13, s3, s4
+; GFX1032-NEXT: s_add_u32 s5, s9, s5
+; GFX1032-NEXT: s_mul_i32 s4, s3, s4
+; GFX1032-NEXT: s_addc_u32 s5, s11, s10
; GFX1032-NEXT: s_addc_u32 s9, s13, 0
-; GFX1032-NEXT: s_add_u32 s5, s8, s5
-; GFX1032-NEXT: s_addc_u32 s8, 0, s9
-; GFX1032-NEXT: s_mul_hi_u32 s9, s0, s5
-; GFX1032-NEXT: s_mul_i32 s10, s0, s8
-; GFX1032-NEXT: s_mul_i32 s11, s1, s5
-; GFX1032-NEXT: s_add_i32 s9, s9, s10
+; GFX1032-NEXT: s_add_u32 s4, s5, s4
+; GFX1032-NEXT: s_addc_u32 s5, 0, s9
+; GFX1032-NEXT: s_mul_hi_u32 s9, s0, s4
; GFX1032-NEXT: s_mul_i32 s10, s0, s5
+; GFX1032-NEXT: s_mul_i32 s11, s1, s4
+; GFX1032-NEXT: s_add_i32 s9, s9, s10
+; GFX1032-NEXT: s_mul_i32 s10, s0, s4
; GFX1032-NEXT: s_add_i32 s9, s9, s11
; GFX1032-NEXT: s_sub_i32 s11, s3, s9
; GFX1032-NEXT: s_sub_u32 s10, s2, s10
@@ -829,10 +828,10 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cselect_b32 s13, -1, 0
; GFX1032-NEXT: s_cmp_eq_u32 s11, s1
; GFX1032-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1032-NEXT: s_add_u32 s13, s5, 1
-; GFX1032-NEXT: s_addc_u32 s14, s8, 0
-; GFX1032-NEXT: s_add_u32 s15, s5, 2
-; GFX1032-NEXT: s_addc_u32 s16, s8, 0
+; GFX1032-NEXT: s_add_u32 s13, s4, 1
+; GFX1032-NEXT: s_addc_u32 s14, s5, 0
+; GFX1032-NEXT: s_add_u32 s15, s4, 2
+; GFX1032-NEXT: s_addc_u32 s16, s5, 0
; GFX1032-NEXT: s_cmp_lg_u32 s11, 0
; GFX1032-NEXT: s_cselect_b32 s11, s15, s13
; GFX1032-NEXT: s_cselect_b32 s13, s16, s14
@@ -845,14 +844,14 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cmp_eq_u32 s3, s1
; GFX1032-NEXT: s_cselect_b32 s1, s10, s9
; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1032-NEXT: s_cselect_b32 s9, s13, s8
-; GFX1032-NEXT: s_cselect_b32 s8, s11, s5
-; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
+; GFX1032-NEXT: s_cselect_b32 s5, s13, s5
+; GFX1032-NEXT: s_cselect_b32 s4, s11, s4
+; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
; GFX1032-NEXT: s_cbranch_vccnz .LBB15_3
; GFX1032-NEXT: .LBB15_2:
; GFX1032-NEXT: v_cvt_f32_u32_e32 v0, s0
; GFX1032-NEXT: s_sub_i32 s3, 0, s0
-; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: s_mov_b32 s5, 0
; GFX1032-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1032-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1032-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -870,15 +869,15 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cselect_b32 s2, s4, s2
; GFX1032-NEXT: s_add_i32 s3, s1, 1
; GFX1032-NEXT: s_cmp_ge_u32 s2, s0
-; GFX1032-NEXT: s_cselect_b32 s8, s3, s1
+; GFX1032-NEXT: s_cselect_b32 s4, s3, s1
; GFX1032-NEXT: .LBB15_3:
-; GFX1032-NEXT: v_mov_b32_e32 v0, s8
+; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, s9
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
; GFX1032-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] offset:16
; GFX1032-NEXT: s_endpgm
; GFX1032-NEXT: .LBB15_4:
-; GFX1032-NEXT: ; implicit-def: $sgpr8_sgpr9
+; GFX1032-NEXT: ; implicit-def: $sgpr4_sgpr5
; GFX1032-NEXT: s_branch .LBB15_2
;
; GFX1064-LABEL: test_udiv64:
@@ -888,8 +887,6 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_or_b64 s[4:5], s[2:3], s[0:1]
-; GFX1064-NEXT: s_mov_b32 s4, 0
-; GFX1064-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064-NEXT: s_cbranch_scc0 .LBB15_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: v_cvt_f32_u32_e32 v0, s0
>From cf03169967ee89e2e701780809c73aafd11894e7 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 18:17:48 -0500
Subject: [PATCH 06/22] Fix bug and update tests
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 +-
.../test/CodeGen/AMDGPU/carryout-selection.ll | 346 +++++++++---------
llvm/test/CodeGen/AMDGPU/srem.ll | 98 +++--
llvm/test/CodeGen/AMDGPU/wave32.ll | 115 +++---
4 files changed, 302 insertions(+), 265 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 57be1cc004af2..75a3a1ffe2cee 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1315,7 +1315,7 @@ Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
MachineInstr *
SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
- if (MI.getOpcode() != AMDGPU::REG_SEQUENCE)
+ if (MI.getOpcode() != AMDGPU::REG_SEQUENCE || MI.getNumOperands() != 5)
return nullptr;
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
@@ -1331,7 +1331,11 @@ SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
}
for (unsigned I : {0, 1})
- if (SubRegIsConst[I] && !SubRegValues[I])
+ if (SubRegIsConst[I] && !SubRegValues[I] &&
+ MRI.getRegClass(RealDefs[(I + 1) % 2]->getOperand(0).getReg())
+ ->MC->getSizeInBits() *
+ 2 ==
+ MRI.getRegClass(MI.getOperand(0).getReg())->MC->getSizeInBits())
return RealDefs[(I + 1) % 2];
return nullptr;
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 356a69b6e8aee..163d7ff9c61fc 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -2132,6 +2132,8 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
+; VI-NEXT: s_mov_b32 s6, 0
+; VI-NEXT: s_cmp_lg_u64 s[6:7], 0
; VI-NEXT: s_cbranch_scc0 .LBB16_3
; VI-NEXT: ; %bb.1:
; VI-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2282,6 +2284,8 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_or_b64 s[4:5], s[2:3], s[6:7]
+; GFX9-NEXT: s_mov_b32 s4, 0
+; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cbranch_scc0 .LBB16_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
@@ -2430,9 +2434,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_clause 0x1
; GFX1010-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1010-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1010-NEXT: s_mov_b32 s8, 0
; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-NEXT: s_or_b64 s[4:5], s[2:3], s[6:7]
+; GFX1010-NEXT: s_mov_b32 s4, 0
+; GFX1010-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1010-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1010-NEXT: ; %bb.1:
; GFX1010-NEXT: v_cvt_f32_u32_e32 v0, s6
@@ -2447,71 +2452,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
; GFX1010-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1010-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1010-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1010-NEXT: v_readfirstlane_b32 s5, v0
-; GFX1010-NEXT: s_mul_i32 s11, s9, s4
-; GFX1010-NEXT: s_mul_hi_u32 s13, s9, s5
-; GFX1010-NEXT: s_mul_i32 s12, s10, s5
+; GFX1010-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1010-NEXT: s_mul_i32 s11, s9, s5
+; GFX1010-NEXT: s_mul_hi_u32 s13, s9, s8
+; GFX1010-NEXT: s_mul_i32 s12, s10, s8
; GFX1010-NEXT: s_add_i32 s11, s13, s11
-; GFX1010-NEXT: s_mul_i32 s14, s9, s5
+; GFX1010-NEXT: s_mul_i32 s14, s9, s8
; GFX1010-NEXT: s_add_i32 s11, s11, s12
-; GFX1010-NEXT: s_mul_hi_u32 s13, s5, s14
-; GFX1010-NEXT: s_mul_i32 s16, s5, s11
-; GFX1010-NEXT: s_mul_hi_u32 s15, s4, s14
-; GFX1010-NEXT: s_mul_i32 s12, s4, s14
-; GFX1010-NEXT: s_mul_hi_u32 s14, s5, s11
+; GFX1010-NEXT: s_mul_hi_u32 s13, s8, s14
+; GFX1010-NEXT: s_mul_i32 s16, s8, s11
+; GFX1010-NEXT: s_mul_hi_u32 s15, s5, s14
+; GFX1010-NEXT: s_mul_i32 s12, s5, s14
+; GFX1010-NEXT: s_mul_hi_u32 s14, s8, s11
; GFX1010-NEXT: s_add_u32 s13, s13, s16
; GFX1010-NEXT: s_addc_u32 s14, 0, s14
-; GFX1010-NEXT: s_mul_hi_u32 s17, s4, s11
+; GFX1010-NEXT: s_mul_hi_u32 s17, s5, s11
; GFX1010-NEXT: s_add_u32 s12, s13, s12
-; GFX1010-NEXT: s_mul_i32 s11, s4, s11
+; GFX1010-NEXT: s_mul_i32 s11, s5, s11
; GFX1010-NEXT: s_addc_u32 s12, s14, s15
; GFX1010-NEXT: s_addc_u32 s13, s17, 0
; GFX1010-NEXT: s_add_u32 s11, s12, s11
; GFX1010-NEXT: s_addc_u32 s12, 0, s13
-; GFX1010-NEXT: s_add_u32 s5, s5, s11
-; GFX1010-NEXT: s_addc_u32 s4, s4, s12
-; GFX1010-NEXT: s_mul_hi_u32 s11, s9, s5
-; GFX1010-NEXT: s_mul_i32 s12, s9, s5
-; GFX1010-NEXT: s_mul_i32 s9, s9, s4
-; GFX1010-NEXT: s_mul_i32 s10, s10, s5
+; GFX1010-NEXT: s_add_u32 s8, s8, s11
+; GFX1010-NEXT: s_addc_u32 s5, s5, s12
+; GFX1010-NEXT: s_mul_hi_u32 s11, s9, s8
+; GFX1010-NEXT: s_mul_i32 s12, s9, s8
+; GFX1010-NEXT: s_mul_i32 s9, s9, s5
+; GFX1010-NEXT: s_mul_i32 s10, s10, s8
; GFX1010-NEXT: s_add_i32 s9, s11, s9
-; GFX1010-NEXT: s_mul_i32 s11, s4, s12
+; GFX1010-NEXT: s_mul_i32 s11, s5, s12
; GFX1010-NEXT: s_add_i32 s9, s9, s10
-; GFX1010-NEXT: s_mul_hi_u32 s10, s5, s12
-; GFX1010-NEXT: s_mul_i32 s15, s5, s9
-; GFX1010-NEXT: s_mul_hi_u32 s14, s5, s9
+; GFX1010-NEXT: s_mul_hi_u32 s10, s8, s12
+; GFX1010-NEXT: s_mul_i32 s15, s8, s9
+; GFX1010-NEXT: s_mul_hi_u32 s14, s8, s9
; GFX1010-NEXT: s_add_u32 s10, s10, s15
-; GFX1010-NEXT: s_mul_hi_u32 s13, s4, s12
+; GFX1010-NEXT: s_mul_hi_u32 s13, s5, s12
; GFX1010-NEXT: s_addc_u32 s14, 0, s14
-; GFX1010-NEXT: s_mul_hi_u32 s12, s4, s9
+; GFX1010-NEXT: s_mul_hi_u32 s12, s5, s9
; GFX1010-NEXT: s_add_u32 s10, s10, s11
-; GFX1010-NEXT: s_mul_i32 s9, s4, s9
+; GFX1010-NEXT: s_mul_i32 s9, s5, s9
; GFX1010-NEXT: s_addc_u32 s10, s14, s13
; GFX1010-NEXT: s_addc_u32 s11, s12, 0
; GFX1010-NEXT: s_add_u32 s9, s10, s9
; GFX1010-NEXT: s_addc_u32 s10, 0, s11
-; GFX1010-NEXT: s_add_u32 s5, s5, s9
-; GFX1010-NEXT: s_addc_u32 s4, s4, s10
-; GFX1010-NEXT: s_mul_hi_u32 s9, s2, s5
-; GFX1010-NEXT: s_mul_i32 s12, s2, s4
-; GFX1010-NEXT: s_mul_hi_u32 s11, s2, s4
-; GFX1010-NEXT: s_mul_hi_u32 s10, s3, s5
-; GFX1010-NEXT: s_mul_i32 s5, s3, s5
+; GFX1010-NEXT: s_add_u32 s8, s8, s9
+; GFX1010-NEXT: s_addc_u32 s5, s5, s10
+; GFX1010-NEXT: s_mul_hi_u32 s9, s2, s8
+; GFX1010-NEXT: s_mul_i32 s12, s2, s5
+; GFX1010-NEXT: s_mul_hi_u32 s11, s2, s5
+; GFX1010-NEXT: s_mul_hi_u32 s10, s3, s8
+; GFX1010-NEXT: s_mul_i32 s8, s3, s8
; GFX1010-NEXT: s_add_u32 s9, s9, s12
; GFX1010-NEXT: s_addc_u32 s11, 0, s11
-; GFX1010-NEXT: s_mul_hi_u32 s13, s3, s4
-; GFX1010-NEXT: s_add_u32 s5, s9, s5
-; GFX1010-NEXT: s_mul_i32 s4, s3, s4
-; GFX1010-NEXT: s_addc_u32 s5, s11, s10
+; GFX1010-NEXT: s_mul_hi_u32 s13, s3, s5
+; GFX1010-NEXT: s_add_u32 s8, s9, s8
+; GFX1010-NEXT: s_mul_i32 s5, s3, s5
+; GFX1010-NEXT: s_addc_u32 s8, s11, s10
; GFX1010-NEXT: s_addc_u32 s9, s13, 0
-; GFX1010-NEXT: s_add_u32 s4, s5, s4
-; GFX1010-NEXT: s_addc_u32 s5, 0, s9
-; GFX1010-NEXT: s_mul_hi_u32 s9, s6, s4
-; GFX1010-NEXT: s_mul_i32 s10, s6, s5
-; GFX1010-NEXT: s_mul_i32 s11, s7, s4
+; GFX1010-NEXT: s_add_u32 s5, s8, s5
+; GFX1010-NEXT: s_addc_u32 s8, 0, s9
+; GFX1010-NEXT: s_mul_hi_u32 s9, s6, s5
+; GFX1010-NEXT: s_mul_i32 s10, s6, s8
+; GFX1010-NEXT: s_mul_i32 s11, s7, s5
; GFX1010-NEXT: s_add_i32 s9, s9, s10
-; GFX1010-NEXT: s_mul_i32 s10, s6, s4
+; GFX1010-NEXT: s_mul_i32 s10, s6, s5
; GFX1010-NEXT: s_add_i32 s9, s9, s11
; GFX1010-NEXT: s_sub_i32 s11, s3, s9
; GFX1010-NEXT: s_sub_u32 s10, s2, s10
@@ -2525,10 +2530,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cselect_b32 s13, -1, 0
; GFX1010-NEXT: s_cmp_eq_u32 s11, s7
; GFX1010-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1010-NEXT: s_add_u32 s13, s4, 1
-; GFX1010-NEXT: s_addc_u32 s14, s5, 0
-; GFX1010-NEXT: s_add_u32 s15, s4, 2
-; GFX1010-NEXT: s_addc_u32 s16, s5, 0
+; GFX1010-NEXT: s_add_u32 s13, s5, 1
+; GFX1010-NEXT: s_addc_u32 s14, s8, 0
+; GFX1010-NEXT: s_add_u32 s15, s5, 2
+; GFX1010-NEXT: s_addc_u32 s16, s8, 0
; GFX1010-NEXT: s_cmp_lg_u32 s11, 0
; GFX1010-NEXT: s_cselect_b32 s11, s15, s13
; GFX1010-NEXT: s_cselect_b32 s13, s16, s14
@@ -2541,13 +2546,14 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cmp_eq_u32 s3, s7
; GFX1010-NEXT: s_cselect_b32 s3, s10, s9
; GFX1010-NEXT: s_cmp_lg_u32 s3, 0
-; GFX1010-NEXT: s_cselect_b32 s5, s13, s5
-; GFX1010-NEXT: s_cselect_b32 s4, s11, s4
-; GFX1010-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
+; GFX1010-NEXT: s_cselect_b32 s9, s13, s8
+; GFX1010-NEXT: s_cselect_b32 s8, s11, s5
+; GFX1010-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
; GFX1010-NEXT: s_cbranch_vccnz .LBB16_3
; GFX1010-NEXT: .LBB16_2:
; GFX1010-NEXT: v_cvt_f32_u32_e32 v0, s6
; GFX1010-NEXT: s_sub_i32 s4, 0, s6
+; GFX1010-NEXT: s_mov_b32 s9, 0
; GFX1010-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1010-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1010-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -2565,16 +2571,15 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1010-NEXT: s_cselect_b32 s2, s5, s2
; GFX1010-NEXT: s_add_i32 s4, s3, 1
; GFX1010-NEXT: s_cmp_ge_u32 s2, s6
-; GFX1010-NEXT: s_mov_b32 s5, 0
-; GFX1010-NEXT: s_cselect_b32 s4, s4, s3
+; GFX1010-NEXT: s_cselect_b32 s8, s4, s3
; GFX1010-NEXT: .LBB16_3:
-; GFX1010-NEXT: v_mov_b32_e32 v0, s4
+; GFX1010-NEXT: v_mov_b32_e32 v0, s8
; GFX1010-NEXT: v_mov_b32_e32 v2, 0
-; GFX1010-NEXT: v_mov_b32_e32 v1, s5
+; GFX1010-NEXT: v_mov_b32_e32 v1, s9
; GFX1010-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-NEXT: s_endpgm
; GFX1010-NEXT: .LBB16_4:
-; GFX1010-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX1010-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX1010-NEXT: s_branch .LBB16_2
;
; GFX1030W32-LABEL: sudiv64:
@@ -2582,9 +2587,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_clause 0x1
; GFX1030W32-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1030W32-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX1030W32-NEXT: s_mov_b32 s8, 0
; GFX1030W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX1030W32-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
+; GFX1030W32-NEXT: s_mov_b32 s6, 0
+; GFX1030W32-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX1030W32-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1030W32-NEXT: ; %bb.1:
; GFX1030W32-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2599,71 +2605,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: v_fmamk_f32 v0, v1, 0xcf800000, v0
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1030W32-NEXT: v_readfirstlane_b32 s6, v1
-; GFX1030W32-NEXT: v_readfirstlane_b32 s7, v0
-; GFX1030W32-NEXT: s_mul_i32 s11, s9, s6
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s9, s7
-; GFX1030W32-NEXT: s_mul_i32 s12, s10, s7
+; GFX1030W32-NEXT: v_readfirstlane_b32 s7, v1
+; GFX1030W32-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1030W32-NEXT: s_mul_i32 s11, s9, s7
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s9, s8
+; GFX1030W32-NEXT: s_mul_i32 s12, s10, s8
; GFX1030W32-NEXT: s_add_i32 s11, s13, s11
-; GFX1030W32-NEXT: s_mul_i32 s14, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s14, s9, s8
; GFX1030W32-NEXT: s_add_i32 s11, s11, s12
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s7, s14
-; GFX1030W32-NEXT: s_mul_i32 s16, s7, s11
-; GFX1030W32-NEXT: s_mul_hi_u32 s15, s6, s14
-; GFX1030W32-NEXT: s_mul_i32 s12, s6, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s14, s7, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s8, s14
+; GFX1030W32-NEXT: s_mul_i32 s16, s8, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s15, s7, s14
+; GFX1030W32-NEXT: s_mul_i32 s12, s7, s14
+; GFX1030W32-NEXT: s_mul_hi_u32 s14, s8, s11
; GFX1030W32-NEXT: s_add_u32 s13, s13, s16
; GFX1030W32-NEXT: s_addc_u32 s14, 0, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s17, s6, s11
+; GFX1030W32-NEXT: s_mul_hi_u32 s17, s7, s11
; GFX1030W32-NEXT: s_add_u32 s12, s13, s12
-; GFX1030W32-NEXT: s_mul_i32 s11, s6, s11
+; GFX1030W32-NEXT: s_mul_i32 s11, s7, s11
; GFX1030W32-NEXT: s_addc_u32 s12, s14, s15
; GFX1030W32-NEXT: s_addc_u32 s13, s17, 0
; GFX1030W32-NEXT: s_add_u32 s11, s12, s11
; GFX1030W32-NEXT: s_addc_u32 s12, 0, s13
-; GFX1030W32-NEXT: s_add_u32 s7, s7, s11
-; GFX1030W32-NEXT: s_addc_u32 s6, s6, s12
-; GFX1030W32-NEXT: s_mul_hi_u32 s11, s9, s7
-; GFX1030W32-NEXT: s_mul_i32 s12, s9, s7
-; GFX1030W32-NEXT: s_mul_i32 s9, s9, s6
-; GFX1030W32-NEXT: s_mul_i32 s10, s10, s7
+; GFX1030W32-NEXT: s_add_u32 s8, s8, s11
+; GFX1030W32-NEXT: s_addc_u32 s7, s7, s12
+; GFX1030W32-NEXT: s_mul_hi_u32 s11, s9, s8
+; GFX1030W32-NEXT: s_mul_i32 s12, s9, s8
+; GFX1030W32-NEXT: s_mul_i32 s9, s9, s7
+; GFX1030W32-NEXT: s_mul_i32 s10, s10, s8
; GFX1030W32-NEXT: s_add_i32 s9, s11, s9
-; GFX1030W32-NEXT: s_mul_i32 s11, s6, s12
+; GFX1030W32-NEXT: s_mul_i32 s11, s7, s12
; GFX1030W32-NEXT: s_add_i32 s9, s9, s10
-; GFX1030W32-NEXT: s_mul_hi_u32 s10, s7, s12
-; GFX1030W32-NEXT: s_mul_i32 s15, s7, s9
-; GFX1030W32-NEXT: s_mul_hi_u32 s14, s7, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s10, s8, s12
+; GFX1030W32-NEXT: s_mul_i32 s15, s8, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s14, s8, s9
; GFX1030W32-NEXT: s_add_u32 s10, s10, s15
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s6, s12
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s7, s12
; GFX1030W32-NEXT: s_addc_u32 s14, 0, s14
-; GFX1030W32-NEXT: s_mul_hi_u32 s12, s6, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s12, s7, s9
; GFX1030W32-NEXT: s_add_u32 s10, s10, s11
-; GFX1030W32-NEXT: s_mul_i32 s9, s6, s9
+; GFX1030W32-NEXT: s_mul_i32 s9, s7, s9
; GFX1030W32-NEXT: s_addc_u32 s10, s14, s13
; GFX1030W32-NEXT: s_addc_u32 s11, s12, 0
; GFX1030W32-NEXT: s_add_u32 s9, s10, s9
; GFX1030W32-NEXT: s_addc_u32 s10, 0, s11
-; GFX1030W32-NEXT: s_add_u32 s7, s7, s9
-; GFX1030W32-NEXT: s_addc_u32 s6, s6, s10
-; GFX1030W32-NEXT: s_mul_hi_u32 s9, s2, s7
-; GFX1030W32-NEXT: s_mul_i32 s12, s2, s6
-; GFX1030W32-NEXT: s_mul_hi_u32 s11, s2, s6
-; GFX1030W32-NEXT: s_mul_hi_u32 s10, s3, s7
-; GFX1030W32-NEXT: s_mul_i32 s7, s3, s7
+; GFX1030W32-NEXT: s_add_u32 s8, s8, s9
+; GFX1030W32-NEXT: s_addc_u32 s7, s7, s10
+; GFX1030W32-NEXT: s_mul_hi_u32 s9, s2, s8
+; GFX1030W32-NEXT: s_mul_i32 s12, s2, s7
+; GFX1030W32-NEXT: s_mul_hi_u32 s11, s2, s7
+; GFX1030W32-NEXT: s_mul_hi_u32 s10, s3, s8
+; GFX1030W32-NEXT: s_mul_i32 s8, s3, s8
; GFX1030W32-NEXT: s_add_u32 s9, s9, s12
; GFX1030W32-NEXT: s_addc_u32 s11, 0, s11
-; GFX1030W32-NEXT: s_mul_hi_u32 s13, s3, s6
-; GFX1030W32-NEXT: s_add_u32 s7, s9, s7
-; GFX1030W32-NEXT: s_mul_i32 s6, s3, s6
-; GFX1030W32-NEXT: s_addc_u32 s7, s11, s10
+; GFX1030W32-NEXT: s_mul_hi_u32 s13, s3, s7
+; GFX1030W32-NEXT: s_add_u32 s8, s9, s8
+; GFX1030W32-NEXT: s_mul_i32 s7, s3, s7
+; GFX1030W32-NEXT: s_addc_u32 s8, s11, s10
; GFX1030W32-NEXT: s_addc_u32 s9, s13, 0
-; GFX1030W32-NEXT: s_add_u32 s6, s7, s6
-; GFX1030W32-NEXT: s_addc_u32 s7, 0, s9
-; GFX1030W32-NEXT: s_mul_hi_u32 s9, s4, s6
-; GFX1030W32-NEXT: s_mul_i32 s10, s4, s7
-; GFX1030W32-NEXT: s_mul_i32 s11, s5, s6
+; GFX1030W32-NEXT: s_add_u32 s7, s8, s7
+; GFX1030W32-NEXT: s_addc_u32 s8, 0, s9
+; GFX1030W32-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1030W32-NEXT: s_mul_i32 s10, s4, s8
+; GFX1030W32-NEXT: s_mul_i32 s11, s5, s7
; GFX1030W32-NEXT: s_add_i32 s9, s9, s10
-; GFX1030W32-NEXT: s_mul_i32 s10, s4, s6
+; GFX1030W32-NEXT: s_mul_i32 s10, s4, s7
; GFX1030W32-NEXT: s_add_i32 s9, s9, s11
; GFX1030W32-NEXT: s_sub_i32 s11, s3, s9
; GFX1030W32-NEXT: s_sub_u32 s10, s2, s10
@@ -2677,10 +2683,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cselect_b32 s13, -1, 0
; GFX1030W32-NEXT: s_cmp_eq_u32 s11, s5
; GFX1030W32-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1030W32-NEXT: s_add_u32 s13, s6, 1
-; GFX1030W32-NEXT: s_addc_u32 s14, s7, 0
-; GFX1030W32-NEXT: s_add_u32 s15, s6, 2
-; GFX1030W32-NEXT: s_addc_u32 s16, s7, 0
+; GFX1030W32-NEXT: s_add_u32 s13, s7, 1
+; GFX1030W32-NEXT: s_addc_u32 s14, s8, 0
+; GFX1030W32-NEXT: s_add_u32 s15, s7, 2
+; GFX1030W32-NEXT: s_addc_u32 s16, s8, 0
; GFX1030W32-NEXT: s_cmp_lg_u32 s11, 0
; GFX1030W32-NEXT: s_cselect_b32 s11, s15, s13
; GFX1030W32-NEXT: s_cselect_b32 s13, s16, s14
@@ -2693,14 +2699,14 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cmp_eq_u32 s3, s5
; GFX1030W32-NEXT: s_cselect_b32 s3, s10, s9
; GFX1030W32-NEXT: s_cmp_lg_u32 s3, 0
-; GFX1030W32-NEXT: s_cselect_b32 s7, s13, s7
-; GFX1030W32-NEXT: s_cselect_b32 s6, s11, s6
-; GFX1030W32-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
+; GFX1030W32-NEXT: s_cselect_b32 s9, s13, s8
+; GFX1030W32-NEXT: s_cselect_b32 s8, s11, s7
+; GFX1030W32-NEXT: s_andn2_b32 vcc_lo, exec_lo, s6
; GFX1030W32-NEXT: s_cbranch_vccnz .LBB16_3
; GFX1030W32-NEXT: .LBB16_2:
; GFX1030W32-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX1030W32-NEXT: s_sub_i32 s5, 0, s4
-; GFX1030W32-NEXT: s_mov_b32 s7, 0
+; GFX1030W32-NEXT: s_mov_b32 s9, 0
; GFX1030W32-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1030W32-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1030W32-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -2718,15 +2724,15 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W32-NEXT: s_cselect_b32 s2, s6, s2
; GFX1030W32-NEXT: s_add_i32 s5, s3, 1
; GFX1030W32-NEXT: s_cmp_ge_u32 s2, s4
-; GFX1030W32-NEXT: s_cselect_b32 s6, s5, s3
+; GFX1030W32-NEXT: s_cselect_b32 s8, s5, s3
; GFX1030W32-NEXT: .LBB16_3:
-; GFX1030W32-NEXT: v_mov_b32_e32 v0, s6
+; GFX1030W32-NEXT: v_mov_b32_e32 v0, s8
; GFX1030W32-NEXT: v_mov_b32_e32 v2, 0
-; GFX1030W32-NEXT: v_mov_b32_e32 v1, s7
+; GFX1030W32-NEXT: v_mov_b32_e32 v1, s9
; GFX1030W32-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1030W32-NEXT: s_endpgm
; GFX1030W32-NEXT: .LBB16_4:
-; GFX1030W32-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX1030W32-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX1030W32-NEXT: s_branch .LBB16_2
;
; GFX1030W64-LABEL: sudiv64:
@@ -2736,6 +2742,8 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1030W64-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX1030W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX1030W64-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
+; GFX1030W64-NEXT: s_mov_b32 s6, 0
+; GFX1030W64-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX1030W64-NEXT: s_cbranch_scc0 .LBB16_4
; GFX1030W64-NEXT: ; %bb.1:
; GFX1030W64-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2884,9 +2892,11 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-NEXT: s_mov_b32 s8, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_or_b64 s[6:7], s[2:3], s[4:5]
+; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX11-NEXT: s_cbranch_scc0 .LBB16_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -2906,71 +2916,71 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-NEXT: v_readfirstlane_b32 s7, v0
-; GFX11-NEXT: s_mul_i32 s11, s9, s6
-; GFX11-NEXT: s_mul_hi_u32 s13, s9, s7
-; GFX11-NEXT: s_mul_i32 s12, s10, s7
+; GFX11-NEXT: v_readfirstlane_b32 s7, v1
+; GFX11-NEXT: v_readfirstlane_b32 s8, v0
+; GFX11-NEXT: s_mul_i32 s11, s9, s7
+; GFX11-NEXT: s_mul_hi_u32 s13, s9, s8
+; GFX11-NEXT: s_mul_i32 s12, s10, s8
; GFX11-NEXT: s_add_i32 s11, s13, s11
-; GFX11-NEXT: s_mul_i32 s14, s9, s7
+; GFX11-NEXT: s_mul_i32 s14, s9, s8
; GFX11-NEXT: s_add_i32 s11, s11, s12
-; GFX11-NEXT: s_mul_hi_u32 s13, s7, s14
-; GFX11-NEXT: s_mul_i32 s16, s7, s11
-; GFX11-NEXT: s_mul_hi_u32 s15, s6, s14
-; GFX11-NEXT: s_mul_i32 s12, s6, s14
-; GFX11-NEXT: s_mul_hi_u32 s14, s7, s11
+; GFX11-NEXT: s_mul_hi_u32 s13, s8, s14
+; GFX11-NEXT: s_mul_i32 s16, s8, s11
+; GFX11-NEXT: s_mul_hi_u32 s15, s7, s14
+; GFX11-NEXT: s_mul_i32 s12, s7, s14
+; GFX11-NEXT: s_mul_hi_u32 s14, s8, s11
; GFX11-NEXT: s_add_u32 s13, s13, s16
; GFX11-NEXT: s_addc_u32 s14, 0, s14
-; GFX11-NEXT: s_mul_hi_u32 s17, s6, s11
+; GFX11-NEXT: s_mul_hi_u32 s17, s7, s11
; GFX11-NEXT: s_add_u32 s12, s13, s12
-; GFX11-NEXT: s_mul_i32 s11, s6, s11
+; GFX11-NEXT: s_mul_i32 s11, s7, s11
; GFX11-NEXT: s_addc_u32 s12, s14, s15
; GFX11-NEXT: s_addc_u32 s13, s17, 0
; GFX11-NEXT: s_add_u32 s11, s12, s11
; GFX11-NEXT: s_addc_u32 s12, 0, s13
-; GFX11-NEXT: s_add_u32 s7, s7, s11
-; GFX11-NEXT: s_addc_u32 s6, s6, s12
-; GFX11-NEXT: s_mul_hi_u32 s11, s9, s7
-; GFX11-NEXT: s_mul_i32 s12, s9, s7
-; GFX11-NEXT: s_mul_i32 s9, s9, s6
-; GFX11-NEXT: s_mul_i32 s10, s10, s7
+; GFX11-NEXT: s_add_u32 s8, s8, s11
+; GFX11-NEXT: s_addc_u32 s7, s7, s12
+; GFX11-NEXT: s_mul_hi_u32 s11, s9, s8
+; GFX11-NEXT: s_mul_i32 s12, s9, s8
+; GFX11-NEXT: s_mul_i32 s9, s9, s7
+; GFX11-NEXT: s_mul_i32 s10, s10, s8
; GFX11-NEXT: s_add_i32 s9, s11, s9
-; GFX11-NEXT: s_mul_i32 s11, s6, s12
+; GFX11-NEXT: s_mul_i32 s11, s7, s12
; GFX11-NEXT: s_add_i32 s9, s9, s10
-; GFX11-NEXT: s_mul_hi_u32 s10, s7, s12
-; GFX11-NEXT: s_mul_i32 s15, s7, s9
-; GFX11-NEXT: s_mul_hi_u32 s14, s7, s9
+; GFX11-NEXT: s_mul_hi_u32 s10, s8, s12
+; GFX11-NEXT: s_mul_i32 s15, s8, s9
+; GFX11-NEXT: s_mul_hi_u32 s14, s8, s9
; GFX11-NEXT: s_add_u32 s10, s10, s15
-; GFX11-NEXT: s_mul_hi_u32 s13, s6, s12
+; GFX11-NEXT: s_mul_hi_u32 s13, s7, s12
; GFX11-NEXT: s_addc_u32 s14, 0, s14
-; GFX11-NEXT: s_mul_hi_u32 s12, s6, s9
+; GFX11-NEXT: s_mul_hi_u32 s12, s7, s9
; GFX11-NEXT: s_add_u32 s10, s10, s11
-; GFX11-NEXT: s_mul_i32 s9, s6, s9
+; GFX11-NEXT: s_mul_i32 s9, s7, s9
; GFX11-NEXT: s_addc_u32 s10, s14, s13
; GFX11-NEXT: s_addc_u32 s11, s12, 0
; GFX11-NEXT: s_add_u32 s9, s10, s9
; GFX11-NEXT: s_addc_u32 s10, 0, s11
-; GFX11-NEXT: s_add_u32 s7, s7, s9
-; GFX11-NEXT: s_addc_u32 s6, s6, s10
-; GFX11-NEXT: s_mul_hi_u32 s9, s2, s7
-; GFX11-NEXT: s_mul_i32 s12, s2, s6
-; GFX11-NEXT: s_mul_hi_u32 s11, s2, s6
-; GFX11-NEXT: s_mul_hi_u32 s10, s3, s7
-; GFX11-NEXT: s_mul_i32 s7, s3, s7
+; GFX11-NEXT: s_add_u32 s8, s8, s9
+; GFX11-NEXT: s_addc_u32 s7, s7, s10
+; GFX11-NEXT: s_mul_hi_u32 s9, s2, s8
+; GFX11-NEXT: s_mul_i32 s12, s2, s7
+; GFX11-NEXT: s_mul_hi_u32 s11, s2, s7
+; GFX11-NEXT: s_mul_hi_u32 s10, s3, s8
+; GFX11-NEXT: s_mul_i32 s8, s3, s8
; GFX11-NEXT: s_add_u32 s9, s9, s12
; GFX11-NEXT: s_addc_u32 s11, 0, s11
-; GFX11-NEXT: s_mul_hi_u32 s13, s3, s6
-; GFX11-NEXT: s_add_u32 s7, s9, s7
-; GFX11-NEXT: s_mul_i32 s6, s3, s6
-; GFX11-NEXT: s_addc_u32 s7, s11, s10
+; GFX11-NEXT: s_mul_hi_u32 s13, s3, s7
+; GFX11-NEXT: s_add_u32 s8, s9, s8
+; GFX11-NEXT: s_mul_i32 s7, s3, s7
+; GFX11-NEXT: s_addc_u32 s8, s11, s10
; GFX11-NEXT: s_addc_u32 s9, s13, 0
-; GFX11-NEXT: s_add_u32 s6, s7, s6
-; GFX11-NEXT: s_addc_u32 s7, 0, s9
-; GFX11-NEXT: s_mul_hi_u32 s9, s4, s6
-; GFX11-NEXT: s_mul_i32 s10, s4, s7
-; GFX11-NEXT: s_mul_i32 s11, s5, s6
+; GFX11-NEXT: s_add_u32 s7, s8, s7
+; GFX11-NEXT: s_addc_u32 s8, 0, s9
+; GFX11-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX11-NEXT: s_mul_i32 s10, s4, s8
+; GFX11-NEXT: s_mul_i32 s11, s5, s7
; GFX11-NEXT: s_add_i32 s9, s9, s10
-; GFX11-NEXT: s_mul_i32 s10, s4, s6
+; GFX11-NEXT: s_mul_i32 s10, s4, s7
; GFX11-NEXT: s_add_i32 s9, s9, s11
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_sub_i32 s11, s3, s9
@@ -2986,10 +2996,10 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s13, -1, 0
; GFX11-NEXT: s_cmp_eq_u32 s11, s5
; GFX11-NEXT: s_cselect_b32 s11, s13, s14
-; GFX11-NEXT: s_add_u32 s13, s6, 1
-; GFX11-NEXT: s_addc_u32 s14, s7, 0
-; GFX11-NEXT: s_add_u32 s15, s6, 2
-; GFX11-NEXT: s_addc_u32 s16, s7, 0
+; GFX11-NEXT: s_add_u32 s13, s7, 1
+; GFX11-NEXT: s_addc_u32 s14, s8, 0
+; GFX11-NEXT: s_add_u32 s15, s7, 2
+; GFX11-NEXT: s_addc_u32 s16, s8, 0
; GFX11-NEXT: s_cmp_lg_u32 s11, 0
; GFX11-NEXT: s_cselect_b32 s11, s15, s13
; GFX11-NEXT: s_cselect_b32 s13, s16, s14
@@ -3004,14 +3014,14 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s3, s10, s9
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s7, s13, s7
-; GFX11-NEXT: s_cselect_b32 s6, s11, s6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
+; GFX11-NEXT: s_cselect_b32 s9, s13, s8
+; GFX11-NEXT: s_cselect_b32 s8, s11, s7
+; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX11-NEXT: s_cbranch_vccnz .LBB16_3
; GFX11-NEXT: .LBB16_2:
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX11-NEXT: s_sub_i32 s5, 0, s4
-; GFX11-NEXT: s_mov_b32 s7, 0
+; GFX11-NEXT: s_mov_b32 s9, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -3034,15 +3044,15 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX11-NEXT: s_cselect_b32 s2, s6, s2
; GFX11-NEXT: s_add_i32 s5, s3, 1
; GFX11-NEXT: s_cmp_ge_u32 s2, s4
-; GFX11-NEXT: s_cselect_b32 s6, s5, s3
+; GFX11-NEXT: s_cselect_b32 s8, s5, s3
; GFX11-NEXT: .LBB16_3:
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT: v_mov_b32_e32 v0, s8
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s9
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
; GFX11-NEXT: .LBB16_4:
-; GFX11-NEXT: ; implicit-def: $sgpr6_sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX11-NEXT: s_branch .LBB16_2
;
; GFX1250-LABEL: sudiv64:
diff --git a/llvm/test/CodeGen/AMDGPU/srem.ll b/llvm/test/CodeGen/AMDGPU/srem.ll
index 3c3d634c96410..e12e31b14e97d 100644
--- a/llvm/test/CodeGen/AMDGPU/srem.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem.ll
@@ -1501,6 +1501,8 @@ define amdgpu_kernel void @srem_i64(ptr addrspace(1) %out, ptr addrspace(1) %in)
; GCN-NEXT: v_readfirstlane_b32 s3, v3
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: s_or_b64 s[6:7], s[4:5], s[2:3]
+; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
; GCN-NEXT: s_cbranch_scc0 .LBB8_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_ashr_i32 s6, s3, 31
@@ -1830,6 +1832,8 @@ define amdgpu_kernel void @srem_i64(ptr addrspace(1) %out, ptr addrspace(1) %in)
; TONGA-NEXT: v_readfirstlane_b32 s3, v3
; TONGA-NEXT: v_readfirstlane_b32 s2, v2
; TONGA-NEXT: s_or_b64 s[6:7], s[4:5], s[2:3]
+; TONGA-NEXT: s_mov_b32 s6, 0
+; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB8_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s3, 31
@@ -2697,10 +2701,12 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s11, v5
; GCN-NEXT: v_readfirstlane_b32 s10, v4
+; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[8:9]
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_readfirstlane_b32 s3, v3
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: v_readfirstlane_b32 s5, v7
-; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[8:9]
+; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
; GCN-NEXT: v_readfirstlane_b32 s4, v6
; GCN-NEXT: s_cbranch_scc0 .LBB10_6
; GCN-NEXT: ; %bb.1:
@@ -2849,6 +2855,8 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_cselect_b32 s6, s9, s6
; GCN-NEXT: .LBB10_3:
; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
+; GCN-NEXT: s_mov_b32 s8, 0
+; GCN-NEXT: s_cmp_lg_u64 s[8:9], 0
; GCN-NEXT: s_cbranch_scc0 .LBB10_7
; GCN-NEXT: ; %bb.4:
; GCN-NEXT: s_ashr_i32 s8, s3, 31
@@ -3336,6 +3344,8 @@ define amdgpu_kernel void @srem_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: v_readfirstlane_b32 s3, v5
; TONGA-NEXT: v_readfirstlane_b32 s2, v4
; TONGA-NEXT: s_or_b64 s[6:7], s[2:3], s[0:1]
+; TONGA-NEXT: s_mov_b32 s6, 0
+; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB10_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s1, 31
@@ -4868,6 +4878,8 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s19, v13
; GCN-NEXT: v_readfirstlane_b32 s18, v12
+; GCN-NEXT: s_or_b64 s[6:7], s[18:19], s[16:17]
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: v_readfirstlane_b32 s9, v1
; GCN-NEXT: v_readfirstlane_b32 s8, v0
@@ -4878,9 +4890,9 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_readfirstlane_b32 s11, v9
; GCN-NEXT: v_readfirstlane_b32 s10, v8
; GCN-NEXT: v_readfirstlane_b32 s15, v15
-; GCN-NEXT: s_or_b64 s[6:7], s[18:19], s[16:17]
+; GCN-NEXT: s_cmp_lg_u64 s[6:7], 0
; GCN-NEXT: v_readfirstlane_b32 s14, v14
-; GCN-NEXT: s_cbranch_scc0 .LBB12_10
+; GCN-NEXT: s_cbranch_scc0 .LBB12_6
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_ashr_i32 s6, s17, 31
; GCN-NEXT: s_add_u32 s20, s16, s6
@@ -5027,7 +5039,9 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_cselect_b32 s6, s17, s6
; GCN-NEXT: .LBB12_3:
; GCN-NEXT: s_or_b64 s[16:17], s[14:15], s[12:13]
-; GCN-NEXT: s_cbranch_scc0 .LBB12_11
+; GCN-NEXT: s_mov_b32 s16, 0
+; GCN-NEXT: s_cmp_lg_u64 s[16:17], 0
+; GCN-NEXT: s_cbranch_scc0 .LBB12_7
; GCN-NEXT: ; %bb.4:
; GCN-NEXT: s_ashr_i32 s16, s13, 31
; GCN-NEXT: s_add_u32 s18, s12, s16
@@ -5151,7 +5165,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[18:19], s[18:19], s[20:21]
; GCN-NEXT: s_sub_u32 s18, s18, s20
; GCN-NEXT: s_subb_u32 s19, s19, s20
-; GCN-NEXT: s_cbranch_execnz .LBB12_12
+; GCN-NEXT: s_cbranch_execnz .LBB12_8
; GCN-NEXT: .LBB12_5:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s12
; GCN-NEXT: s_sub_i32 s13, 0, s12
@@ -5171,35 +5185,22 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s12, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s12, v0
; GCN-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-NEXT: s_or_b64 s[12:13], s[10:11], s[8:9]
-; GCN-NEXT: s_cbranch_scc1 .LBB12_13
+; GCN-NEXT: s_branch .LBB12_9
; GCN-NEXT: .LBB12_6:
-; GCN-NEXT: ; implicit-def: $sgpr14_sgpr15
-; GCN-NEXT: s_branch .LBB12_14
-; GCN-NEXT: .LBB12_7:
-; GCN-NEXT: v_mov_b32_e32 v4, s14
-; GCN-NEXT: v_mov_b32_e32 v5, s15
-; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
-; GCN-NEXT: s_cbranch_scc1 .LBB12_15
-; GCN-NEXT: .LBB12_8:
-; GCN-NEXT: ; implicit-def: $sgpr10_sgpr11
-; GCN-NEXT: s_branch .LBB12_16
-; GCN-NEXT: .LBB12_9:
-; GCN-NEXT: v_mov_b32_e32 v6, s10
-; GCN-NEXT: v_mov_b32_e32 v7, s11
-; GCN-NEXT: s_branch .LBB12_17
-; GCN-NEXT: .LBB12_10:
; GCN-NEXT: ; implicit-def: $sgpr6_sgpr7
; GCN-NEXT: s_branch .LBB12_2
-; GCN-NEXT: .LBB12_11:
+; GCN-NEXT: .LBB12_7:
; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19
; GCN-NEXT: s_branch .LBB12_5
-; GCN-NEXT: .LBB12_12:
+; GCN-NEXT: .LBB12_8:
; GCN-NEXT: v_mov_b32_e32 v2, s18
; GCN-NEXT: v_mov_b32_e32 v3, s19
+; GCN-NEXT: .LBB12_9:
; GCN-NEXT: s_or_b64 s[12:13], s[10:11], s[8:9]
-; GCN-NEXT: s_cbranch_scc0 .LBB12_6
-; GCN-NEXT: .LBB12_13:
+; GCN-NEXT: s_mov_b32 s12, 0
+; GCN-NEXT: s_cmp_lg_u64 s[12:13], 0
+; GCN-NEXT: s_cbranch_scc0 .LBB12_12
+; GCN-NEXT: ; %bb.10:
; GCN-NEXT: s_ashr_i32 s12, s9, 31
; GCN-NEXT: s_add_u32 s14, s8, s12
; GCN-NEXT: s_mov_b32 s13, s12
@@ -5322,8 +5323,8 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[14:15], s[14:15], s[16:17]
; GCN-NEXT: s_sub_u32 s14, s14, s16
; GCN-NEXT: s_subb_u32 s15, s15, s16
-; GCN-NEXT: s_cbranch_execnz .LBB12_7
-; GCN-NEXT: .LBB12_14:
+; GCN-NEXT: s_cbranch_execnz .LBB12_13
+; GCN-NEXT: .LBB12_11:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8
; GCN-NEXT: s_sub_i32 s9, 0, s8
; GCN-NEXT: v_mov_b32_e32 v5, 0
@@ -5342,9 +5343,19 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s8, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s8, v0
; GCN-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
+; GCN-NEXT: s_branch .LBB12_14
+; GCN-NEXT: .LBB12_12:
+; GCN-NEXT: ; implicit-def: $sgpr14_sgpr15
+; GCN-NEXT: s_branch .LBB12_11
+; GCN-NEXT: .LBB12_13:
+; GCN-NEXT: v_mov_b32_e32 v4, s14
+; GCN-NEXT: v_mov_b32_e32 v5, s15
+; GCN-NEXT: .LBB12_14:
; GCN-NEXT: s_or_b64 s[8:9], s[4:5], s[2:3]
-; GCN-NEXT: s_cbranch_scc0 .LBB12_8
-; GCN-NEXT: .LBB12_15:
+; GCN-NEXT: s_mov_b32 s8, 0
+; GCN-NEXT: s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT: s_cbranch_scc0 .LBB12_17
+; GCN-NEXT: ; %bb.15:
; GCN-NEXT: s_ashr_i32 s8, s3, 31
; GCN-NEXT: s_add_u32 s10, s2, s8
; GCN-NEXT: s_mov_b32 s9, s8
@@ -5467,7 +5478,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_xor_b64 s[10:11], s[10:11], s[12:13]
; GCN-NEXT: s_sub_u32 s10, s10, s12
; GCN-NEXT: s_subb_u32 s11, s11, s12
-; GCN-NEXT: s_cbranch_execnz .LBB12_9
+; GCN-NEXT: s_cbranch_execnz .LBB12_18
; GCN-NEXT: .LBB12_16:
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2
; GCN-NEXT: s_sub_i32 s3, 0, s2
@@ -5487,7 +5498,14 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_subrev_u32_e32 v1, s2, v0
; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0
; GCN-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
+; GCN-NEXT: s_branch .LBB12_19
; GCN-NEXT: .LBB12_17:
+; GCN-NEXT: ; implicit-def: $sgpr10_sgpr11
+; GCN-NEXT: s_branch .LBB12_16
+; GCN-NEXT: .LBB12_18:
+; GCN-NEXT: v_mov_b32_e32 v6, s10
+; GCN-NEXT: v_mov_b32_e32 v7, s11
+; GCN-NEXT: .LBB12_19:
; GCN-NEXT: v_mov_b32_e32 v8, 0
; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: v_mov_b32_e32 v1, s7
@@ -6101,23 +6119,23 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA: ; %bb.0:
; TONGA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: v_mov_b32_e32 v0, s6
; TONGA-NEXT: s_add_u32 s0, s6, 48
-; TONGA-NEXT: v_mov_b32_e32 v1, s7
+; TONGA-NEXT: v_mov_b32_e32 v0, s6
; TONGA-NEXT: s_addc_u32 s1, s7, 0
+; TONGA-NEXT: v_mov_b32_e32 v1, s7
+; TONGA-NEXT: s_add_u32 s2, s6, 32
; TONGA-NEXT: flat_load_dwordx4 v[14:17], v[0:1]
+; TONGA-NEXT: s_addc_u32 s3, s7, 0
+; TONGA-NEXT: v_mov_b32_e32 v0, s2
+; TONGA-NEXT: v_mov_b32_e32 v1, s3
+; TONGA-NEXT: flat_load_dwordx4 v[10:13], v[0:1]
; TONGA-NEXT: v_mov_b32_e32 v0, s0
; TONGA-NEXT: v_mov_b32_e32 v1, s1
-; TONGA-NEXT: s_add_u32 s0, s6, 32
-; TONGA-NEXT: s_addc_u32 s1, s7, 0
-; TONGA-NEXT: v_mov_b32_e32 v3, s1
-; TONGA-NEXT: v_mov_b32_e32 v2, s0
; TONGA-NEXT: s_add_u32 s0, s6, 16
; TONGA-NEXT: s_addc_u32 s1, s7, 0
; TONGA-NEXT: v_mov_b32_e32 v5, s1
-; TONGA-NEXT: flat_load_dwordx4 v[10:13], v[2:3]
-; TONGA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; TONGA-NEXT: v_mov_b32_e32 v4, s0
+; TONGA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; TONGA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; TONGA-NEXT: s_waitcnt vmcnt(3)
; TONGA-NEXT: v_readfirstlane_b32 s3, v15
@@ -6126,6 +6144,8 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: v_readfirstlane_b32 s1, v11
; TONGA-NEXT: v_readfirstlane_b32 s0, v10
; TONGA-NEXT: s_or_b64 s[6:7], s[2:3], s[0:1]
+; TONGA-NEXT: s_mov_b32 s6, 0
+; TONGA-NEXT: s_cmp_lg_u64 s[6:7], 0
; TONGA-NEXT: s_cbranch_scc0 .LBB12_3
; TONGA-NEXT: ; %bb.1:
; TONGA-NEXT: s_ashr_i32 s6, s1, 31
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 5f82a5e4b2fa1..8e7dc4072c012 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -731,11 +731,12 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-LABEL: test_udiv64:
; GFX1032: ; %bb.0: ; %bb
; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s8, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_or_b64 s[4:5], s[2:3], s[0:1]
+; GFX1032-NEXT: s_mov_b32 s4, 0
+; GFX1032-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1032-NEXT: s_cbranch_scc0 .LBB15_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: v_cvt_f32_u32_e32 v0, s0
@@ -750,71 +751,71 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
; GFX1032-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1032-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX1032-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1032-NEXT: v_readfirstlane_b32 s5, v0
-; GFX1032-NEXT: s_mul_i32 s11, s9, s4
-; GFX1032-NEXT: s_mul_hi_u32 s13, s9, s5
-; GFX1032-NEXT: s_mul_i32 s12, s10, s5
+; GFX1032-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1032-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1032-NEXT: s_mul_i32 s11, s9, s5
+; GFX1032-NEXT: s_mul_hi_u32 s13, s9, s8
+; GFX1032-NEXT: s_mul_i32 s12, s10, s8
; GFX1032-NEXT: s_add_i32 s11, s13, s11
-; GFX1032-NEXT: s_mul_i32 s14, s9, s5
+; GFX1032-NEXT: s_mul_i32 s14, s9, s8
; GFX1032-NEXT: s_add_i32 s11, s11, s12
-; GFX1032-NEXT: s_mul_hi_u32 s13, s5, s14
-; GFX1032-NEXT: s_mul_i32 s16, s5, s11
-; GFX1032-NEXT: s_mul_hi_u32 s15, s4, s14
-; GFX1032-NEXT: s_mul_i32 s12, s4, s14
-; GFX1032-NEXT: s_mul_hi_u32 s14, s5, s11
+; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s14
+; GFX1032-NEXT: s_mul_i32 s16, s8, s11
+; GFX1032-NEXT: s_mul_hi_u32 s15, s5, s14
+; GFX1032-NEXT: s_mul_i32 s12, s5, s14
+; GFX1032-NEXT: s_mul_hi_u32 s14, s8, s11
; GFX1032-NEXT: s_add_u32 s13, s13, s16
; GFX1032-NEXT: s_addc_u32 s14, 0, s14
-; GFX1032-NEXT: s_mul_hi_u32 s17, s4, s11
+; GFX1032-NEXT: s_mul_hi_u32 s17, s5, s11
; GFX1032-NEXT: s_add_u32 s12, s13, s12
-; GFX1032-NEXT: s_mul_i32 s11, s4, s11
+; GFX1032-NEXT: s_mul_i32 s11, s5, s11
; GFX1032-NEXT: s_addc_u32 s12, s14, s15
; GFX1032-NEXT: s_addc_u32 s13, s17, 0
; GFX1032-NEXT: s_add_u32 s11, s12, s11
; GFX1032-NEXT: s_addc_u32 s12, 0, s13
-; GFX1032-NEXT: s_add_u32 s5, s5, s11
-; GFX1032-NEXT: s_addc_u32 s4, s4, s12
-; GFX1032-NEXT: s_mul_hi_u32 s11, s9, s5
-; GFX1032-NEXT: s_mul_i32 s12, s9, s5
-; GFX1032-NEXT: s_mul_i32 s9, s9, s4
-; GFX1032-NEXT: s_mul_i32 s10, s10, s5
+; GFX1032-NEXT: s_add_u32 s8, s8, s11
+; GFX1032-NEXT: s_addc_u32 s5, s5, s12
+; GFX1032-NEXT: s_mul_hi_u32 s11, s9, s8
+; GFX1032-NEXT: s_mul_i32 s12, s9, s8
+; GFX1032-NEXT: s_mul_i32 s9, s9, s5
+; GFX1032-NEXT: s_mul_i32 s10, s10, s8
; GFX1032-NEXT: s_add_i32 s9, s11, s9
-; GFX1032-NEXT: s_mul_i32 s11, s4, s12
+; GFX1032-NEXT: s_mul_i32 s11, s5, s12
; GFX1032-NEXT: s_add_i32 s9, s9, s10
-; GFX1032-NEXT: s_mul_hi_u32 s10, s5, s12
-; GFX1032-NEXT: s_mul_i32 s15, s5, s9
-; GFX1032-NEXT: s_mul_hi_u32 s14, s5, s9
+; GFX1032-NEXT: s_mul_hi_u32 s10, s8, s12
+; GFX1032-NEXT: s_mul_i32 s15, s8, s9
+; GFX1032-NEXT: s_mul_hi_u32 s14, s8, s9
; GFX1032-NEXT: s_add_u32 s10, s10, s15
-; GFX1032-NEXT: s_mul_hi_u32 s13, s4, s12
+; GFX1032-NEXT: s_mul_hi_u32 s13, s5, s12
; GFX1032-NEXT: s_addc_u32 s14, 0, s14
-; GFX1032-NEXT: s_mul_hi_u32 s12, s4, s9
+; GFX1032-NEXT: s_mul_hi_u32 s12, s5, s9
; GFX1032-NEXT: s_add_u32 s10, s10, s11
-; GFX1032-NEXT: s_mul_i32 s9, s4, s9
+; GFX1032-NEXT: s_mul_i32 s9, s5, s9
; GFX1032-NEXT: s_addc_u32 s10, s14, s13
; GFX1032-NEXT: s_addc_u32 s11, s12, 0
; GFX1032-NEXT: s_add_u32 s9, s10, s9
; GFX1032-NEXT: s_addc_u32 s10, 0, s11
-; GFX1032-NEXT: s_add_u32 s5, s5, s9
-; GFX1032-NEXT: s_addc_u32 s4, s4, s10
-; GFX1032-NEXT: s_mul_hi_u32 s9, s2, s5
-; GFX1032-NEXT: s_mul_i32 s12, s2, s4
-; GFX1032-NEXT: s_mul_hi_u32 s11, s2, s4
-; GFX1032-NEXT: s_mul_hi_u32 s10, s3, s5
-; GFX1032-NEXT: s_mul_i32 s5, s3, s5
+; GFX1032-NEXT: s_add_u32 s8, s8, s9
+; GFX1032-NEXT: s_addc_u32 s5, s5, s10
+; GFX1032-NEXT: s_mul_hi_u32 s9, s2, s8
+; GFX1032-NEXT: s_mul_i32 s12, s2, s5
+; GFX1032-NEXT: s_mul_hi_u32 s11, s2, s5
+; GFX1032-NEXT: s_mul_hi_u32 s10, s3, s8
+; GFX1032-NEXT: s_mul_i32 s8, s3, s8
; GFX1032-NEXT: s_add_u32 s9, s9, s12
; GFX1032-NEXT: s_addc_u32 s11, 0, s11
-; GFX1032-NEXT: s_mul_hi_u32 s13, s3, s4
-; GFX1032-NEXT: s_add_u32 s5, s9, s5
-; GFX1032-NEXT: s_mul_i32 s4, s3, s4
-; GFX1032-NEXT: s_addc_u32 s5, s11, s10
+; GFX1032-NEXT: s_mul_hi_u32 s13, s3, s5
+; GFX1032-NEXT: s_add_u32 s8, s9, s8
+; GFX1032-NEXT: s_mul_i32 s5, s3, s5
+; GFX1032-NEXT: s_addc_u32 s8, s11, s10
; GFX1032-NEXT: s_addc_u32 s9, s13, 0
-; GFX1032-NEXT: s_add_u32 s4, s5, s4
-; GFX1032-NEXT: s_addc_u32 s5, 0, s9
-; GFX1032-NEXT: s_mul_hi_u32 s9, s0, s4
-; GFX1032-NEXT: s_mul_i32 s10, s0, s5
-; GFX1032-NEXT: s_mul_i32 s11, s1, s4
+; GFX1032-NEXT: s_add_u32 s5, s8, s5
+; GFX1032-NEXT: s_addc_u32 s8, 0, s9
+; GFX1032-NEXT: s_mul_hi_u32 s9, s0, s5
+; GFX1032-NEXT: s_mul_i32 s10, s0, s8
+; GFX1032-NEXT: s_mul_i32 s11, s1, s5
; GFX1032-NEXT: s_add_i32 s9, s9, s10
-; GFX1032-NEXT: s_mul_i32 s10, s0, s4
+; GFX1032-NEXT: s_mul_i32 s10, s0, s5
; GFX1032-NEXT: s_add_i32 s9, s9, s11
; GFX1032-NEXT: s_sub_i32 s11, s3, s9
; GFX1032-NEXT: s_sub_u32 s10, s2, s10
@@ -828,10 +829,10 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cselect_b32 s13, -1, 0
; GFX1032-NEXT: s_cmp_eq_u32 s11, s1
; GFX1032-NEXT: s_cselect_b32 s11, s13, s14
-; GFX1032-NEXT: s_add_u32 s13, s4, 1
-; GFX1032-NEXT: s_addc_u32 s14, s5, 0
-; GFX1032-NEXT: s_add_u32 s15, s4, 2
-; GFX1032-NEXT: s_addc_u32 s16, s5, 0
+; GFX1032-NEXT: s_add_u32 s13, s5, 1
+; GFX1032-NEXT: s_addc_u32 s14, s8, 0
+; GFX1032-NEXT: s_add_u32 s15, s5, 2
+; GFX1032-NEXT: s_addc_u32 s16, s8, 0
; GFX1032-NEXT: s_cmp_lg_u32 s11, 0
; GFX1032-NEXT: s_cselect_b32 s11, s15, s13
; GFX1032-NEXT: s_cselect_b32 s13, s16, s14
@@ -844,14 +845,14 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cmp_eq_u32 s3, s1
; GFX1032-NEXT: s_cselect_b32 s1, s10, s9
; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1032-NEXT: s_cselect_b32 s5, s13, s5
-; GFX1032-NEXT: s_cselect_b32 s4, s11, s4
-; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s8
+; GFX1032-NEXT: s_cselect_b32 s9, s13, s8
+; GFX1032-NEXT: s_cselect_b32 s8, s11, s5
+; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_vccnz .LBB15_3
; GFX1032-NEXT: .LBB15_2:
; GFX1032-NEXT: v_cvt_f32_u32_e32 v0, s0
; GFX1032-NEXT: s_sub_i32 s3, 0, s0
-; GFX1032-NEXT: s_mov_b32 s5, 0
+; GFX1032-NEXT: s_mov_b32 s9, 0
; GFX1032-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1032-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX1032-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -869,15 +870,15 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1032-NEXT: s_cselect_b32 s2, s4, s2
; GFX1032-NEXT: s_add_i32 s3, s1, 1
; GFX1032-NEXT: s_cmp_ge_u32 s2, s0
-; GFX1032-NEXT: s_cselect_b32 s4, s3, s1
+; GFX1032-NEXT: s_cselect_b32 s8, s3, s1
; GFX1032-NEXT: .LBB15_3:
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032-NEXT: v_mov_b32_e32 v0, s8
; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: v_mov_b32_e32 v1, s9
; GFX1032-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] offset:16
; GFX1032-NEXT: s_endpgm
; GFX1032-NEXT: .LBB15_4:
-; GFX1032-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX1032-NEXT: ; implicit-def: $sgpr8_sgpr9
; GFX1032-NEXT: s_branch .LBB15_2
;
; GFX1064-LABEL: test_udiv64:
@@ -887,6 +888,8 @@ define amdgpu_kernel void @test_udiv64(ptr addrspace(1) %arg) #0 {
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_or_b64 s[4:5], s[2:3], s[0:1]
+; GFX1064-NEXT: s_mov_b32 s4, 0
+; GFX1064-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064-NEXT: s_cbranch_scc0 .LBB15_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: v_cvt_f32_u32_e32 v0, s0
>From a8250dbdfe5fff1a5c7b4e0e6a6e930c54598be7 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 10 Nov 2025 18:21:59 -0500
Subject: [PATCH 07/22] clang-format-diff
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 75a3a1ffe2cee..e8631df1275e5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1335,7 +1335,7 @@ SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
MRI.getRegClass(RealDefs[(I + 1) % 2]->getOperand(0).getReg())
->MC->getSizeInBits() *
2 ==
- MRI.getRegClass(MI.getOperand(0).getReg())->MC->getSizeInBits())
+ MRI.getRegClass(MI.getOperand(0).getReg())->MC->getSizeInBits())
return RealDefs[(I + 1) % 2];
return nullptr;
>From 63a9dc5bbc948569b30ea11f85630e6890eb026c Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 23 Jan 2026 16:54:54 -0500
Subject: [PATCH 08/22] capitalization
---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index dd4c519192110..25806e588f32c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -732,7 +732,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
MachineInstr *pierceThroughRegSequence(const MachineInstr &MI) const;
- static bool setsSCCifResultIsNonZero(const MachineInstr &MI) {
+ static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
switch (MI.getOpcode()) {
case AMDGPU::S_ABSDIFF_I32:
case AMDGPU::S_ABS_I32:
>From b58b8afdc40d697cf28d3eb2155fae857df60763 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 23 Jan 2026 17:28:03 -0500
Subject: [PATCH 09/22] Strength reduction test (nonfunctional for now)
---
llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll | 6 ++++++
1 file changed, 6 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
new file mode 100644
index 0000000000000..2a150111091a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
@@ -0,0 +1,6 @@
+define i32 @cmp_strength_reduce(i64 inreg %val0) {
+ %result2 = lshr i64 %val0, 32
+ %cmp = icmp ne i64 %result2, 42
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
>From 74b076d28854786c3005acd0f542d1240a7e9861 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 26 Jan 2026 18:03:35 -0500
Subject: [PATCH 10/22] Still need to implement two comments
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 73 ++++++++++++++++++++++----
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 +-
2 files changed, 65 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e8631df1275e5..5ed984fa693a7 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1313,10 +1313,10 @@ Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
return Reg;
}
-MachineInstr *
+std::pair<MachineInstr *, unsigned>
SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
if (MI.getOpcode() != AMDGPU::REG_SEQUENCE || MI.getNumOperands() != 5)
- return nullptr;
+ return std::make_pair(nullptr,0);
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
int64_t SubRegValues[2];
@@ -1336,9 +1336,9 @@ SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
->MC->getSizeInBits() *
2 ==
MRI.getRegClass(MI.getOperand(0).getReg())->MC->getSizeInBits())
- return RealDefs[(I + 1) % 2];
+ return std::make_pair(RealDefs[(I + 1) % 2],(I+1)%2);
- return nullptr;
+ return std::make_pair(nullptr,0);
}
bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
@@ -11016,7 +11016,62 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue))
return false;
- const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
+ const auto replaceSourceReg = [](MachineInstr &MI, Register Old, Register New) {
+ for (int I = 0; I < MI.getNumOperands(); I++)
+ if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
+ {
+ MI.getOperand(I).setReg(New);
+ break;
+ }
+ };
+
+ const auto replaceSourceImm = [](MachineInstr &MI, uint64_t Old, uint64_t New) {
+ for (int I = 0; I < MI.getNumOperands(); I++)
+ if (MI.getOperand(I).isImm() && MI.getOperand(I).getImm() == Old)
+ {
+ MI.getOperand(I).setImm(New);
+ break;
+ }
+ };
+
+ const auto strengthReduceSCMP = [&CmpInstr, &SrcReg, &CmpValue, MRI,
+ this, &replaceSourceReg,&replaceSourceImm]() -> bool {
+ MachineInstr *Def = MRI->getVRegDef(SrcReg);
+ if (!Def)
+ return false;
+
+ auto RegSequence = pierceThroughRegSequence(*Def);
+ if (!RegSequence.first)
+ return false;
+
+ unsigned OrigOpcode = CmpInstr.getOpcode();
+ if (OrigOpcode != AMDGPU::S_CMP_EQ_U64 &&
+ OrigOpcode != AMDGPU::S_CMP_LG_U64)
+ return false;
+
+ if(!RegSequence.second) //Lower 32 bits nonzero
+ if (CmpValue > UINT32_MAX) {
+ //Build MOV instruction to hard-code EQ ? 0 : 1
+ } else {
+ CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
+ replaceSourceReg(CmpInstr, SrcReg,
+ RegSequence.first->getOperand(0).getReg());
+ SrcReg = RegSequence.first->getOperand(0).getReg();
+ }
+ else // Upper 32 bits nonzero
+ if (CmpValue % UINT32_MAX) {
+ // Build MOV instruction to hard-code EQ ? 0 : 1
+ } else {
+ CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
+ replaceSourceReg(CmpInstr,SrcReg,RegSequence.first->getOperand(0).getReg());
+ replaceSourceImm(CmpInstr, CmpValue, (uint64_t)CmpValue >> 32);
+ RegSequence.first->getOperand(0).getReg();
+ CmpValue = (uint64_t)CmpValue >> 32;
+ }
+ return true;
+ };
+
+ const auto optimizeCmpSelect = [&CmpInstr, &SrcReg, &CmpValue, MRI,
this](bool NeedInversion) -> bool {
if (CmpValue != 0)
return false;
@@ -11025,9 +11080,6 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!Def)
return false;
- if (MachineInstr *RegSequenceDef = pierceThroughRegSequence(*Def))
- Def = RegSequenceDef;
-
// For S_OP that set SCC = DST!=0, do the transformation
//
// s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
@@ -11199,7 +11251,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
case AMDGPU::S_CMPK_GE_I32:
return optimizeCmpAnd(1, 32, false, true);
case AMDGPU::S_CMP_EQ_U64:
- return optimizeCmpAnd(1, 64, true, false);
+ return strengthReduceSCMP() | optimizeCmpAnd(1, 64, true, false);
case AMDGPU::S_CMP_LG_U32:
case AMDGPU::S_CMP_LG_I32:
case AMDGPU::S_CMPK_LG_U32:
@@ -11213,7 +11265,8 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
case AMDGPU::S_CMPK_GT_I32:
return optimizeCmpAnd(0, 32, false, true);
case AMDGPU::S_CMP_LG_U64:
- return optimizeCmpAnd(0, 64, true, false) ||
+ return optimizeCmpAnd(0, 64, true, false) |
+ strengthReduceSCMP() |
optimizeCmpSelect(/*NeedInversion=*/false);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 25806e588f32c..f3ecfd883510e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -730,7 +730,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
}
- MachineInstr *pierceThroughRegSequence(const MachineInstr &MI) const;
+ std::pair<MachineInstr *, unsigned>
+ pierceThroughRegSequence(const MachineInstr &MI) const;
static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
switch (MI.getOpcode()) {
>From 9c369e1bdac044283913c8dcac114a0324e6ecc5 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 27 Jan 2026 15:26:43 -0500
Subject: [PATCH 11/22] Woops
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 5ed984fa693a7..e4689991aeefe 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11065,7 +11065,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr,SrcReg,RegSequence.first->getOperand(0).getReg());
replaceSourceImm(CmpInstr, CmpValue, (uint64_t)CmpValue >> 32);
- RegSequence.first->getOperand(0).getReg();
+ SrcReg = RegSequence.first->getOperand(0).getReg();
CmpValue = (uint64_t)CmpValue >> 32;
}
return true;
>From 019b67bfbff6c71e365f9b098d3ce55b3b79bdd1 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 27 Jan 2026 16:53:56 -0500
Subject: [PATCH 12/22] Done
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e4689991aeefe..9b3a2ae654902 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11051,7 +11051,10 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if(!RegSequence.second) //Lower 32 bits nonzero
if (CmpValue > UINT32_MAX) {
- //Build MOV instruction to hard-code EQ ? 0 : 1
+ // Hard-code EQ ? 0 : 1
+ CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
+ CmpInstr.getOperand(0).setImm(0);
+ CmpInstr.getOperand(1).setImm(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
} else {
CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr, SrcReg,
@@ -11060,7 +11063,10 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
}
else // Upper 32 bits nonzero
if (CmpValue % UINT32_MAX) {
- // Build MOV instruction to hard-code EQ ? 0 : 1
+ // Hard-code EQ ? 0 : 1
+ CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
+ CmpInstr.getOperand(0).setImm(0);
+ CmpInstr.getOperand(1).setImm(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
} else {
CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr,SrcReg,RegSequence.first->getOperand(0).getReg());
>From a4c52ebbcf75c2aff147ab4daf9c37e48627d95b Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 27 Jan 2026 18:11:51 -0500
Subject: [PATCH 13/22] Should be good, probably, maybe
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 49 ++++++++++++++-----
.../CodeGen/AMDGPU/cmp_strength_reduce.ll | 22 +++++++++
2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9b3a2ae654902..9a787ebfa4f95 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11013,16 +11013,26 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!SrcReg || SrcReg.isPhysical())
return false;
- if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue))
- return false;
+ if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue)) {
+ std::pair<MachineInstr *, unsigned> RegSequence2;
+ MachineInstr *Src2Def = MRI->getVRegDef(SrcReg2);
+ if (!Src2Def ||
+ !(Src2Def =
+ (RegSequence2 = pierceThroughRegSequence(*Src2Def)).first) ||
+ !getFoldableImm(Src2Def->getOperand(0).getReg(), *MRI, CmpValue))
+ return false;
+ else if(RegSequence2.second)
+ CmpValue <<= 32;
+ }
const auto replaceSourceReg = [](MachineInstr &MI, Register Old, Register New) {
for (int I = 0; I < MI.getNumOperands(); I++)
if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
{
MI.getOperand(I).setReg(New);
- break;
+ return true;
}
+ return false;
};
const auto replaceSourceImm = [](MachineInstr &MI, uint64_t Old, uint64_t New) {
@@ -11030,12 +11040,24 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (MI.getOperand(I).isImm() && MI.getOperand(I).getImm() == Old)
{
MI.getOperand(I).setImm(New);
- break;
+ return true;
}
+ return false;
+ };
+
+ const auto replaceSourceRegWithImm = [](MachineInstr &MI, Register Old, uint64_t New) {
+ for (int I = 0; I < MI.getNumOperands(); I++)
+ if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
+ {
+ MI.getOperand(I).ChangeToImmediate(New);
+ return true;
+ }
+ return false;
};
- const auto strengthReduceSCMP = [&CmpInstr, &SrcReg, &CmpValue, MRI,
- this, &replaceSourceReg,&replaceSourceImm]() -> bool {
+ const auto strengthReduceSCMP = [&CmpInstr, &SrcReg, &SrcReg2, &CmpValue, MRI,
+ this, &replaceSourceReg, &replaceSourceImm,
+ &replaceSourceRegWithImm]() -> bool {
MachineInstr *Def = MRI->getVRegDef(SrcReg);
if (!Def)
return false;
@@ -11050,11 +11072,11 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
return false;
if(!RegSequence.second) //Lower 32 bits nonzero
- if (CmpValue > UINT32_MAX) {
+ if (CmpValue > UINT32_MAX + 1UL) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
- CmpInstr.getOperand(0).setImm(0);
- CmpInstr.getOperand(1).setImm(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
+ CmpInstr.getOperand(0).ChangeToImmediate(0);
+ CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
} else {
CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr, SrcReg,
@@ -11062,15 +11084,16 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
SrcReg = RegSequence.first->getOperand(0).getReg();
}
else // Upper 32 bits nonzero
- if (CmpValue % UINT32_MAX) {
+ if (CmpValue % (UINT32_MAX + 1UL)) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
- CmpInstr.getOperand(0).setImm(0);
- CmpInstr.getOperand(1).setImm(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
+ CmpInstr.getOperand(0).ChangeToImmediate(0);
+ CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
} else {
CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr,SrcReg,RegSequence.first->getOperand(0).getReg());
- replaceSourceImm(CmpInstr, CmpValue, (uint64_t)CmpValue >> 32);
+ if (!replaceSourceImm(CmpInstr, CmpValue, (uint64_t)CmpValue >> 32))
+ replaceSourceRegWithImm(CmpInstr, SrcReg2, (uint64_t)CmpValue >> 32);
SrcReg = RegSequence.first->getOperand(0).getReg();
CmpValue = (uint64_t)CmpValue >> 32;
}
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
index 2a150111091a6..cf128abbb70f8 100644
--- a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
@@ -4,3 +4,25 @@ define i32 @cmp_strength_reduce(i64 inreg %val0) {
%zext = zext i1 %cmp to i32
ret i32 %zext
}
+
+define i32 @cmp_strength_reduce_low32(i64 inreg %val0) {
+ %result2 = and i64 %val0, 4294967295
+ %cmp = icmp ne i64 %result2, 42
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @cmp_strength_reduce_low31(i64 inreg %val0) {
+ %result2 = and i64 %val0, 2147483647
+ %cmp = icmp ne i64 %result2, 42
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @cmp_strength_reduce_high32(i64 inreg %val0) {
+ %result2 = and i64 %val0, 18446744069414584320
+ %cmp = icmp ne i64 %result2, 17883651179481661440
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
>From 9bbfabd2238de6d5423465669f1bd9f40b99a57e Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 30 Jan 2026 16:06:42 -0500
Subject: [PATCH 14/22] Silence unnecessary warnings; update lit tests
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 12 +++----
.../CodeGen/AMDGPU/cmp_strength_reduce.ll | 31 +++++++++++++++++++
.../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 13 +++-----
llvm/test/CodeGen/AMDGPU/optimize-compare.ll | 7 ++---
.../test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll | 2 +-
llvm/test/CodeGen/AMDGPU/wqm.ll | 10 +++---
6 files changed, 50 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9a787ebfa4f95..429debaba0947 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11026,7 +11026,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
}
const auto replaceSourceReg = [](MachineInstr &MI, Register Old, Register New) {
- for (int I = 0; I < MI.getNumOperands(); I++)
+ for (unsigned I = 0; I < MI.getNumOperands(); I++)
if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
{
MI.getOperand(I).setReg(New);
@@ -11036,8 +11036,8 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
};
const auto replaceSourceImm = [](MachineInstr &MI, uint64_t Old, uint64_t New) {
- for (int I = 0; I < MI.getNumOperands(); I++)
- if (MI.getOperand(I).isImm() && MI.getOperand(I).getImm() == Old)
+ for (unsigned I = 0; I < MI.getNumOperands(); I++)
+ if (MI.getOperand(I).isImm() && (uint64_t)MI.getOperand(I).getImm() == Old)
{
MI.getOperand(I).setImm(New);
return true;
@@ -11046,7 +11046,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
};
const auto replaceSourceRegWithImm = [](MachineInstr &MI, Register Old, uint64_t New) {
- for (int I = 0; I < MI.getNumOperands(); I++)
+ for (unsigned I = 0; I < MI.getNumOperands(); I++)
if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
{
MI.getOperand(I).ChangeToImmediate(New);
@@ -11072,7 +11072,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
return false;
if(!RegSequence.second) //Lower 32 bits nonzero
- if (CmpValue > UINT32_MAX + 1UL) {
+ if ((uint64_t)CmpValue > UINT32_MAX + 1UL) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
@@ -11084,7 +11084,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
SrcReg = RegSequence.first->getOperand(0).getReg();
}
else // Upper 32 bits nonzero
- if (CmpValue % (UINT32_MAX + 1UL)) {
+ if ((uint64_t)CmpValue % (UINT32_MAX + 1UL)) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
index cf128abbb70f8..6ea217a2e313f 100644
--- a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
@@ -1,4 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s
define i32 @cmp_strength_reduce(i64 inreg %val0) {
+; CHECK-LABEL: cmp_strength_reduce:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_cmp_lg_u32 s17, 42
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = lshr i64 %val0, 32
%cmp = icmp ne i64 %result2, 42
%zext = zext i1 %cmp to i32
@@ -6,6 +15,13 @@ define i32 @cmp_strength_reduce(i64 inreg %val0) {
}
define i32 @cmp_strength_reduce_low32(i64 inreg %val0) {
+; CHECK-LABEL: cmp_strength_reduce_low32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_cmp_lg_u32 s16, 42
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 4294967295
%cmp = icmp ne i64 %result2, 42
%zext = zext i1 %cmp to i32
@@ -13,6 +29,14 @@ define i32 @cmp_strength_reduce_low32(i64 inreg %val0) {
}
define i32 @cmp_strength_reduce_low31(i64 inreg %val0) {
+; CHECK-LABEL: cmp_strength_reduce_low31:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_bitset0_b32 s16, 31
+; CHECK-NEXT: s_cmp_lg_u32 s16, 42
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 2147483647
%cmp = icmp ne i64 %result2, 42
%zext = zext i1 %cmp to i32
@@ -20,6 +44,13 @@ define i32 @cmp_strength_reduce_low31(i64 inreg %val0) {
}
define i32 @cmp_strength_reduce_high32(i64 inreg %val0) {
+; CHECK-LABEL: cmp_strength_reduce_high32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_cmp_lg_u32 s17, 0xf82f7dff
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 18446744069414584320
%cmp = icmp ne i64 %result2, 17883651179481661440
%zext = zext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
index 91aba09e942f0..b00538f34e291 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
@@ -126,9 +126,8 @@ define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {
define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) {
; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0
+; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0
+; CHECK-NEXT: s_cmp_eq_u32 vcc_lo, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB7_2
; CHECK-NEXT: ; %bb.1: ; %true
; CHECK-NEXT: s_mov_b32 s0, 42
@@ -152,11 +151,10 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) {
; DAGISEL: ; %bb.0:
; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0
; DAGISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1
-; DAGISEL-NEXT: s_mov_b32 s1, 0
; DAGISEL-NEXT: s_and_b32 s0, vcc_lo, s0
; DAGISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; DAGISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0
-; DAGISEL-NEXT: s_cmp_eq_u64 s[0:1], 0
+; DAGISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; DAGISEL-NEXT: s_cmp_eq_u32 vcc_lo, 0
; DAGISEL-NEXT: s_cbranch_scc1 .LBB8_2
; DAGISEL-NEXT: ; %bb.1: ; %true
; DAGISEL-NEXT: s_mov_b32 s0, 42
@@ -170,9 +168,8 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) {
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0
; GISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1
-; GISEL-NEXT: s_mov_b32 s1, 0
; GISEL-NEXT: s_and_b32 s0, vcc_lo, s0
-; GISEL-NEXT: s_cmp_eq_u64 s[0:1], 0
+; GISEL-NEXT: s_cmp_eq_u32 s0, 0
; GISEL-NEXT: s_cbranch_scc1 .LBB8_2
; GISEL-NEXT: ; %bb.1: ; %true
; GISEL-NEXT: s_mov_b32 s0, 42
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-compare.ll b/llvm/test/CodeGen/AMDGPU/optimize-compare.ll
index 51db31d99d7e2..55df7ff31f357 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-compare.ll
+++ b/llvm/test/CodeGen/AMDGPU/optimize-compare.ll
@@ -63,11 +63,10 @@ define amdgpu_kernel void @if_masked_0x8000000000000000(i64 %arg, ptr addrspace(
; GCN-LABEL: if_masked_0x8000000000000000:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s0, 0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_and_b32 s1, s1, 0x80000000
-; GCN-NEXT: s_cmp_eq_u64 s[0:1], 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_and_b32 s0, s1, 0x80000000
+; GCN-NEXT: s_cmp_eq_u32 s0, 0
; GCN-NEXT: s_cselect_b32 s0, 22, 33
; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: global_store_dword v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
index 7c6ae6cad5ae7..c0c50af14b5ef 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
@@ -21,7 +21,7 @@ define amdgpu_kernel void @copy_to_vreg_1(i32 %0) {
; GCN-NEXT: v_readfirstlane_b32 s1, v1
; GCN-NEXT: s_cselect_b32 s4, s0, s1
; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: s_cmp_lg_u64 0, 0
+; GCN-NEXT: s_cmp_lg_u32 0, 0
; GCN-NEXT: s_mov_b64 s[0:1], 0
; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index aa68ee0e77934..5759cfb855d96 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -3475,9 +3475,8 @@ define amdgpu_gs void @wqm_init_exec_wwm() {
; GFX9-W64-LABEL: wqm_init_exec_wwm:
; GFX9-W64: ; %bb.0:
; GFX9-W64-NEXT: s_mov_b64 exec, 0
-; GFX9-W64-NEXT: s_mov_b32 s1, 0
-; GFX9-W64-NEXT: s_mov_b32 s0, s1
-; GFX9-W64-NEXT: s_cmp_eq_u64 s[0:1], 0
+; GFX9-W64-NEXT: s_mov_b32 s0, 0
+; GFX9-W64-NEXT: s_cmp_eq_u32 s0, 0
; GFX9-W64-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0
; GFX9-W64-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
@@ -3487,10 +3486,9 @@ define amdgpu_gs void @wqm_init_exec_wwm() {
; GFX10-W32-LABEL: wqm_init_exec_wwm:
; GFX10-W32: ; %bb.0:
; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0
-; GFX10-W32-NEXT: s_mov_b32 s1, 0
+; GFX10-W32-NEXT: s_mov_b32 s0, 0
; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-W32-NEXT: s_mov_b32 s0, s1
-; GFX10-W32-NEXT: s_cmp_eq_u64 s[0:1], 0
+; GFX10-W32-NEXT: s_cmp_eq_u32 s0, 0
; GFX10-W32-NEXT: s_cselect_b32 s0, -1, 0
; GFX10-W32-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s0
; GFX10-W32-NEXT: exp mrt0 off, off, off, off
>From 51b51c3c76e21d2df855171fece5b843c13d6613 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 2 Feb 2026 12:27:03 -0500
Subject: [PATCH 15/22] Silence more unnecessary warnings
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 429debaba0947..0bc51190b39a6 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11280,7 +11280,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
case AMDGPU::S_CMPK_GE_I32:
return optimizeCmpAnd(1, 32, false, true);
case AMDGPU::S_CMP_EQ_U64:
- return strengthReduceSCMP() | optimizeCmpAnd(1, 64, true, false);
+ return (int)strengthReduceSCMP() | (int)optimizeCmpAnd(1, 64, true, false);
case AMDGPU::S_CMP_LG_U32:
case AMDGPU::S_CMP_LG_I32:
case AMDGPU::S_CMPK_LG_U32:
@@ -11294,9 +11294,8 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
case AMDGPU::S_CMPK_GT_I32:
return optimizeCmpAnd(0, 32, false, true);
case AMDGPU::S_CMP_LG_U64:
- return optimizeCmpAnd(0, 64, true, false) |
- strengthReduceSCMP() |
- optimizeCmpSelect(/*NeedInversion=*/false);
+ return (int)optimizeCmpAnd(0, 64, true, false) | (int)strengthReduceSCMP() |
+ (int)optimizeCmpSelect(/*NeedInversion=*/false);
}
return false;
>From 17b258026751dede3a4b3371dbbb8e7713b84e8d Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 6 Feb 2026 13:38:23 -0500
Subject: [PATCH 16/22] Make pierceThroughRegSequence take additional parameter
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0bc51190b39a6..f253551eb2a21 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1314,11 +1314,11 @@ Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
}
std::pair<MachineInstr *, unsigned>
-SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI) const {
+SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI) const {
if (MI.getOpcode() != AMDGPU::REG_SEQUENCE || MI.getNumOperands() != 5)
return std::make_pair(nullptr,0);
- const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
int64_t SubRegValues[2];
bool SubRegIsConst[2];
MachineInstr *RealDefs[2];
@@ -11018,7 +11018,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
MachineInstr *Src2Def = MRI->getVRegDef(SrcReg2);
if (!Src2Def ||
!(Src2Def =
- (RegSequence2 = pierceThroughRegSequence(*Src2Def)).first) ||
+ (RegSequence2 = pierceThroughRegSequence(*Src2Def,*MRI)).first) ||
!getFoldableImm(Src2Def->getOperand(0).getReg(), *MRI, CmpValue))
return false;
else if(RegSequence2.second)
@@ -11062,7 +11062,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!Def)
return false;
- auto RegSequence = pierceThroughRegSequence(*Def);
+ auto RegSequence = pierceThroughRegSequence(*Def,*MRI);
if (!RegSequence.first)
return false;
>From 067c0edeb288128363e816c5293d6fe0ae970715 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 6 Feb 2026 13:38:57 -0500
Subject: [PATCH 17/22] clang-format-diff
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 52 +++++++++++++++-----------
1 file changed, 30 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f253551eb2a21..47ba00017c216 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1317,7 +1317,7 @@ std::pair<MachineInstr *, unsigned>
SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI,
const MachineRegisterInfo &MRI) const {
if (MI.getOpcode() != AMDGPU::REG_SEQUENCE || MI.getNumOperands() != 5)
- return std::make_pair(nullptr,0);
+ return std::make_pair(nullptr, 0);
int64_t SubRegValues[2];
bool SubRegIsConst[2];
@@ -1336,9 +1336,9 @@ SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI,
->MC->getSizeInBits() *
2 ==
MRI.getRegClass(MI.getOperand(0).getReg())->MC->getSizeInBits())
- return std::make_pair(RealDefs[(I + 1) % 2],(I+1)%2);
+ return std::make_pair(RealDefs[(I + 1) % 2], (I + 1) % 2);
- return std::make_pair(nullptr,0);
+ return std::make_pair(nullptr, 0);
}
bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
@@ -11017,38 +11017,39 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
std::pair<MachineInstr *, unsigned> RegSequence2;
MachineInstr *Src2Def = MRI->getVRegDef(SrcReg2);
if (!Src2Def ||
- !(Src2Def =
- (RegSequence2 = pierceThroughRegSequence(*Src2Def,*MRI)).first) ||
+ !(Src2Def = (RegSequence2 = pierceThroughRegSequence(*Src2Def, *MRI))
+ .first) ||
!getFoldableImm(Src2Def->getOperand(0).getReg(), *MRI, CmpValue))
return false;
- else if(RegSequence2.second)
+ else if (RegSequence2.second)
CmpValue <<= 32;
}
- const auto replaceSourceReg = [](MachineInstr &MI, Register Old, Register New) {
+ const auto replaceSourceReg = [](MachineInstr &MI, Register Old,
+ Register New) {
for (unsigned I = 0; I < MI.getNumOperands(); I++)
- if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
- {
+ if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old) {
MI.getOperand(I).setReg(New);
return true;
}
return false;
};
- const auto replaceSourceImm = [](MachineInstr &MI, uint64_t Old, uint64_t New) {
+ const auto replaceSourceImm = [](MachineInstr &MI, uint64_t Old,
+ uint64_t New) {
for (unsigned I = 0; I < MI.getNumOperands(); I++)
- if (MI.getOperand(I).isImm() && (uint64_t)MI.getOperand(I).getImm() == Old)
- {
+ if (MI.getOperand(I).isImm() &&
+ (uint64_t)MI.getOperand(I).getImm() == Old) {
MI.getOperand(I).setImm(New);
return true;
}
return false;
};
- const auto replaceSourceRegWithImm = [](MachineInstr &MI, Register Old, uint64_t New) {
+ const auto replaceSourceRegWithImm = [](MachineInstr &MI, Register Old,
+ uint64_t New) {
for (unsigned I = 0; I < MI.getNumOperands(); I++)
- if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old)
- {
+ if (MI.getOperand(I).isReg() && MI.getOperand(I).getReg() == Old) {
MI.getOperand(I).ChangeToImmediate(New);
return true;
}
@@ -11062,7 +11063,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!Def)
return false;
- auto RegSequence = pierceThroughRegSequence(*Def,*MRI);
+ auto RegSequence = pierceThroughRegSequence(*Def, *MRI);
if (!RegSequence.first)
return false;
@@ -11071,14 +11072,17 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
OrigOpcode != AMDGPU::S_CMP_LG_U64)
return false;
- if(!RegSequence.second) //Lower 32 bits nonzero
+ if (!RegSequence.second) // Lower 32 bits nonzero
if ((uint64_t)CmpValue > UINT32_MAX + 1UL) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
- CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
+ CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode ==
+ AMDGPU::S_CMP_EQ_U64);
} else {
- CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
+ CmpInstr.setDesc(get(OrigOpcode == AMDGPU::S_CMP_EQ_U64
+ ? AMDGPU::S_CMP_EQ_U32
+ : AMDGPU::S_CMP_LG_U32));
replaceSourceReg(CmpInstr, SrcReg,
RegSequence.first->getOperand(0).getReg());
SrcReg = RegSequence.first->getOperand(0).getReg();
@@ -11088,10 +11092,14 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
- CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode==AMDGPU::S_CMP_EQ_U64);
+ CmpInstr.getOperand(1).ChangeToImmediate(OrigOpcode ==
+ AMDGPU::S_CMP_EQ_U64);
} else {
- CmpInstr.setDesc(get(OrigOpcode==AMDGPU::S_CMP_EQ_U64 ? AMDGPU::S_CMP_EQ_U32 : AMDGPU::S_CMP_LG_U32));
- replaceSourceReg(CmpInstr,SrcReg,RegSequence.first->getOperand(0).getReg());
+ CmpInstr.setDesc(get(OrigOpcode == AMDGPU::S_CMP_EQ_U64
+ ? AMDGPU::S_CMP_EQ_U32
+ : AMDGPU::S_CMP_LG_U32));
+ replaceSourceReg(CmpInstr, SrcReg,
+ RegSequence.first->getOperand(0).getReg());
if (!replaceSourceImm(CmpInstr, CmpValue, (uint64_t)CmpValue >> 32))
replaceSourceRegWithImm(CmpInstr, SrcReg2, (uint64_t)CmpValue >> 32);
SrcReg = RegSequence.first->getOperand(0).getReg();
>From 64784cf3bb5d8f8d87df24875d831012cbd5ac6d Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 6 Feb 2026 13:44:00 -0500
Subject: [PATCH 18/22] Fix prototype
---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f3ecfd883510e..aba8e950c1f2b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -731,7 +731,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
std::pair<MachineInstr *, unsigned>
- pierceThroughRegSequence(const MachineInstr &MI) const;
+ pierceThroughRegSequence(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI) const;
static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
switch (MI.getOpcode()) {
>From 14f7509a03a5d4d0a92f19e70e9c6a5b9d4221db Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 7 Jul 2026 10:27:27 -0500
Subject: [PATCH 19/22] Update tests
---
llvm/test/CodeGen/AMDGPU/clmul.ll | 2518 ++++++++---------
.../CodeGen/AMDGPU/cmp_strength_reduce.ll | 16 +-
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 2 +-
.../AMDGPU/redundant-cmp-reg-sequence.ll | 10 +-
llvm/test/CodeGen/AMDGPU/s_cmp_0.ll | 2 -
5 files changed, 1230 insertions(+), 1318 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/clmul.ll b/llvm/test/CodeGen/AMDGPU/clmul.ll
index fea2ff0681a88..3f56ce7043645 100644
--- a/llvm/test/CodeGen/AMDGPU/clmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/clmul.ll
@@ -799,228 +799,208 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
;
; VI-LABEL: test_clmulr_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b32 s10, s2
-; VI-NEXT: s_mov_b32 s11, s3
+; VI-NEXT: s_mov_b32 s6, s2
+; VI-NEXT: s_mov_b32 s7, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s8, s6
-; VI-NEXT: s_mov_b32 s9, s7
-; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; VI-NEXT: s_mov_b32 s0, s4
-; VI-NEXT: s_mov_b32 s7, 0
-; VI-NEXT: s_mov_b32 s1, s5
-; VI-NEXT: s_mov_b32 s11, s7
-; VI-NEXT: s_mov_b32 s9, s7
-; VI-NEXT: s_mov_b32 s13, s7
-; VI-NEXT: s_mov_b32 s15, s7
-; VI-NEXT: s_mov_b32 s17, s7
-; VI-NEXT: s_mov_b32 s19, s7
-; VI-NEXT: s_mov_b32 s21, s7
-; VI-NEXT: s_mov_b32 s23, s7
-; VI-NEXT: s_mov_b32 s25, s7
-; VI-NEXT: s_mov_b32 s27, s7
-; VI-NEXT: s_mov_b32 s29, s7
-; VI-NEXT: s_mov_b32 s31, s7
-; VI-NEXT: s_mov_b32 s35, s7
-; VI-NEXT: s_mov_b32 s37, s7
-; VI-NEXT: s_mov_b32 s39, s7
-; VI-NEXT: s_mov_b32 s41, s7
-; VI-NEXT: s_mov_b32 s43, s7
-; VI-NEXT: s_mov_b32 s45, s7
-; VI-NEXT: s_mov_b32 s47, s7
+; VI-NEXT: s_mov_b32 s4, s10
+; VI-NEXT: s_mov_b32 s5, s11
+; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_mov_b32 s0, s8
+; VI-NEXT: s_mov_b32 s1, s9
+; VI-NEXT: s_mov_b32 s11, s5
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: s_bfe_i32 s5, s4, 0x10000
-; VI-NEXT: s_lshl_b64 s[48:49], s[6:7], 1
-; VI-NEXT: s_and_b32 s10, s4, 2
-; VI-NEXT: s_and_b32 s8, s5, s6
-; VI-NEXT: s_cmp_eq_u64 s[10:11], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s49
-; VI-NEXT: s_cselect_b32 s10, 0, s48
-; VI-NEXT: s_lshl_b64 s[48:49], s[6:7], 2
-; VI-NEXT: s_and_b32 s12, s4, 4
-; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s49
-; VI-NEXT: s_cselect_b32 s10, 0, s48
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 3
-; VI-NEXT: s_and_b32 s14, s4, 8
+; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: v_readfirstlane_b32 s4, v0
+; VI-NEXT: s_bfe_i32 s7, s6, 0x10000
+; VI-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; VI-NEXT: s_and_b32 s12, s6, 2
+; VI-NEXT: s_and_b32 s10, s7, s4
+; VI-NEXT: s_cmp_eq_u32 s12, 0
+; VI-NEXT: s_cselect_b32 s9, 0, s9
+; VI-NEXT: s_cselect_b32 s8, 0, s8
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
+; VI-NEXT: s_and_b32 s7, s6, 4
+; VI-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 3
+; VI-NEXT: s_and_b32 s7, s6, 8
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[14:15], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 4
-; VI-NEXT: s_and_b32 s16, s4, 16
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 4
+; VI-NEXT: s_and_b32 s7, s6, 16
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[16:17], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 5
-; VI-NEXT: s_and_b32 s18, s4, 32
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
+; VI-NEXT: s_and_b32 s7, s6, 32
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[18:19], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 6
-; VI-NEXT: s_and_b32 s20, s4, 64
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 6
+; VI-NEXT: s_and_b32 s7, s6, 64
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[20:21], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 7
-; VI-NEXT: s_and_b32 s22, s4, 0x80
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 7
+; VI-NEXT: s_and_b32 s7, s6, 0x80
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[22:23], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 8
-; VI-NEXT: s_and_b32 s24, s4, 0x100
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 8
+; VI-NEXT: s_and_b32 s7, s6, 0x100
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[24:25], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 9
-; VI-NEXT: s_and_b32 s26, s4, 0x200
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 9
+; VI-NEXT: s_and_b32 s7, s6, 0x200
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[26:27], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 10
-; VI-NEXT: s_and_b32 s28, s4, 0x400
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 10
+; VI-NEXT: s_and_b32 s7, s6, 0x400
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[28:29], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 11
-; VI-NEXT: s_and_b32 s30, s4, 0x800
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 11
+; VI-NEXT: s_and_b32 s7, s6, 0x800
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[30:31], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 12
-; VI-NEXT: s_and_b32 s34, s4, 0x1000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 12
+; VI-NEXT: s_and_b32 s7, s6, 0x1000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[34:35], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 13
-; VI-NEXT: s_and_b32 s36, s4, 0x2000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 13
+; VI-NEXT: s_and_b32 s7, s6, 0x2000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[36:37], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 14
-; VI-NEXT: s_and_b32 s38, s4, 0x4000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 14
+; VI-NEXT: s_and_b32 s7, s6, 0x4000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[38:39], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 15
-; VI-NEXT: s_and_b32 s40, s4, 0x8000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 15
+; VI-NEXT: s_and_b32 s7, s6, 0x8000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[40:41], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 16
-; VI-NEXT: s_and_b32 s42, s4, 0x10000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 16
+; VI-NEXT: s_and_b32 s7, s6, 0x10000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[42:43], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 17
-; VI-NEXT: s_and_b32 s44, s4, 0x20000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 17
+; VI-NEXT: s_and_b32 s7, s6, 0x20000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[44:45], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 18
-; VI-NEXT: s_and_b32 s46, s4, 0x40000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 18
+; VI-NEXT: s_and_b32 s7, s6, 0x40000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[46:47], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 19
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 19
+; VI-NEXT: s_and_b32 s7, s6, 0x80000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_and_b32 s10, s4, 0x80000
-; VI-NEXT: s_mov_b32 s11, s7
-; VI-NEXT: s_cmp_eq_u64 s[10:11], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 20
+; VI-NEXT: s_and_b32 s7, s6, 0x100000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 20
-; VI-NEXT: s_and_b32 s12, s4, 0x100000
-; VI-NEXT: s_mov_b32 s13, s7
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 21
+; VI-NEXT: s_and_b32 s7, s6, 0x200000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 21
-; VI-NEXT: s_and_b32 s12, s4, 0x200000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 22
+; VI-NEXT: s_and_b32 s7, s6, 0x400000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 22
-; VI-NEXT: s_and_b32 s12, s4, 0x400000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 23
+; VI-NEXT: s_and_b32 s7, s6, 0x800000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 23
-; VI-NEXT: s_and_b32 s12, s4, 0x800000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 24
+; VI-NEXT: s_and_b32 s7, s6, 0x1000000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 24
-; VI-NEXT: s_and_b32 s12, s4, 0x1000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 25
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 25
-; VI-NEXT: s_and_b32 s12, s4, 0x2000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_and_b32 s7, s6, 0x2000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 26
-; VI-NEXT: s_and_b32 s12, s4, 0x4000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 26
+; VI-NEXT: s_and_b32 s7, s6, 0x4000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 27
-; VI-NEXT: s_and_b32 s12, s4, 0x8000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 27
+; VI-NEXT: s_and_b32 s7, s6, 0x8000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 28
-; VI-NEXT: s_and_b32 s12, s4, 0x10000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 28
+; VI-NEXT: s_and_b32 s7, s6, 0x10000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 29
-; VI-NEXT: s_and_b32 s12, s4, 0x20000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 29
+; VI-NEXT: s_and_b32 s7, s6, 0x20000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 30
-; VI-NEXT: s_and_b32 s12, s4, 2.0
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 30
+; VI-NEXT: s_and_b32 s7, s6, 2.0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 31
-; VI-NEXT: s_cmp_gt_i32 s4, -1
-; VI-NEXT: s_cselect_b32 s5, 0, s7
-; VI-NEXT: s_cselect_b32 s4, 0, s6
+; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 31
+; VI-NEXT: s_cmp_gt_i32 s6, -1
+; VI-NEXT: s_cselect_b32 s5, 0, s5
+; VI-NEXT: s_cselect_b32 s4, 0, s4
; VI-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; VI-NEXT: s_lshr_b64 s[4:5], s[4:5], 31
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -1038,220 +1018,200 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_mov_b32 s4, s10
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; GFX9-NEXT: s_mov_b32 s0, s8
; GFX9-NEXT: s_mov_b32 s5, 0
-; GFX9-NEXT: s_mov_b32 s11, s5
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s13, s5
-; GFX9-NEXT: s_mov_b32 s15, s5
-; GFX9-NEXT: s_mov_b32 s17, s5
-; GFX9-NEXT: s_mov_b32 s19, s5
-; GFX9-NEXT: s_mov_b32 s21, s5
-; GFX9-NEXT: s_mov_b32 s23, s5
-; GFX9-NEXT: s_mov_b32 s25, s5
-; GFX9-NEXT: s_mov_b32 s27, s5
-; GFX9-NEXT: s_mov_b32 s29, s5
-; GFX9-NEXT: s_mov_b32 s31, s5
-; GFX9-NEXT: s_mov_b32 s35, s5
-; GFX9-NEXT: s_mov_b32 s37, s5
-; GFX9-NEXT: s_mov_b32 s39, s5
-; GFX9-NEXT: s_mov_b32 s41, s5
-; GFX9-NEXT: s_mov_b32 s43, s5
-; GFX9-NEXT: s_mov_b32 s45, s5
-; GFX9-NEXT: s_mov_b32 s47, s5
+; GFX9-NEXT: s_mov_b32 s0, s8
; GFX9-NEXT: s_mov_b32 s1, s9
+; GFX9-NEXT: s_mov_b32 s11, s5
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_readfirstlane_b32 s8, v1
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_bfe_i32 s6, s8, 0x10000
-; GFX9-NEXT: s_lshl_b64 s[48:49], s[4:5], 1
-; GFX9-NEXT: s_and_b32 s10, s8, 2
-; GFX9-NEXT: s_and_b32 s6, s6, s4
-; GFX9-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s49
-; GFX9-NEXT: s_cselect_b32 s10, 0, s48
-; GFX9-NEXT: s_lshl_b64 s[48:49], s[4:5], 2
-; GFX9-NEXT: s_and_b32 s12, s8, 4
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s49
-; GFX9-NEXT: s_cselect_b32 s10, 0, s48
+; GFX9-NEXT: s_bfe_i32 s7, s6, 0x10000
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; GFX9-NEXT: s_and_b32 s12, s6, 2
+; GFX9-NEXT: s_and_b32 s10, s7, s4
+; GFX9-NEXT: s_cmp_eq_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s9, 0, s9
+; GFX9-NEXT: s_cselect_b32 s8, 0, s8
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
+; GFX9-NEXT: s_and_b32 s7, s6, 4
+; GFX9-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 3
-; GFX9-NEXT: s_and_b32 s14, s8, 8
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[14:15], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 8
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 4
-; GFX9-NEXT: s_and_b32 s16, s8, 16
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 16
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
-; GFX9-NEXT: s_and_b32 s18, s8, 32
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[18:19], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 32
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 6
-; GFX9-NEXT: s_and_b32 s20, s8, 64
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[20:21], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 64
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 7
-; GFX9-NEXT: s_and_b32 s22, s8, 0x80
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[22:23], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x80
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 8
-; GFX9-NEXT: s_and_b32 s24, s8, 0x100
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[24:25], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x100
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 9
-; GFX9-NEXT: s_and_b32 s26, s8, 0x200
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[26:27], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x200
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 10
-; GFX9-NEXT: s_and_b32 s28, s8, 0x400
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[28:29], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x400
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 11
-; GFX9-NEXT: s_and_b32 s30, s8, 0x800
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[30:31], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x800
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 12
-; GFX9-NEXT: s_and_b32 s34, s8, 0x1000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[34:35], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x1000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 13
-; GFX9-NEXT: s_and_b32 s36, s8, 0x2000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[36:37], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x2000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 14
-; GFX9-NEXT: s_and_b32 s38, s8, 0x4000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[38:39], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x4000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 15
-; GFX9-NEXT: s_and_b32 s40, s8, 0x8000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[40:41], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x8000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 16
-; GFX9-NEXT: s_and_b32 s42, s8, 0x10000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[42:43], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x10000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 17
-; GFX9-NEXT: s_and_b32 s44, s8, 0x20000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[44:45], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x20000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 18
-; GFX9-NEXT: s_and_b32 s46, s8, 0x40000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[46:47], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x40000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 19
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_and_b32 s10, s8, 0x80000
-; GFX9-NEXT: s_mov_b32 s11, s5
-; GFX9-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x80000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 20
-; GFX9-NEXT: s_and_b32 s12, s8, 0x100000
-; GFX9-NEXT: s_mov_b32 s13, s5
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 21
-; GFX9-NEXT: s_and_b32 s12, s8, 0x200000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 22
-; GFX9-NEXT: s_and_b32 s12, s8, 0x400000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 23
-; GFX9-NEXT: s_and_b32 s12, s8, 0x800000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 24
-; GFX9-NEXT: s_and_b32 s12, s8, 0x1000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 25
-; GFX9-NEXT: s_and_b32 s12, s8, 0x2000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 20
+; GFX9-NEXT: s_and_b32 s7, s6, 0x100000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 21
+; GFX9-NEXT: s_and_b32 s7, s6, 0x200000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 22
+; GFX9-NEXT: s_and_b32 s7, s6, 0x400000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 23
+; GFX9-NEXT: s_and_b32 s7, s6, 0x800000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 24
+; GFX9-NEXT: s_and_b32 s7, s6, 0x1000000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 25
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_and_b32 s7, s6, 0x2000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 26
-; GFX9-NEXT: s_and_b32 s12, s8, 0x4000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x4000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 27
-; GFX9-NEXT: s_and_b32 s12, s8, 0x8000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x8000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 28
-; GFX9-NEXT: s_and_b32 s12, s8, 0x10000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x10000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 29
-; GFX9-NEXT: s_and_b32 s12, s8, 0x20000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x20000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 30
-; GFX9-NEXT: s_and_b32 s12, s8, 2.0
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 2.0
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 31
-; GFX9-NEXT: s_cmp_gt_i32 s8, -1
+; GFX9-NEXT: s_cmp_gt_i32 s6, -1
; GFX9-NEXT: s_cselect_b32 s5, 0, s5
; GFX9-NEXT: s_cselect_b32 s4, 0, s4
-; GFX9-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 31
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1269,194 +1229,192 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: s_mov_b32 s9, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 s11, s3
-; GFX10-NEXT: s_mov_b32 s9, s3
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-NEXT: s_bfe_i32 s5, s4, 0x10000
-; GFX10-NEXT: s_and_b32 s10, s4, 2
-; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 1
-; GFX10-NEXT: s_and_b32 s8, s5, s2
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX10-NEXT: s_and_b32 s11, s4, 2
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX10-NEXT: s_and_b32 s10, s5, s2
+; GFX10-NEXT: s_cmp_eq_u32 s11, 0
+; GFX10-NEXT: s_mov_b32 s11, s3
+; GFX10-NEXT: s_cselect_b32 s9, 0, s9
+; GFX10-NEXT: s_cselect_b32 s8, 0, s8
+; GFX10-NEXT: s_and_b32 s5, s4, 4
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 2
+; GFX10-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 8
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 3
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 4
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 32
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 64
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 6
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x80
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 7
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x100
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 8
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x200
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 9
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x400
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 10
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x800
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 11
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x1000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 12
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x2000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 13
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x4000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 14
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x8000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 15
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x10000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 16
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x20000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 17
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x40000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 18
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x80000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 19
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x100000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 20
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x200000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 21
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x400000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 22
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x800000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 23
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x1000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 24
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x2000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 25
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x4000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 26
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x8000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 27
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x10000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 28
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x20000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 29
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 2.0
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 30
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_cselect_b32 s13, 0, s13
-; GFX10-NEXT: s_cselect_b32 s12, 0, s12
-; GFX10-NEXT: s_and_b32 s10, s4, 4
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 2
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 8
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 3
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 16
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 4
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 32
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 5
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 64
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 6
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x80
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 7
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x100
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 8
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x200
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 9
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x400
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 10
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x800
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 11
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x1000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 12
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x2000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 13
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x4000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 14
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x8000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 15
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x10000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 16
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x20000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 17
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x40000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 18
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x80000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 19
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x100000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 20
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x200000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 21
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x400000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 22
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x800000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 23
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x1000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 24
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x2000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 25
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x4000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 26
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x8000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 27
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x10000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 28
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x20000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 29
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 2.0
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 30
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s11, 0, s15
-; GFX10-NEXT: s_cselect_b32 s10, 0, s14
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 31
; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX10-NEXT: s_cmp_gt_i32 s4, -1
@@ -1481,193 +1439,192 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX11-NEXT: s_mov_b32 s9, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT: s_mov_b32 s11, s3
-; GFX11-NEXT: s_mov_b32 s9, s3
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s2, v0
; GFX11-NEXT: s_bfe_i32 s5, s4, 0x10000
-; GFX11-NEXT: s_and_b32 s10, s4, 2
-; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 1
-; GFX11-NEXT: s_and_b32 s8, s5, s2
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX11-NEXT: s_and_b32 s11, s4, 2
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX11-NEXT: s_and_b32 s10, s5, s2
+; GFX11-NEXT: s_cmp_eq_u32 s11, 0
+; GFX11-NEXT: s_mov_b32 s11, s3
+; GFX11-NEXT: s_cselect_b32 s9, 0, s9
+; GFX11-NEXT: s_cselect_b32 s8, 0, s8
+; GFX11-NEXT: s_and_b32 s5, s4, 4
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 2
+; GFX11-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 8
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 3
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 16
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 4
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 32
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 64
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 6
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x80
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 7
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x100
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 8
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x200
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 9
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x400
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 10
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x800
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 11
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x1000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 12
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x2000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 13
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x4000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 14
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x8000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 15
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x10000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 16
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x20000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 17
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x40000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 18
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x80000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 19
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x100000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 20
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x200000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 21
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x400000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 22
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x800000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 23
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x1000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 24
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x2000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 25
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x4000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 26
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x8000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 27
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x10000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 28
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x20000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 29
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 2.0
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 30
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_cselect_b32 s13, 0, s13
-; GFX11-NEXT: s_cselect_b32 s12, 0, s12
-; GFX11-NEXT: s_and_b32 s10, s4, 4
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 2
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 8
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 3
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 16
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 4
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 32
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 5
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 64
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 6
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x80
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 7
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x100
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 8
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x200
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 9
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x400
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 10
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x800
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 11
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x1000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 12
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x2000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 13
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x4000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 14
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x8000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 15
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x10000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 16
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x20000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 17
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x40000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 18
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x80000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 19
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x100000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 20
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x200000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 21
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x400000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 22
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x800000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 23
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x1000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 24
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x2000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 25
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x4000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 26
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x8000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 27
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x10000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 28
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x20000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 29
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 2.0
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 30
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s11, 0, s15
-; GFX11-NEXT: s_cselect_b32 s10, 0, s14
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 31
; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX11-NEXT: s_cmp_gt_i32 s4, -1
@@ -2277,228 +2234,208 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
;
; VI-LABEL: test_clmulh_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b32 s10, s2
-; VI-NEXT: s_mov_b32 s11, s3
+; VI-NEXT: s_mov_b32 s6, s2
+; VI-NEXT: s_mov_b32 s7, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s8, s6
-; VI-NEXT: s_mov_b32 s9, s7
-; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; VI-NEXT: s_mov_b32 s0, s4
-; VI-NEXT: s_mov_b32 s7, 0
-; VI-NEXT: s_mov_b32 s1, s5
-; VI-NEXT: s_mov_b32 s11, s7
-; VI-NEXT: s_mov_b32 s9, s7
-; VI-NEXT: s_mov_b32 s13, s7
-; VI-NEXT: s_mov_b32 s15, s7
-; VI-NEXT: s_mov_b32 s17, s7
-; VI-NEXT: s_mov_b32 s19, s7
-; VI-NEXT: s_mov_b32 s21, s7
-; VI-NEXT: s_mov_b32 s23, s7
-; VI-NEXT: s_mov_b32 s25, s7
-; VI-NEXT: s_mov_b32 s27, s7
-; VI-NEXT: s_mov_b32 s29, s7
-; VI-NEXT: s_mov_b32 s31, s7
-; VI-NEXT: s_mov_b32 s35, s7
-; VI-NEXT: s_mov_b32 s37, s7
-; VI-NEXT: s_mov_b32 s39, s7
-; VI-NEXT: s_mov_b32 s41, s7
-; VI-NEXT: s_mov_b32 s43, s7
-; VI-NEXT: s_mov_b32 s45, s7
-; VI-NEXT: s_mov_b32 s47, s7
+; VI-NEXT: s_mov_b32 s4, s10
+; VI-NEXT: s_mov_b32 s5, s11
+; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_mov_b32 s0, s8
+; VI-NEXT: s_mov_b32 s1, s9
+; VI-NEXT: s_mov_b32 s11, s5
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: s_bfe_i32 s5, s4, 0x10000
-; VI-NEXT: s_lshl_b64 s[48:49], s[6:7], 1
-; VI-NEXT: s_and_b32 s10, s4, 2
-; VI-NEXT: s_and_b32 s8, s5, s6
-; VI-NEXT: s_cmp_eq_u64 s[10:11], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s49
-; VI-NEXT: s_cselect_b32 s10, 0, s48
-; VI-NEXT: s_lshl_b64 s[48:49], s[6:7], 2
-; VI-NEXT: s_and_b32 s12, s4, 4
-; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s49
-; VI-NEXT: s_cselect_b32 s10, 0, s48
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 3
-; VI-NEXT: s_and_b32 s14, s4, 8
+; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: v_readfirstlane_b32 s4, v0
+; VI-NEXT: s_bfe_i32 s7, s6, 0x10000
+; VI-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; VI-NEXT: s_and_b32 s12, s6, 2
+; VI-NEXT: s_and_b32 s10, s7, s4
+; VI-NEXT: s_cmp_eq_u32 s12, 0
+; VI-NEXT: s_cselect_b32 s9, 0, s9
+; VI-NEXT: s_cselect_b32 s8, 0, s8
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
+; VI-NEXT: s_and_b32 s7, s6, 4
+; VI-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 3
+; VI-NEXT: s_and_b32 s7, s6, 8
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[14:15], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 4
-; VI-NEXT: s_and_b32 s16, s4, 16
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 4
+; VI-NEXT: s_and_b32 s7, s6, 16
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[16:17], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 5
-; VI-NEXT: s_and_b32 s18, s4, 32
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
+; VI-NEXT: s_and_b32 s7, s6, 32
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[18:19], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 6
-; VI-NEXT: s_and_b32 s20, s4, 64
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 6
+; VI-NEXT: s_and_b32 s7, s6, 64
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[20:21], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 7
-; VI-NEXT: s_and_b32 s22, s4, 0x80
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 7
+; VI-NEXT: s_and_b32 s7, s6, 0x80
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[22:23], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 8
-; VI-NEXT: s_and_b32 s24, s4, 0x100
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 8
+; VI-NEXT: s_and_b32 s7, s6, 0x100
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[24:25], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 9
-; VI-NEXT: s_and_b32 s26, s4, 0x200
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 9
+; VI-NEXT: s_and_b32 s7, s6, 0x200
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[26:27], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 10
-; VI-NEXT: s_and_b32 s28, s4, 0x400
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 10
+; VI-NEXT: s_and_b32 s7, s6, 0x400
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[28:29], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 11
-; VI-NEXT: s_and_b32 s30, s4, 0x800
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 11
+; VI-NEXT: s_and_b32 s7, s6, 0x800
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[30:31], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 12
-; VI-NEXT: s_and_b32 s34, s4, 0x1000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 12
+; VI-NEXT: s_and_b32 s7, s6, 0x1000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[34:35], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 13
-; VI-NEXT: s_and_b32 s36, s4, 0x2000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 13
+; VI-NEXT: s_and_b32 s7, s6, 0x2000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[36:37], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 14
-; VI-NEXT: s_and_b32 s38, s4, 0x4000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 14
+; VI-NEXT: s_and_b32 s7, s6, 0x4000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[38:39], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 15
-; VI-NEXT: s_and_b32 s40, s4, 0x8000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 15
+; VI-NEXT: s_and_b32 s7, s6, 0x8000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[40:41], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 16
-; VI-NEXT: s_and_b32 s42, s4, 0x10000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 16
+; VI-NEXT: s_and_b32 s7, s6, 0x10000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[42:43], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 17
-; VI-NEXT: s_and_b32 s44, s4, 0x20000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 17
+; VI-NEXT: s_and_b32 s7, s6, 0x20000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[44:45], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 18
-; VI-NEXT: s_and_b32 s46, s4, 0x40000
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 18
+; VI-NEXT: s_and_b32 s7, s6, 0x40000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_cmp_eq_u64 s[46:47], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
-; VI-NEXT: s_lshl_b64 s[12:13], s[6:7], 19
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 19
+; VI-NEXT: s_and_b32 s7, s6, 0x80000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_and_b32 s10, s4, 0x80000
-; VI-NEXT: s_mov_b32 s11, s7
-; VI-NEXT: s_cmp_eq_u64 s[10:11], 0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s13
; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 20
+; VI-NEXT: s_and_b32 s7, s6, 0x100000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 20
-; VI-NEXT: s_and_b32 s12, s4, 0x100000
-; VI-NEXT: s_mov_b32 s13, s7
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 21
+; VI-NEXT: s_and_b32 s7, s6, 0x200000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 21
-; VI-NEXT: s_and_b32 s12, s4, 0x200000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 22
+; VI-NEXT: s_and_b32 s7, s6, 0x400000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 22
-; VI-NEXT: s_and_b32 s12, s4, 0x400000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 23
+; VI-NEXT: s_and_b32 s7, s6, 0x800000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 23
-; VI-NEXT: s_and_b32 s12, s4, 0x800000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 24
+; VI-NEXT: s_and_b32 s7, s6, 0x1000000
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 24
-; VI-NEXT: s_and_b32 s12, s4, 0x1000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
+; VI-NEXT: s_lshl_b64 s[12:13], s[4:5], 25
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 25
-; VI-NEXT: s_and_b32 s12, s4, 0x2000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
-; VI-NEXT: s_cselect_b32 s11, 0, s11
-; VI-NEXT: s_cselect_b32 s10, 0, s10
+; VI-NEXT: s_and_b32 s7, s6, 0x2000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s11, 0, s13
+; VI-NEXT: s_cselect_b32 s10, 0, s12
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 26
-; VI-NEXT: s_and_b32 s12, s4, 0x4000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 26
+; VI-NEXT: s_and_b32 s7, s6, 0x4000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 27
-; VI-NEXT: s_and_b32 s12, s4, 0x8000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 27
+; VI-NEXT: s_and_b32 s7, s6, 0x8000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 28
-; VI-NEXT: s_and_b32 s12, s4, 0x10000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 28
+; VI-NEXT: s_and_b32 s7, s6, 0x10000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 29
-; VI-NEXT: s_and_b32 s12, s4, 0x20000000
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 29
+; VI-NEXT: s_and_b32 s7, s6, 0x20000000
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[10:11], s[6:7], 30
-; VI-NEXT: s_and_b32 s12, s4, 2.0
-; VI-NEXT: s_cmp_eq_u64 s[12:13], 0
+; VI-NEXT: s_lshl_b64 s[10:11], s[4:5], 30
+; VI-NEXT: s_and_b32 s7, s6, 2.0
+; VI-NEXT: s_cmp_eq_u32 s7, 0
; VI-NEXT: s_cselect_b32 s11, 0, s11
; VI-NEXT: s_cselect_b32 s10, 0, s10
; VI-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 31
-; VI-NEXT: s_cmp_gt_i32 s4, -1
-; VI-NEXT: s_cselect_b32 s5, 0, s7
-; VI-NEXT: s_cselect_b32 s4, 0, s6
+; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 31
+; VI-NEXT: s_cmp_gt_i32 s6, -1
+; VI-NEXT: s_cselect_b32 s5, 0, s5
+; VI-NEXT: s_cselect_b32 s4, 0, s4
; VI-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, s5
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -2515,220 +2452,200 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_mov_b32 s4, s10
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; GFX9-NEXT: s_mov_b32 s0, s8
; GFX9-NEXT: s_mov_b32 s5, 0
-; GFX9-NEXT: s_mov_b32 s11, s5
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s13, s5
-; GFX9-NEXT: s_mov_b32 s15, s5
-; GFX9-NEXT: s_mov_b32 s17, s5
-; GFX9-NEXT: s_mov_b32 s19, s5
-; GFX9-NEXT: s_mov_b32 s21, s5
-; GFX9-NEXT: s_mov_b32 s23, s5
-; GFX9-NEXT: s_mov_b32 s25, s5
-; GFX9-NEXT: s_mov_b32 s27, s5
-; GFX9-NEXT: s_mov_b32 s29, s5
-; GFX9-NEXT: s_mov_b32 s31, s5
-; GFX9-NEXT: s_mov_b32 s35, s5
-; GFX9-NEXT: s_mov_b32 s37, s5
-; GFX9-NEXT: s_mov_b32 s39, s5
-; GFX9-NEXT: s_mov_b32 s41, s5
-; GFX9-NEXT: s_mov_b32 s43, s5
-; GFX9-NEXT: s_mov_b32 s45, s5
-; GFX9-NEXT: s_mov_b32 s47, s5
+; GFX9-NEXT: s_mov_b32 s0, s8
; GFX9-NEXT: s_mov_b32 s1, s9
+; GFX9-NEXT: s_mov_b32 s11, s5
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_readfirstlane_b32 s8, v1
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_bfe_i32 s6, s8, 0x10000
-; GFX9-NEXT: s_lshl_b64 s[48:49], s[4:5], 1
-; GFX9-NEXT: s_and_b32 s10, s8, 2
-; GFX9-NEXT: s_and_b32 s6, s6, s4
-; GFX9-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s49
-; GFX9-NEXT: s_cselect_b32 s10, 0, s48
-; GFX9-NEXT: s_lshl_b64 s[48:49], s[4:5], 2
-; GFX9-NEXT: s_and_b32 s12, s8, 4
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s49
-; GFX9-NEXT: s_cselect_b32 s10, 0, s48
+; GFX9-NEXT: s_bfe_i32 s7, s6, 0x10000
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; GFX9-NEXT: s_and_b32 s12, s6, 2
+; GFX9-NEXT: s_and_b32 s10, s7, s4
+; GFX9-NEXT: s_cmp_eq_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s9, 0, s9
+; GFX9-NEXT: s_cselect_b32 s8, 0, s8
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
+; GFX9-NEXT: s_and_b32 s7, s6, 4
+; GFX9-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 3
-; GFX9-NEXT: s_and_b32 s14, s8, 8
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[14:15], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 8
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 4
-; GFX9-NEXT: s_and_b32 s16, s8, 16
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 16
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
-; GFX9-NEXT: s_and_b32 s18, s8, 32
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[18:19], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 32
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 6
-; GFX9-NEXT: s_and_b32 s20, s8, 64
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[20:21], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 64
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 7
-; GFX9-NEXT: s_and_b32 s22, s8, 0x80
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[22:23], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x80
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 8
-; GFX9-NEXT: s_and_b32 s24, s8, 0x100
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[24:25], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x100
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 9
-; GFX9-NEXT: s_and_b32 s26, s8, 0x200
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[26:27], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x200
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 10
-; GFX9-NEXT: s_and_b32 s28, s8, 0x400
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[28:29], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x400
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 11
-; GFX9-NEXT: s_and_b32 s30, s8, 0x800
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[30:31], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x800
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 12
-; GFX9-NEXT: s_and_b32 s34, s8, 0x1000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[34:35], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x1000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 13
-; GFX9-NEXT: s_and_b32 s36, s8, 0x2000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[36:37], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x2000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 14
-; GFX9-NEXT: s_and_b32 s38, s8, 0x4000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[38:39], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x4000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 15
-; GFX9-NEXT: s_and_b32 s40, s8, 0x8000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[40:41], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x8000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 16
-; GFX9-NEXT: s_and_b32 s42, s8, 0x10000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[42:43], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x10000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 17
-; GFX9-NEXT: s_and_b32 s44, s8, 0x20000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[44:45], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x20000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 18
-; GFX9-NEXT: s_and_b32 s46, s8, 0x40000
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_cmp_eq_u64 s[46:47], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x40000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 19
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_and_b32 s10, s8, 0x80000
-; GFX9-NEXT: s_mov_b32 s11, s5
-; GFX9-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x80000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s13
; GFX9-NEXT: s_cselect_b32 s10, 0, s12
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 20
-; GFX9-NEXT: s_and_b32 s12, s8, 0x100000
-; GFX9-NEXT: s_mov_b32 s13, s5
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 21
-; GFX9-NEXT: s_and_b32 s12, s8, 0x200000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 22
-; GFX9-NEXT: s_and_b32 s12, s8, 0x400000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 23
-; GFX9-NEXT: s_and_b32 s12, s8, 0x800000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 24
-; GFX9-NEXT: s_and_b32 s12, s8, 0x1000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 25
-; GFX9-NEXT: s_and_b32 s12, s8, 0x2000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 20
+; GFX9-NEXT: s_and_b32 s7, s6, 0x100000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 21
+; GFX9-NEXT: s_and_b32 s7, s6, 0x200000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 22
+; GFX9-NEXT: s_and_b32 s7, s6, 0x400000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 23
+; GFX9-NEXT: s_and_b32 s7, s6, 0x800000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 24
+; GFX9-NEXT: s_and_b32 s7, s6, 0x1000000
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[4:5], 25
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_and_b32 s7, s6, 0x2000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s11, 0, s13
+; GFX9-NEXT: s_cselect_b32 s10, 0, s12
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 26
-; GFX9-NEXT: s_and_b32 s12, s8, 0x4000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x4000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 27
-; GFX9-NEXT: s_and_b32 s12, s8, 0x8000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x8000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 28
-; GFX9-NEXT: s_and_b32 s12, s8, 0x10000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x10000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 29
-; GFX9-NEXT: s_and_b32 s12, s8, 0x20000000
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 0x20000000
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], 30
-; GFX9-NEXT: s_and_b32 s12, s8, 2.0
-; GFX9-NEXT: s_cmp_eq_u64 s[12:13], 0
+; GFX9-NEXT: s_and_b32 s7, s6, 2.0
+; GFX9-NEXT: s_cmp_eq_u32 s7, 0
; GFX9-NEXT: s_cselect_b32 s11, 0, s11
; GFX9-NEXT: s_cselect_b32 s10, 0, s10
-; GFX9-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 31
-; GFX9-NEXT: s_cmp_gt_i32 s8, -1
+; GFX9-NEXT: s_cmp_gt_i32 s6, -1
; GFX9-NEXT: s_cselect_b32 s5, 0, s5
; GFX9-NEXT: s_cselect_b32 s4, 0, s4
-; GFX9-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
@@ -2745,194 +2662,192 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: s_mov_b32 s9, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_mov_b32 s11, s3
-; GFX10-NEXT: s_mov_b32 s9, s3
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-NEXT: s_bfe_i32 s5, s4, 0x10000
-; GFX10-NEXT: s_and_b32 s10, s4, 2
-; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 1
-; GFX10-NEXT: s_and_b32 s8, s5, s2
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX10-NEXT: s_and_b32 s11, s4, 2
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX10-NEXT: s_and_b32 s10, s5, s2
+; GFX10-NEXT: s_cmp_eq_u32 s11, 0
+; GFX10-NEXT: s_mov_b32 s11, s3
+; GFX10-NEXT: s_cselect_b32 s9, 0, s9
+; GFX10-NEXT: s_cselect_b32 s8, 0, s8
+; GFX10-NEXT: s_and_b32 s5, s4, 4
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 2
+; GFX10-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 8
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 3
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 4
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 32
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 64
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 6
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x80
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 7
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x100
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 8
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x200
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 9
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x400
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 10
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x800
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 11
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x1000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 12
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x2000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 13
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x4000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 14
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x8000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 15
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x10000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 16
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x20000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 17
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x40000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 18
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x80000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 19
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x100000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 20
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x200000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 21
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x400000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 22
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x800000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 23
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x1000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 24
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x2000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 25
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x4000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 26
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x8000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 27
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x10000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 28
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 0x20000000
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 29
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
+; GFX10-NEXT: s_and_b32 s5, s4, 2.0
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], 30
+; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_cselect_b32 s13, 0, s13
-; GFX10-NEXT: s_cselect_b32 s12, 0, s12
-; GFX10-NEXT: s_and_b32 s10, s4, 4
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 2
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 8
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 3
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 16
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 4
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 32
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 5
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 64
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 6
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x80
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 7
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x100
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 8
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x200
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 9
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x400
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 10
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x800
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 11
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x1000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 12
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x2000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 13
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x4000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 14
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x8000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 15
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x10000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 16
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x20000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 17
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x40000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 18
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x80000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 19
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x100000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 20
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x200000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 21
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x400000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 22
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x800000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 23
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x1000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 24
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x2000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 25
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x4000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 26
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x8000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 27
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x10000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 28
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 0x20000000
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 29
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s13, 0, s15
-; GFX10-NEXT: s_cselect_b32 s12, 0, s14
-; GFX10-NEXT: s_and_b32 s10, s4, 2.0
-; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], 30
-; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s11, 0, s15
-; GFX10-NEXT: s_cselect_b32 s10, 0, s14
+; GFX10-NEXT: s_cselect_b32 s11, 0, s13
+; GFX10-NEXT: s_cselect_b32 s10, 0, s12
; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 31
; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX10-NEXT: s_cmp_gt_i32 s4, -1
@@ -2956,193 +2871,192 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX11-NEXT: s_mov_b32 s9, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-NEXT: s_mov_b32 s11, s3
-; GFX11-NEXT: s_mov_b32 s9, s3
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s2, v0
; GFX11-NEXT: s_bfe_i32 s5, s4, 0x10000
-; GFX11-NEXT: s_and_b32 s10, s4, 2
-; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 1
-; GFX11-NEXT: s_and_b32 s8, s5, s2
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
+; GFX11-NEXT: s_and_b32 s11, s4, 2
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX11-NEXT: s_and_b32 s10, s5, s2
+; GFX11-NEXT: s_cmp_eq_u32 s11, 0
+; GFX11-NEXT: s_mov_b32 s11, s3
+; GFX11-NEXT: s_cselect_b32 s9, 0, s9
+; GFX11-NEXT: s_cselect_b32 s8, 0, s8
+; GFX11-NEXT: s_and_b32 s5, s4, 4
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 2
+; GFX11-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 8
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 3
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 16
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 4
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 32
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 64
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 6
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x80
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 7
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x100
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 8
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x200
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 9
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x400
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 10
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x800
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 11
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x1000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 12
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x2000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 13
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x4000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 14
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x8000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 15
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x10000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 16
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x20000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 17
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x40000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 18
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x80000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 19
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x100000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 20
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x200000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 21
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x400000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 22
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x800000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 23
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x1000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 24
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x2000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 25
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x4000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 26
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x8000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 27
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x10000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 28
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 0x20000000
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 29
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
+; GFX11-NEXT: s_and_b32 s5, s4, 2.0
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], 30
+; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
; GFX11-NEXT: s_mov_b32 s5, s1
-; GFX11-NEXT: s_cselect_b32 s13, 0, s13
-; GFX11-NEXT: s_cselect_b32 s12, 0, s12
-; GFX11-NEXT: s_and_b32 s10, s4, 4
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 2
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 8
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 3
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 16
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 4
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 32
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 5
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 64
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 6
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x80
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 7
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x100
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 8
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x200
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 9
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x400
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 10
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x800
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 11
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x1000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 12
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x2000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 13
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x4000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 14
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x8000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 15
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x10000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 16
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x20000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 17
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x40000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 18
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x80000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 19
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x100000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 20
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x200000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 21
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x400000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 22
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x800000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 23
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x1000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 24
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x2000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 25
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x4000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 26
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x8000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 27
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x10000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 28
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 0x20000000
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 29
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s13, 0, s15
-; GFX11-NEXT: s_cselect_b32 s12, 0, s14
-; GFX11-NEXT: s_and_b32 s10, s4, 2.0
-; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], 30
-; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s11, 0, s15
-; GFX11-NEXT: s_cselect_b32 s10, 0, s14
+; GFX11-NEXT: s_cselect_b32 s11, 0, s13
+; GFX11-NEXT: s_cselect_b32 s10, 0, s12
; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 31
; GFX11-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
; GFX11-NEXT: s_cmp_gt_i32 s4, -1
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
index 6ea217a2e313f..f53baf5435504 100644
--- a/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/cmp_strength_reduce.ll
@@ -5,8 +5,8 @@ define i32 @cmp_strength_reduce(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_cmp_lg_u32 s17, 42
-; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = lshr i64 %val0, 32
%cmp = icmp ne i64 %result2, 42
@@ -19,8 +19,8 @@ define i32 @cmp_strength_reduce_low32(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_cmp_lg_u32 s16, 42
-; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 4294967295
%cmp = icmp ne i64 %result2, 42
@@ -34,8 +34,8 @@ define i32 @cmp_strength_reduce_low31(i64 inreg %val0) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_bitset0_b32 s16, 31
; CHECK-NEXT: s_cmp_lg_u32 s16, 42
-; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 2147483647
%cmp = icmp ne i64 %result2, 42
@@ -48,8 +48,8 @@ define i32 @cmp_strength_reduce_high32(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_cmp_lg_u32 s17, 0xf82f7dff
-; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result2 = and i64 %val0, 18446744069414584320
%cmp = icmp ne i64 %result2, 17883651179481661440
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index f2e7192df56ea..7444b4721fbe7 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -1115,7 +1115,7 @@ define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7
; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB15_2
; GFX942-SDAG-NEXT: .LBB15_3: ; %memset.pattern-expansion-residual-cond
-; GFX942-SDAG-NEXT: s_cmp_eq_u64 s[4:5], 0
+; GFX942-SDAG-NEXT: s_cmp_eq_u32 s4, 0
; GFX942-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT: s_lshl_b32 s6, s17, 3
diff --git a/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
index 750a9027d47a4..6424347ffc5b5 100644
--- a/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
+++ b/llvm/test/CodeGen/AMDGPU/redundant-cmp-reg-sequence.ll
@@ -4,12 +4,12 @@ define amdgpu_ps i64 @ordertest(i64 inreg %val0) {
; CHECK-LABEL: ordertest:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_lshr_b32 s0, s1, 2
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, s[0:1]
-; CHECK-NEXT: v_xor_b32_e32 v0, v2, v0
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: s_cselect_b64 s[2:3], 1, 0
+; CHECK-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; CHECK-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: ; return to shader part epilog
%shl = lshr i64 %val0, 34
%result = and i64 %shl, 4294967295
diff --git a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
index 8877cda9c12e6..d6a6ecd174d9b 100644
--- a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
@@ -433,8 +433,6 @@ define amdgpu_ps i32 @bfe_i64(i64 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_bfe_i64 s[2:3], s[0:1], 0x80000
; CHECK-NEXT: s_and_b32 s0, s0, 0xff
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; use s[2:3]
>From c506995d6dce6b01a89140f88b145a9366295dab Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 7 Jul 2026 14:20:25 -0500
Subject: [PATCH 20/22] Fix bug on Windows (thanks Claude)
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 23d23b6ba1c27..1d5d5426f8c1f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11355,7 +11355,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
return false;
if (!RegSequence.second) // Lower 32 bits nonzero
- if ((uint64_t)CmpValue > UINT32_MAX + 1UL) {
+ if ((uint64_t)CmpValue > (uint64_t)UINT32_MAX + 1) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
@@ -11370,7 +11370,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
SrcReg = RegSequence.first->getOperand(0).getReg();
}
else // Upper 32 bits nonzero
- if ((uint64_t)CmpValue % (UINT32_MAX + 1UL)) {
+ if ((uint64_t)CmpValue % ((uint64_t)UINT32_MAX + 1UL)) {
// Hard-code EQ ? 0 : 1
CmpInstr.setDesc(get(AMDGPU::S_CMP_EQ_U32));
CmpInstr.getOperand(0).ChangeToImmediate(0);
>From aae1505a72669bb303e19ca2aff1697cfe494125 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 28 Jul 2026 15:59:03 -0500
Subject: [PATCH 21/22] Final review changes
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 19 ++++++++++++++-----
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 ++--
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 1d5d5426f8c1f..f6d3c7ff844c2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1138,23 +1138,32 @@ int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
return Opcode;
}
+// For a 64-bit value defined by a REG_SEQUENCE with half of the result being 0,
+// find the instruction that defines exactly the bits in the other half, and
+// return a pair containing that instruction and an unsigned indicating which 32
+// bits are nonzero. The unsigned is 0 if the lower 32 bits are nonzero and 1
+// if the upper 32 bits are nonzero.
std::pair<MachineInstr *, unsigned>
-SIInstrInfo::pierceThroughRegSequence(const MachineInstr &MI,
- const MachineRegisterInfo &MRI) const {
+SIInstrInfo::analyzePartiallyZeroRegSequence(
+ const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
if (MI.getOpcode() != AMDGPU::REG_SEQUENCE || MI.getNumOperands() != 5)
return std::make_pair(nullptr, 0);
int64_t SubRegValues[2];
bool SubRegIsConst[2];
MachineInstr *RealDefs[2];
+
+ // Visit subreg-index operands of the REG_SEQUENCE instruction (MI)
for (unsigned I : {2, 4}) {
unsigned ArrayIdx = MI.getOperand(I).getImm() == AMDGPU::sub0 ? 0 : 1;
Register Subreg = MI.getOperand(I - 1).getReg();
- RealDefs[ArrayIdx] = MRI.getUniqueVRegDef(Subreg);
+ RealDefs[ArrayIdx] = MRI.getVRegDef(Subreg);
SubRegIsConst[ArrayIdx] = getConstValDefinedInReg(
*RealDefs[ArrayIdx], Subreg, SubRegValues[ArrayIdx]);
}
+ // Visit indices of SubRegValues and SubRegIsConst arrays filled in by
+ // previous loop
for (unsigned I : {0, 1})
if (SubRegIsConst[I] && !SubRegValues[I] &&
MRI.getRegClass(RealDefs[(I + 1) % 2]->getOperand(0).getReg())
@@ -11299,7 +11308,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
std::pair<MachineInstr *, unsigned> RegSequence2;
MachineInstr *Src2Def = MRI->getVRegDef(SrcReg2);
if (!Src2Def ||
- !(Src2Def = (RegSequence2 = pierceThroughRegSequence(*Src2Def, *MRI))
+ !(Src2Def = (RegSequence2 = analyzePartiallyZeroRegSequence(*Src2Def, *MRI))
.first) ||
!getFoldableImm(Src2Def->getOperand(0).getReg(), *MRI, CmpValue))
return false;
@@ -11345,7 +11354,7 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (!Def)
return false;
- auto RegSequence = pierceThroughRegSequence(*Def, *MRI);
+ auto RegSequence = analyzePartiallyZeroRegSequence(*Def, *MRI);
if (!RegSequence.first)
return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f342c2121c3b3..0dce65f838c81 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -753,8 +753,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
std::pair<MachineInstr *, unsigned>
- pierceThroughRegSequence(const MachineInstr &MI,
- const MachineRegisterInfo &MRI) const;
+ analyzePartiallyZeroRegSequence(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI) const;
static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
switch (MI.getOpcode()) {
>From c7a8b7a79058b5fa36b7009105769a06fe807711 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Tue, 28 Jul 2026 19:03:49 -0500
Subject: [PATCH 22/22] Fix formatting
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f6d3c7ff844c2..a5f4d4869a310 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11308,8 +11308,9 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
std::pair<MachineInstr *, unsigned> RegSequence2;
MachineInstr *Src2Def = MRI->getVRegDef(SrcReg2);
if (!Src2Def ||
- !(Src2Def = (RegSequence2 = analyzePartiallyZeroRegSequence(*Src2Def, *MRI))
- .first) ||
+ !(Src2Def =
+ (RegSequence2 = analyzePartiallyZeroRegSequence(*Src2Def, *MRI))
+ .first) ||
!getFoldableImm(Src2Def->getOperand(0).getReg(), *MRI, CmpValue))
return false;
else if (RegSequence2.second)
More information about the llvm-commits
mailing list