[llvm] [AMDGPU][GlobalISel] Fold zext(carry) into add/sub-with-carry (PR #224093)
Mirko BrkuĊĦanin via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 02:26:13 PDT 2026
https://github.com/mbrkusanin updated https://github.com/llvm/llvm-project/pull/224093
>From 3a61b50c6d0978bd7ae1530de4bd84d4bd63951b Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Wed, 16 Sep 2026 19:14:13 +0200
Subject: [PATCH 1/4] precommit tests
---
...alizer-combiner-carry-into-uadde-usube.mir | 258 +++++++++++++
.../AMDGPU/combine-carry-into-uadde-usube.ll | 352 ++++++++++++++++++
2 files changed, 610 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
new file mode 100644
index 000000000000000..6649f61fcf5d0ee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
@@ -0,0 +1,258 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name: add_zext_uaddo_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_zext_uaddo_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UADDO]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+ %4:_(s32) = G_ZEXT %3(s1)
+ %5:_(s32) = G_ADD %2, %4
+ $vgpr0 = COPY %5(s32)
+...
+
+---
+name: add_zext_uaddo_carry_commuted
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_zext_uaddo_carry_commuted
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[UADDO]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+ %4:_(s32) = G_ZEXT %3(s1)
+ %5:_(s32) = G_ADD %4, %2
+ $vgpr0 = COPY %5(s32)
+...
+
+---
+name: sub_zext_usubo_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: sub_zext_usubo_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[USUBO]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+ %4:_(s32) = G_ZEXT %3(s1)
+ %5:_(s32) = G_SUB %2, %4
+ $vgpr0 = COPY %5(s32)
+...
+
+---
+name: sub_zext_usubo_carry_lhs_not_folded
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: sub_zext_usubo_carry_lhs_not_folded
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[ZEXT]], [[USUBO]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+ %4:_(s32) = G_ZEXT %3(s1)
+ %5:_(s32) = G_SUB %4, %2
+ $vgpr0 = COPY %5(s32)
+...
+
+---
+name: add_zext_uadde_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-LABEL: name: add_zext_uadde_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[COPY2]], [[COPY2]], [[UADDO1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDE1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UADDE]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32) = COPY $vgpr2
+ %3:_(s32), %4:_(s1) = G_UADDO %0, %1
+ %5:_(s32), %6:_(s1) = G_UADDE %2, %2, %4
+ %7:_(s32) = G_ZEXT %6(s1)
+ %8:_(s32) = G_ADD %5, %7
+ $vgpr0 = COPY %8(s32)
+...
+
+---
+name: sub_zext_usube_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-LABEL: name: sub_zext_usube_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[COPY2]], [[COPY2]], [[USUBO1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBE1]](s1)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[USUBE]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32) = COPY $vgpr2
+ %3:_(s32), %4:_(s1) = G_USUBO %0, %1
+ %5:_(s32), %6:_(s1) = G_USUBE %2, %2, %4
+ %7:_(s32) = G_ZEXT %6(s1)
+ %8:_(s32) = G_SUB %5, %7
+ $vgpr0 = COPY %8(s32)
+...
+
+---
+name: add_zext_usubo_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-LABEL: name: add_zext_usubo_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32) = COPY $vgpr2
+ %3:_(s32), %4:_(s1) = G_USUBO %0, %1
+ %5:_(s32) = G_ZEXT %4(s1)
+ %6:_(s32) = G_ADD %2, %5
+ $vgpr0 = COPY %6(s32)
+...
+
+---
+name: sub_zext_uaddo_carry
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-LABEL: name: sub_zext_uaddo_carry
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32) = COPY $vgpr2
+ %3:_(s32), %4:_(s1) = G_UADDO %0, %1
+ %5:_(s32) = G_ZEXT %4(s1)
+ %6:_(s32) = G_SUB %2, %5
+ $vgpr0 = COPY %6(s32)
+...
+
+---
+name: add_zext_carry_s64_not_folded
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_zext_carry_s64_not_folded
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[UADDO]](s32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[UADDO1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ADD]](s64)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+ %4:_(s64) = G_ZEXT %2(s32)
+ %5:_(s64) = G_ZEXT %3(s1)
+ %6:_(s64) = G_ADD %4, %5
+ $vgpr0_vgpr1 = COPY %6(s64)
+...
+
+---
+name: add_zext_multi_use_not_folded
+tracksRegLiveness: true
+legalized: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_zext_multi_use_not_folded
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UADDO]], [[ZEXT]]
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ADD]], [[ZEXT]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[MUL]](s32)
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+ %4:_(s32) = G_ZEXT %3(s1)
+ %5:_(s32) = G_ADD %2, %4
+ %6:_(s32) = G_MUL %5, %4
+ $vgpr0 = COPY %6(s32)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
new file mode 100644
index 000000000000000..771374648fe9748
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
@@ -0,0 +1,352 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+
+define amdgpu_ps void @s_uaddo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: s_uaddo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s1
+; GFX9-SDAG-NEXT: s_addc_u32 s0, s0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: s_uaddo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s1
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_add_i32 s0, s0, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: s_uaddo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_u32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT: s_add_co_ci_u32 s0, s0, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: s_uaddo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %sum = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = add i32 %sum, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_uaddo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_uaddo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_uaddo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_uaddo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_uaddo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %sum = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = add i32 %sum, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @s_usubo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: s_usubo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_sub_u32 s0, s0, s1
+; GFX9-SDAG-NEXT: s_subb_u32 s0, s0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: s_usubo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_sub_u32 s0, s0, s1
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: s_usubo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_sub_co_u32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT: s_sub_co_ci_u32 s0, s0, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: s_usubo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_sub_co_u32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
+ %diff = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = sub i32 %diff, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_usubo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_usubo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_usubo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_sub_u32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_usubo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_usubo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_sub_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX12-GISEL-NEXT: v_sub_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
+ %diff = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = sub i32 %diff, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_uaddo_carry_commuted(i32 %a, i32 %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_uaddo_carry_commuted:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_uaddo_carry_commuted:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_uaddo_carry_commuted:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_uaddo_carry_commuted:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v1, v0
+; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %sum = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = add i32 %ce, %sum
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_add_of_usubo_carry(i32 %a, i32 %b, i32 %x, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_add_of_usubo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-SDAG-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_add_of_usubo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_add_of_usubo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
+; GFX12-SDAG-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_add_of_usubo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_sub_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0
+; GFX12-GISEL-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = add i32 %x, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_sub_of_uaddo_carry(i32 %a, i32 %b, i32 %x, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_sub_of_uaddo_carry:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-SDAG-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_sub_of_uaddo_carry:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_sub_u32_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_sub_of_uaddo_carry:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
+; GFX12-SDAG-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_sub_of_uaddo_carry:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX12-GISEL-NEXT: v_sub_nc_u32_e32 v0, v2, v0
+; GFX12-GISEL-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = sub i32 %x, %ce
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; Negative test: the materialized 0/1 has another user.
+define amdgpu_ps void @v_uaddo_carry_multi_use(i32 %a, i32 %b, ptr addrspace(1) %out) {
+; GFX9-SDAG-LABEL: v_uaddo_carry_multi_use:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_uaddo_carry_multi_use:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_uaddo_carry_multi_use:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_uaddo_carry_multi_use:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %sum = extractvalue { i32, i1 } %o, 0
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %r = add i32 %sum, %ce
+ %r2 = mul i32 %r, %ce
+ store i32 %r2, ptr addrspace(1) %out
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
+; GFX9: {{.*}}
>From 5c342c910f72cd95b75ca49370eeccd5a3441a65 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Wed, 16 Sep 2026 19:31:10 +0200
Subject: [PATCH 2/4] [AMDGPU][GlobalISel] Fold zext(carry) into
add/sub-with-carry
---
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 24 +-
.../AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 34 +
llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll | 63 +-
...alizer-combiner-carry-into-uadde-usube.mir | 42 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 2098 ++++++++---------
.../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 625 +++--
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 1964 +++++++--------
llvm/test/CodeGen/AMDGPU/GlobalISel/subo.ll | 63 +-
.../CodeGen/AMDGPU/GlobalISel/udiv.i64.ll | 708 +++---
.../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 673 +++---
.../CodeGen/AMDGPU/GlobalISel/urem.i64.ll | 772 +++---
.../AMDGPU/combine-carry-into-uadde-usube.ll | 330 +--
12 files changed, 3275 insertions(+), 4121 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 2f0ff66668f2b62..37492ada79136af 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -284,6 +284,27 @@ def trunc_lshr16_to_unmerge : GICombineGroup<[
trunc_lshr16_unmerge_bf16
]>;
+// (add x, (zext carry:i1)) -> (uadde x, 0, carry)
+// (sub x, (zext carry:i1)) -> (usube x, 0, carry)
+class fold_carry_into_addsub<dag addsubPat, Instruction carryOpcode,
+ bit CheckRHS = 0> : GICombineRule<
+ (defs root:$dst),
+ (match (G_ZEXT i32:$zext, i1:$carry),
+ addsubPat,
+ !strconcat(
+ [{ return MRI.hasOneNonDBGUse(${zext}.getReg()) &&
+ isCarryOut(${carry}.getReg()) }],
+ !if(CheckRHS, [{ && shouldFoldCarryIntoAdd(${src}.getReg()) }], ""),
+ [{;}])),
+ (apply (carryOpcode $dst, GITypeOf<"$carry">:$carry_out,
+ $src, (GITypeOf<"$dst"> 0), $carry))>;
+
+def fold_carry_into_addsub_combines : GICombineGroup<[
+ fold_carry_into_addsub<(G_ADD i32:$dst, i32:$src, $zext), G_UADDE>,
+ fold_carry_into_addsub<(G_ADD i32:$dst, $zext, i32:$src), G_UADDE, CheckRHS = 1>,
+ fold_carry_into_addsub<(G_SUB i32:$dst, i32:$src, $zext), G_USUBE>
+]>;
+
let Predicates = [Has16BitInsts, NotHasMed3_16] in {
// For gfx8, expand f16-fmed3-as-f32 into a min/max f16 sequence. This
// saves one instruction compared to the promotion.
@@ -325,7 +346,8 @@ def AMDGPUPostLegalizerCombiner: GICombiner<
uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize, foldable_fneg,
fold_fabs_fptrunc, rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16,
sign_extension_in_reg, smulu64, trunc_lshr16_to_unmerge,
- binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
+ binop_s64_with_s32_mask_combines, combine_or_s64_s32,
+ fold_carry_into_addsub_combines]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 558258868642bfb..b18ef441bf02583 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -111,6 +111,11 @@ class AMDGPUPostLegalizerCombinerImpl : public Combiner {
// bits are zero extended.
bool matchCombine_s_mul_u64(MachineInstr &MI, unsigned &NewOpcode) const;
+ /// Check if register is carry-out (operand 1) of a carry-producing add/sub
+ bool isCarryOut(Register Reg) const;
+
+ bool shouldFoldCarryIntoAdd(Register Src) const;
+
private:
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
@@ -393,6 +398,35 @@ bool AMDGPUPostLegalizerCombinerImpl::matchCombine_s_mul_u64(
return false;
}
+bool AMDGPUPostLegalizerCombinerImpl::isCarryOut(Register Reg) const {
+ const MachineInstr *Def = MRI.getVRegDef(Reg);
+ if (!Def)
+ return false;
+
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_UADDO:
+ case TargetOpcode::G_UADDE:
+ case TargetOpcode::G_USUBO:
+ case TargetOpcode::G_USUBE:
+ return Def->getOperand(1).getReg() == Reg;
+ default:
+ return false;
+ }
+}
+
+bool AMDGPUPostLegalizerCombinerImpl::shouldFoldCarryIntoAdd(
+ Register Src) const {
+ // Let right_identity_zero combine delete this instead.
+ if (mi_match(Src, MRI, m_SpecificICst(0)))
+ return false;
+
+ // When both operands are a carry only the right hand one is folded, so that
+ // the two carries never have to be live at the same time.
+ Register Carry;
+ return !(mi_match(Src, MRI, m_GZExt(m_Reg(Carry))) &&
+ MRI.hasOneNonDBGUse(Src) && isCarryOut(Carry));
+}
+
// Pass boilerplate
// ================
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll
index 74d075ccc5099d5..a0de97ecce95c23 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll
@@ -8,24 +8,21 @@ define i32 @v_uaddo_i32(i32 %a, i32 %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_uaddo_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddo_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
%uaddo = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%add = extractvalue {i32, i1} %uaddo, 0
@@ -163,33 +160,27 @@ define <2 x i32> @v_uaddo_v2i32(<2 x i32> %a, <2 x i32> %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_uaddo_v2i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddo_v2i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
%uaddo = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b)
%add = extractvalue {<2 x i32>, <2 x i1>} %uaddo, 0
@@ -448,22 +439,19 @@ define amdgpu_ps i32 @s_uaddo_i32(i32 inreg %a, i32 inreg %b) {
; GFX7-LABEL: s_uaddo_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_add_u32 s0, s0, s1
-; GFX7-NEXT: s_cselect_b32 s1, 1, 0
-; GFX7-NEXT: s_add_i32 s0, s0, s1
+; GFX7-NEXT: s_addc_u32 s0, s0, 0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_uaddo_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_add_u32 s0, s0, s1
-; GFX8-NEXT: s_cselect_b32 s1, 1, 0
-; GFX8-NEXT: s_add_i32 s0, s0, s1
+; GFX8-NEXT: s_addc_u32 s0, s0, 0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddo_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_add_u32 s0, s0, s1
-; GFX9-NEXT: s_cselect_b32 s1, 1, 0
-; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_addc_u32 s0, s0, 0
; GFX9-NEXT: ; return to shader part epilog
%uaddo = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%add = extractvalue {i32, i1} %uaddo, 0
@@ -516,11 +504,9 @@ define amdgpu_ps <2 x i32> @s_uaddo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX7-NEXT: s_add_u32 s1, s1, s3
; GFX7-NEXT: s_cselect_b32 s3, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s2, 0
-; GFX7-NEXT: s_cselect_b32 s2, 1, 0
+; GFX7-NEXT: s_addc_u32 s0, s0, 0
; GFX7-NEXT: s_cmp_lg_u32 s3, 0
-; GFX7-NEXT: s_cselect_b32 s3, 1, 0
-; GFX7-NEXT: s_add_i32 s0, s0, s2
-; GFX7-NEXT: s_add_i32 s1, s1, s3
+; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_uaddo_v2i32:
@@ -530,11 +516,9 @@ define amdgpu_ps <2 x i32> @s_uaddo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX8-NEXT: s_add_u32 s1, s1, s3
; GFX8-NEXT: s_cselect_b32 s3, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s2, 0
-; GFX8-NEXT: s_cselect_b32 s2, 1, 0
+; GFX8-NEXT: s_addc_u32 s0, s0, 0
; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_add_i32 s0, s0, s2
-; GFX8-NEXT: s_add_i32 s1, s1, s3
+; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddo_v2i32:
@@ -544,11 +528,9 @@ define amdgpu_ps <2 x i32> @s_uaddo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX9-NEXT: s_add_u32 s1, s1, s3
; GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s2, 0
-; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_addc_u32 s0, s0, 0
; GFX9-NEXT: s_cmp_lg_u32 s3, 0
-; GFX9-NEXT: s_cselect_b32 s3, 1, 0
-; GFX9-NEXT: s_add_i32 s0, s0, s2
-; GFX9-NEXT: s_add_i32 s1, s1, s3
+; GFX9-NEXT: s_addc_u32 s1, s1, 0
; GFX9-NEXT: ; return to shader part epilog
%uaddo = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b)
%add = extractvalue {<2 x i32>, <2 x i1>} %uaddo, 0
@@ -931,24 +913,21 @@ define amdgpu_ps i32 @uaddo_i32_sv(i32 inreg %a, i32 %b) {
; GFX7-LABEL: uaddo_i32_sv:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddo_i32_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: uaddo_i32_sv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
%uaddo = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
index 6649f61fcf5d0ee..4b30c9f5cc3c7a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
@@ -14,9 +14,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UADDO]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UADDO]], [[C]], [[UADDO1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[UADDE]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32), %3:_(s1) = G_UADDO %0, %1
@@ -38,9 +38,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[UADDO]]
- ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UADDO]], [[C]], [[UADDO1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[UADDE]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32), %3:_(s1) = G_UADDO %0, %1
@@ -62,9 +62,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[USUBO]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[USUBO]], [[C]], [[USUBO1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[USUBE]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32), %3:_(s1) = G_USUBO %0, %1
@@ -112,9 +112,9 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[COPY2]], [[COPY2]], [[UADDO1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDE1]](s1)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UADDE]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UADDE]], [[C]], [[UADDE1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[UADDE2]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
@@ -140,9 +140,9 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[COPY2]], [[COPY2]], [[USUBO1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBE1]](s1)
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[USUBE]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[USUBE]], [[C]], [[USUBE1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[USUBE2]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
@@ -167,9 +167,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[COPY2]], [[C]], [[USUBO1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[UADDE]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
@@ -193,9 +193,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[ZEXT]]
- ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[COPY2]], [[C]], [[UADDO1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[USUBE]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 62a9c6daf0a02a1..8a6961958d0050c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -53,17 +53,14 @@ define i64 @v_sdiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v15
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; CHECK-NEXT: v_mul_hi_u32 v9, v12, v10
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
; CHECK-NEXT: v_addc_u32_e32 v12, vcc, v12, v7, vcc
@@ -79,21 +76,18 @@ define i64 @v_sdiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
; CHECK-NEXT: v_mul_hi_u32 v6, v12, v6
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_mul_hi_u32 v11, v3, v10
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v7, v3, v10
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v8, v4
-; CHECK-NEXT: v_mul_lo_u32 v8, v12, v10
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_mul_lo_u32 v7, v12, v10
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v8, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_mul_hi_u32 v8, v12, v10
; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v12, v6, vcc
@@ -103,23 +97,20 @@ define i64 @v_sdiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v5, v10, v3
; CHECK-NEXT: v_mul_hi_u32 v3, v10, v3
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v8, v9, v4
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v6, v9, v4
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; CHECK-NEXT: v_mul_lo_u32 v7, v10, v4
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; CHECK-NEXT: v_mul_lo_u32 v6, v10, v4
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v8
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v5
-; CHECK-NEXT: v_mul_hi_u32 v7, v10, v4
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v10, v4
; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v11, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v7, v5
+; CHECK-NEXT: v_add_i32_e32 v12, vcc, v6, v5
; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v2, v12, v[4:5]
; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v9, v3
; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v11, v[5:6]
@@ -232,34 +223,31 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_mul_i32 s16, s5, s13
; CHECK-NEXT: s_add_u32 s14, s16, s14
; CHECK-NEXT: v_readfirstlane_b32 s16, v2
-; CHECK-NEXT: s_mul_i32 s17, s13, s14
; CHECK-NEXT: v_mul_hi_u32 v2, v1, s15
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s14
+; CHECK-NEXT: s_mul_i32 s17, s13, s14
; CHECK-NEXT: s_mul_i32 s18, s12, s15
; CHECK-NEXT: s_add_u32 s16, s16, s17
; CHECK-NEXT: s_cselect_b32 s17, 1, 0
-; CHECK-NEXT: s_add_u32 s16, s16, s18
-; CHECK-NEXT: s_cselect_b32 s15, 1, 0
-; CHECK-NEXT: s_add_i32 s17, s17, s15
-; CHECK-NEXT: v_readfirstlane_b32 s15, v2
-; CHECK-NEXT: v_readfirstlane_b32 s16, v0
+; CHECK-NEXT: s_add_u32 s15, s16, s18
+; CHECK-NEXT: s_addc_u32 s15, s17, 0
+; CHECK-NEXT: v_readfirstlane_b32 s16, v2
+; CHECK-NEXT: v_readfirstlane_b32 s17, v0
; CHECK-NEXT: s_mul_i32 s18, s12, s14
-; CHECK-NEXT: s_add_u32 s15, s15, s16
-; CHECK-NEXT: s_cselect_b32 s16, 1, 0
-; CHECK-NEXT: s_add_u32 s15, s15, s18
-; CHECK-NEXT: s_cselect_b32 s18, 1, 0
+; CHECK-NEXT: s_add_u32 s16, s16, s17
; CHECK-NEXT: v_mul_hi_u32 v0, v1, s14
-; CHECK-NEXT: s_add_i32 s16, s16, s18
-; CHECK-NEXT: s_add_u32 s14, s15, s17
-; CHECK-NEXT: s_cselect_b32 s15, 1, 0
-; CHECK-NEXT: s_add_i32 s16, s16, s15
-; CHECK-NEXT: v_readfirstlane_b32 s15, v0
-; CHECK-NEXT: s_add_i32 s15, s15, s16
-; CHECK-NEXT: s_add_u32 s13, s13, s14
+; CHECK-NEXT: s_cselect_b32 s17, 1, 0
+; CHECK-NEXT: s_add_u32 s16, s16, s18
+; CHECK-NEXT: s_addc_u32 s14, s17, 0
+; CHECK-NEXT: s_add_u32 s15, s16, s15
+; CHECK-NEXT: s_addc_u32 s14, s14, 0
+; CHECK-NEXT: v_readfirstlane_b32 s16, v0
+; CHECK-NEXT: s_add_i32 s16, s16, s14
+; CHECK-NEXT: s_add_u32 s13, s13, s15
; CHECK-NEXT: v_mov_b32_e32 v0, s13
; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
; CHECK-NEXT: s_mul_i32 s14, s3, s13
-; CHECK-NEXT: s_addc_u32 s12, s12, s15
+; CHECK-NEXT: s_addc_u32 s12, s12, s16
; CHECK-NEXT: s_mul_i32 s3, s3, s12
; CHECK-NEXT: v_readfirstlane_b32 s15, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s14
@@ -269,27 +257,24 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s3, s5, s3
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: s_mul_i32 s15, s13, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s12, v0
; CHECK-NEXT: v_mul_hi_u32 v2, s13, v1
+; CHECK-NEXT: s_mul_i32 s15, s13, s3
; CHECK-NEXT: s_mul_i32 s14, s12, s14
; CHECK-NEXT: s_add_u32 s5, s5, s15
; CHECK-NEXT: s_cselect_b32 s15, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s14
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s15, s15, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: s_addc_u32 s5, s15, 0
+; CHECK-NEXT: v_readfirstlane_b32 s14, v0
+; CHECK-NEXT: v_readfirstlane_b32 s15, v2
; CHECK-NEXT: s_mul_i32 s3, s12, s3
-; CHECK-NEXT: s_add_u32 s5, s5, s14
-; CHECK-NEXT: s_cselect_b32 s14, 1, 0
-; CHECK-NEXT: s_add_u32 s3, s5, s3
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_u32 s14, s14, s15
; CHECK-NEXT: v_mul_hi_u32 v0, s12, v1
-; CHECK-NEXT: s_add_i32 s5, s14, s5
-; CHECK-NEXT: s_add_u32 s3, s3, s15
-; CHECK-NEXT: s_cselect_b32 s14, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s14
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s14, s3
+; CHECK-NEXT: s_addc_u32 s14, s15, 0
+; CHECK-NEXT: s_add_u32 s3, s3, s5
+; CHECK-NEXT: s_addc_u32 s5, s14, 0
; CHECK-NEXT: v_readfirstlane_b32 s14, v0
; CHECK-NEXT: s_add_i32 s14, s14, s5
; CHECK-NEXT: s_add_u32 s3, s13, s3
@@ -297,33 +282,30 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
; CHECK-NEXT: s_addc_u32 s5, s12, s14
; CHECK-NEXT: v_mov_b32_e32 v1, s5
-; CHECK-NEXT: s_mul_i32 s13, s10, s5
+; CHECK-NEXT: v_mul_hi_u32 v2, s10, v1
; CHECK-NEXT: v_readfirstlane_b32 s12, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s11
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
-; CHECK-NEXT: v_mul_hi_u32 v2, s10, v1
+; CHECK-NEXT: s_mul_i32 s13, s10, s5
; CHECK-NEXT: s_mul_i32 s14, s11, s3
; CHECK-NEXT: s_add_u32 s12, s12, s13
; CHECK-NEXT: s_cselect_b32 s13, 1, 0
-; CHECK-NEXT: s_add_u32 s12, s12, s14
-; CHECK-NEXT: s_cselect_b32 s3, 1, 0
-; CHECK-NEXT: s_add_i32 s13, s13, s3
-; CHECK-NEXT: v_readfirstlane_b32 s3, v0
-; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: s_add_u32 s3, s12, s14
+; CHECK-NEXT: s_addc_u32 s3, s13, 0
+; CHECK-NEXT: v_readfirstlane_b32 s12, v0
+; CHECK-NEXT: v_readfirstlane_b32 s13, v2
; CHECK-NEXT: s_mul_i32 s5, s11, s5
-; CHECK-NEXT: s_add_u32 s3, s3, s12
-; CHECK-NEXT: s_cselect_b32 s12, 1, 0
-; CHECK-NEXT: s_add_u32 s3, s3, s5
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s12, s5
-; CHECK-NEXT: s_add_u32 s12, s3, s13
+; CHECK-NEXT: s_add_u32 s12, s12, s13
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s12, s5
+; CHECK-NEXT: s_addc_u32 s13, s13, 0
+; CHECK-NEXT: s_add_u32 s12, s5, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s11, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s12
; CHECK-NEXT: v_mul_hi_u32 v1, s8, v1
-; CHECK-NEXT: s_cselect_b32 s3, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s3
-; CHECK-NEXT: v_readfirstlane_b32 s3, v0
-; CHECK-NEXT: s_add_i32 s13, s3, s5
+; CHECK-NEXT: s_addc_u32 s3, s13, 0
+; CHECK-NEXT: v_readfirstlane_b32 s5, v0
+; CHECK-NEXT: s_add_i32 s13, s5, s3
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: s_mul_i32 s14, s8, s13
; CHECK-NEXT: s_add_u32 s5, s14, s5
@@ -442,243 +424,225 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v9, v[12:13]
; GISEL-NEXT: v_mul_hi_u32 v12, v5, v11
; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v5, v[13:14]
-; GISEL-NEXT: v_mul_lo_u32 v13, v5, v15
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_mul_lo_u32 v13, v9, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v14, v5, v15
+; GISEL-NEXT: v_mul_hi_u32 v16, v5, v15
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v5, v15
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v14, v12
-; GISEL-NEXT: v_mul_lo_u32 v14, v9, v15
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
+; GISEL-NEXT: v_mul_lo_u32 v13, v9, v15
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v14, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v16
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_mul_hi_u32 v14, v9, v15
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v14, v12
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
-; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v9, v12, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v5, 0
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v19, v[12:13]
-; GISEL-NEXT: v_mul_hi_u32 v12, v5, v11
-; GISEL-NEXT: v_xor_b32_e32 v17, v0, v9
-; GISEL-NEXT: v_mul_lo_u32 v0, v19, v11
-; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v5, v[13:14]
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v19, v11
-; GISEL-NEXT: v_mul_lo_u32 v13, v5, v15
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GISEL-NEXT: v_add_i32_e32 v19, vcc, v5, v11
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v12, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v19, 0
+; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v9, v[12:13]
+; GISEL-NEXT: v_mul_hi_u32 v12, v19, v11
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v19, v[13:14]
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v5
+; GISEL-NEXT: v_mul_lo_u32 v0, v9, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v15
+; GISEL-NEXT: v_mul_hi_u32 v14, v19, v15
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v5, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
-; GISEL-NEXT: v_mul_lo_u32 v13, v19, v15
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_mul_lo_u32 v12, v9, v15
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v13, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; GISEL-NEXT: v_mul_hi_u32 v13, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
+; GISEL-NEXT: v_mul_hi_u32 v13, v9, v15
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v5, v0
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v19, v11, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v17, v5
-; GISEL-NEXT: v_mul_lo_u32 v13, v1, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v1, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v11, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v9
+; GISEL-NEXT: v_xor_b32_e32 v15, v1, v5
+; GISEL-NEXT: v_mul_lo_u32 v1, v15, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_mul_hi_u32 v13, v16, v9
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v17, v5
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT: v_mul_lo_u32 v12, v1, v5
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v9
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v12
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; GISEL-NEXT: v_mul_hi_u32 v5, v1, v5
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v0, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v5, v13
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v10, v18, v[12:13]
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v17, v11
-; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v7
-; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v4, v0, v[13:14]
-; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v1, v15, vcc
-; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v15
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v5
-; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v7, v5, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v7, v6, v5
-; GISEL-NEXT: v_xor_b32_e32 v6, v12, v5
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v13, v6
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v17, 0
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v9, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v18, v[1:2]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
+; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v7
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v4, v17, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], v15, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v15, v13
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v9
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v7, v9, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v7, v6, v9
+; GISEL-NEXT: v_xor_b32_e32 v6, v11, v9
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v6
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v10
-; GISEL-NEXT: v_mac_f32_e32 v12, 0x4f800000, v13
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v12, v12
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v11, v10
-; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v10
-; GISEL-NEXT: v_mul_f32_e32 v10, 0x5f7ffffc, v12
-; GISEL-NEXT: v_mul_f32_e32 v11, 0x2f800000, v10
-; GISEL-NEXT: v_trunc_f32_e32 v11, v11
-; GISEL-NEXT: v_mac_f32_e32 v10, 0xcf800000, v11
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v10
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v10
+; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
+; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GISEL-NEXT: v_trunc_f32_e32 v1, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v0
; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v7
-; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v11
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v6, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v17, 0
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v1, v4
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v20, v19, v[11:12]
-; GISEL-NEXT: v_mul_hi_u32 v11, v17, v10
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[6:7], v21, v17, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v4
-; GISEL-NEXT: v_mul_lo_u32 v12, v17, v14
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[6:7]
-; GISEL-NEXT: v_mul_hi_u32 v10, v19, v10
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v18, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v15, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 1, v11
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v12, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v11, v15, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v16, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v15, v4
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v19, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v16, v0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v21, v16, v[10:11]
+; GISEL-NEXT: v_mul_lo_u32 v10, v16, v12
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_mul_lo_u32 v10, v19, v0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_mul_hi_u32 v13, v17, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_mul_lo_u32 v13, v19, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v14, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v11
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v17, v10
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v11, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v13, 0
+; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v15, v4
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v14, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v13, v15, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v18, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, 1, v10
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v13, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v18, v12, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v0, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v14, v[11:12]
-; GISEL-NEXT: v_xor_b32_e32 v15, v9, v8
-; GISEL-NEXT: v_xor_b32_e32 v11, v12, v15
-; GISEL-NEXT: v_ashrrev_i32_e32 v12, 31, v3
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v13, v[0:1]
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v12
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v3, v12, vcc
-; GISEL-NEXT: v_mul_hi_u32 v2, v13, v10
-; GISEL-NEXT: v_mul_lo_u32 v3, v13, v8
-; GISEL-NEXT: v_xor_b32_e32 v16, v0, v12
-; GISEL-NEXT: v_mul_lo_u32 v0, v14, v10
-; GISEL-NEXT: v_mul_hi_u32 v9, v13, v8
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v2, v14, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0
-; GISEL-NEXT: v_mul_lo_u32 v3, v14, v8
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_mul_hi_u32 v1, v16, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v11, s[6:7]
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_mul_lo_u32 v1, v19, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v1, v19, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v14, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v0
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v12, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v17, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v14, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v5, v8
+; GISEL-NEXT: v_xor_b32_e32 v8, v4, v1
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v12, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v5, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v16, v2, v15
+; GISEL-NEXT: v_mul_lo_u32 v2, v14, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v12, v4
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v12, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_mul_lo_u32 v5, v14, v4
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_mul_hi_u32 v4, v14, v4
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v14, v2, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v16, v0
-; GISEL-NEXT: v_mul_lo_u32 v8, v16, v2
-; GISEL-NEXT: v_xor_b32_e32 v10, v1, v12
-; GISEL-NEXT: v_mul_lo_u32 v1, v10, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v10, v0
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v3, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v16, v2
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT: v_mul_lo_u32 v8, v10, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v3, v8
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v1
-; GISEL-NEXT: v_mul_hi_u32 v1, v10, v2
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v13, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_xor_b32_e32 v4, v4, v15
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v1, v0
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v7, v14, v[3:4]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v4, v15
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v15, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v13, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v4, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v5, v16, v2
+; GISEL-NEXT: v_xor_b32_e32 v12, v3, v15
+; GISEL-NEXT: v_mul_lo_u32 v3, v12, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v12, v0
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_mul_hi_u32 v10, v16, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GISEL-NEXT: v_mul_lo_u32 v4, v12, v2
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v0, v3
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v12, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v14, 0
+; GISEL-NEXT: v_xor_b32_e32 v10, v13, v1
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v4, v0
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v13, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v8, v1
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v10, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v14, v[4:5]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v16, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v10, v3
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v6
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v12, v10, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v10
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v6
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v7
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v6
-; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v8, v9, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v13
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v14, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v6
+; GISEL-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, v8, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v14
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v13, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v7
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v8
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v5
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v8, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v13, v2, vcc
-; GISEL-NEXT: v_xor_b32_e32 v4, v12, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v14, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v14, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v4, v15, v9
; GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
; GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
@@ -730,16 +694,13 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v13, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v18
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v19
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v12, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v3
; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v4, vcc
@@ -755,21 +716,18 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v5, v14, v3
; CGP-NEXT: v_mul_hi_u32 v3, v14, v3
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v17, v15, v4
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v11, v15, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v12, v5
-; CGP-NEXT: v_mul_lo_u32 v12, v14, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
+; CGP-NEXT: v_mul_lo_u32 v11, v14, v4
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v12, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v17
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v11
; CGP-NEXT: v_mul_hi_u32 v4, v14, v4
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_add_i32_e32 v3, vcc, v15, v3
; CGP-NEXT: v_addc_u32_e32 v4, vcc, v14, v4, vcc
@@ -779,23 +737,20 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v10, v12, v3
; CGP-NEXT: v_mul_hi_u32 v3, v12, v3
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; CGP-NEXT: v_mul_hi_u32 v14, v13, v4
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v10, v13, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
-; CGP-NEXT: v_mul_lo_u32 v11, v12, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v11
+; CGP-NEXT: v_mul_lo_u32 v10, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v11, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v14
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v3, v5
-; CGP-NEXT: v_mul_hi_u32 v11, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v12, v4
; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v14, 0
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v11, v5
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v10, v5
; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v2, v15, v[4:5]
; CGP-NEXT: v_sub_i32_e32 v3, vcc, v13, v3
; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v14, v[10:11]
@@ -903,16 +858,13 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v10
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v16
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v17
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v5
; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v6, vcc
@@ -928,21 +880,18 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v7, v12, v5
; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v15, v13, v6
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v9, v13, v6
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v7
-; CGP-NEXT: v_mul_lo_u32 v10, v12, v6
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_mul_lo_u32 v9, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v15
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
; CGP-NEXT: v_mul_hi_u32 v6, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CGP-NEXT: v_add_i32_e32 v5, vcc, v13, v5
; CGP-NEXT: v_addc_u32_e32 v6, vcc, v12, v6, vcc
@@ -952,23 +901,20 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v8, v12, v5
; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT: v_mul_hi_u32 v10, v11, v6
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v11, v6
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; CGP-NEXT: v_mul_lo_u32 v9, v12, v6
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_mul_lo_u32 v8, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
-; CGP-NEXT: v_mul_hi_u32 v9, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v12, v6
; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v4, v13, 0
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v9, v7
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v8, v7
; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v15, v[6:7]
; CGP-NEXT: v_sub_i32_e32 v5, vcc, v11, v5
; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v13, v[7:8]
@@ -1089,24 +1035,24 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
; CHECK-NEXT: v_mov_b32_e32 v4, 0x4f800000
-; CHECK-NEXT: v_mov_b32_e32 v5, 0xffed2705
; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_mac_f32_e32 v3, 0, v4
; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; CHECK-NEXT: v_trunc_f32_e32 v4, v4
; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
-; CHECK-NEXT: v_mul_hi_u32 v6, v5, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
+; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
; CHECK-NEXT: v_readfirstlane_b32 s4, v3
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s5, v4
+; CHECK-NEXT: v_readfirstlane_b32 s5, v0
; CHECK-NEXT: v_readfirstlane_b32 s7, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
@@ -1114,115 +1060,106 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: v_readfirstlane_b32 s8, v6
-; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v6, v4, s6
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: s_mul_i32 s10, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_u32 s8, s8, s10
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v6
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_add_u32 s6, s8, s10
+; CHECK-NEXT: s_addc_u32 s6, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v6
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s10, s5, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s10
-; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v3, v4, s7
-; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: s_add_u32 s6, s6, s9
-; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s10
+; CHECK-NEXT: s_addc_u32 s7, s9, 0
+; CHECK-NEXT: s_add_u32 s6, s8, s6
+; CHECK-NEXT: s_addc_u32 s7, s7, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s7, v3
-; CHECK-NEXT: s_add_i32 s7, s7, s8
; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: v_mul_hi_u32 v3, v5, s4
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: s_addc_u32 s5, s5, s7
+; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
+; CHECK-NEXT: s_addc_u32 s5, s5, s8
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s7, v3
-; CHECK-NEXT: v_mov_b32_e32 v3, s6
-; CHECK-NEXT: v_mul_hi_u32 v4, s4, v3
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s8, v4
-; CHECK-NEXT: v_mov_b32_e32 v4, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
+; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v3, s5, v3
-; CHECK-NEXT: v_mul_hi_u32 v5, s4, v4
; CHECK-NEXT: s_mul_i32 s6, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s6, s8, s6
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v3
-; CHECK-NEXT: v_readfirstlane_b32 s8, v5
+; CHECK-NEXT: s_addc_u32 s6, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s7, s5, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s7
-; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v3, s5, v4
-; CHECK-NEXT: s_add_i32 s7, s8, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s9
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_i32 s7, s7, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_addc_u32 s8, s9, 0
+; CHECK-NEXT: s_add_u32 s6, s7, s6
+; CHECK-NEXT: s_addc_u32 s7, s8, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s7
; CHECK-NEXT: s_add_u32 s4, s4, s6
; CHECK-NEXT: s_addc_u32 s5, s5, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v0, s4
-; CHECK-NEXT: v_mul_lo_u32 v4, v0, s5
-; CHECK-NEXT: v_mul_lo_u32 v5, v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v6, v0, s5
-; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
+; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
+; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v4, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v8, v5, s5
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v8
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v5, v1, s4
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_mul_lo_u32 v4, v1, s5
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v4, v3
-; CHECK-NEXT: v_mul_hi_u32 v9, v1, s5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v8, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v5
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v9, v[4:5]
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v1, v5, vcc
-; CHECK-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v5
-; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v1, v0
+; CHECK-NEXT: v_mul_hi_u32 v4, v6, s5
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v8, 0
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v4, v3
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v9, v[1:2]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
+; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
+; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v3, -1, v4, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v1, -1, v4, s[4:5]
; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v8
; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
; CHECK-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, 1, v4
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v4
; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; CHECK-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1249,164 +1186,151 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_trunc_f32_e32 v6, v6
; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_xor_b32_e32 v14, v0, v4
; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v7
-; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
-; GISEL-NEXT: v_readfirstlane_b32 s6, v5
-; GISEL-NEXT: v_readfirstlane_b32 s9, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, v7
+; GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; GISEL-NEXT: s_mul_i32 s5, s6, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s7, v5
+; GISEL-NEXT: v_readfirstlane_b32 s9, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v7, s5
; GISEL-NEXT: s_subb_u32 s4, 0, 0
-; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
+; GISEL-NEXT: s_mul_i32 s10, s7, 0xffed2705
; GISEL-NEXT: s_add_u32 s9, s10, s9
-; GISEL-NEXT: s_mul_i32 s10, s4, s7
+; GISEL-NEXT: s_mul_i32 s10, s4, s6
; GISEL-NEXT: s_add_u32 s10, s10, s9
-; GISEL-NEXT: v_readfirstlane_b32 s11, v8
-; GISEL-NEXT: s_mul_i32 s12, s7, s10
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
-; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
-; GISEL-NEXT: s_mul_i32 s13, s6, s5
+; GISEL-NEXT: v_readfirstlane_b32 s11, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v8, v7, s10
+; GISEL-NEXT: s_mul_i32 s12, s6, s10
+; GISEL-NEXT: s_mul_i32 s13, s7, s5
; GISEL-NEXT: s_add_u32 s12, s11, s12
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: s_add_u32 s12, s12, s13
-; GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GISEL-NEXT: s_add_i32 s14, s14, s5
-; GISEL-NEXT: v_readfirstlane_b32 s12, v8
-; GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GISEL-NEXT: s_mul_i32 s15, s6, s10
-; GISEL-NEXT: s_add_u32 s5, s12, s5
+; GISEL-NEXT: s_add_u32 s5, s12, s13
+; GISEL-NEXT: s_addc_u32 s5, s14, 0
+; GISEL-NEXT: v_readfirstlane_b32 s12, v6
+; GISEL-NEXT: v_readfirstlane_b32 s14, v8
+; GISEL-NEXT: s_mul_i32 s15, s7, s10
+; GISEL-NEXT: s_add_u32 s14, s12, s14
+; GISEL-NEXT: v_mul_hi_u32 v6, v5, s10
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
-; GISEL-NEXT: s_add_u32 s5, s5, s15
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
-; GISEL-NEXT: s_add_i32 s15, s16, s15
-; GISEL-NEXT: s_add_u32 s5, s5, s14
-; GISEL-NEXT: s_cselect_b32 s10, 1, 0
-; GISEL-NEXT: s_add_i32 s15, s15, s10
-; GISEL-NEXT: v_readfirstlane_b32 s10, v8
-; GISEL-NEXT: s_add_i32 s10, s10, s15
-; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: s_add_u32 s14, s14, s15
+; GISEL-NEXT: s_addc_u32 s10, s16, 0
+; GISEL-NEXT: s_add_u32 s5, s14, s5
+; GISEL-NEXT: s_addc_u32 s10, s10, 0
+; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: s_add_i32 s14, s14, s10
+; GISEL-NEXT: s_add_u32 s5, s6, s5
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, s5
+; GISEL-NEXT: s_addc_u32 s10, s7, s14
; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
-; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
+; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
; GISEL-NEXT: v_mov_b32_e32 v6, s14
-; GISEL-NEXT: s_addc_u32 s10, s6, s10
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
-; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
; GISEL-NEXT: s_add_u32 s15, s16, s15
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s15
; GISEL-NEXT: v_readfirstlane_b32 s15, v8
; GISEL-NEXT: v_mov_b32_e32 v8, s4
-; GISEL-NEXT: s_mul_i32 s16, s5, s4
; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: s_mul_i32 s16, s5, s4
; GISEL-NEXT: s_mul_i32 s14, s10, s14
; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s14, s15, s14
-; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: s_add_i32 s16, s16, s14
-; GISEL-NEXT: v_readfirstlane_b32 s14, v6
-; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: s_addc_u32 s14, s16, 0
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: v_readfirstlane_b32 s16, v9
; GISEL-NEXT: s_mul_i32 s4, s10, s4
-; GISEL-NEXT: s_add_u32 s14, s14, s15
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: s_add_u32 s4, s14, s4
-; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
-; GISEL-NEXT: s_add_i32 s14, s15, s14
-; GISEL-NEXT: s_add_u32 s4, s4, s16
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: s_add_i32 s14, s14, s15
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s15, s4
+; GISEL-NEXT: s_addc_u32 s15, s16, 0
+; GISEL-NEXT: s_add_u32 s4, s4, s14
+; GISEL-NEXT: s_addc_u32 s14, s15, 0
; GISEL-NEXT: v_readfirstlane_b32 s15, v6
; GISEL-NEXT: s_add_i32 s15, s15, s14
; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
; GISEL-NEXT: s_addc_u32 s5, s10, s15
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
-; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
-; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, s4
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
+; GISEL-NEXT: v_mul_lo_u32 v10, v1, s4
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s4
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
+; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
-; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v11, v10
; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v16, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v14, v8
; GISEL-NEXT: s_cmp_lg_u32 s8, 0
; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
-; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v1, v12, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v12
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v6
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, -1, v10, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, -1, v10, s[4:5]
; GISEL-NEXT: s_subb_u32 s4, 0, 0
-; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: s_mul_i32 s5, s4, s6
; GISEL-NEXT: s_add_u32 s5, s5, s9
-; GISEL-NEXT: s_mul_i32 s8, s7, s5
; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: s_mul_i32 s8, s6, s5
; GISEL-NEXT: s_add_u32 s8, s11, s8
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s13
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_i32 s9, s9, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: s_mul_i32 s10, s6, s5
-; GISEL-NEXT: s_add_u32 s8, s12, s8
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v6
-; GISEL-NEXT: s_cselect_b32 s11, 1, 0
-; GISEL-NEXT: s_add_u32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: s_addc_u32 s8, s9, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v7
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v6
+; GISEL-NEXT: s_mul_i32 s10, s7, s5
+; GISEL-NEXT: s_add_u32 s9, s12, s9
; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: s_cselect_b32 s11, 1, 0
+; GISEL-NEXT: s_add_u32 s9, s9, s10
; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v15
-; GISEL-NEXT: s_add_i32 s10, s11, s10
+; GISEL-NEXT: s_addc_u32 s5, s11, 0
; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v16, vcc
-; GISEL-NEXT: s_add_u32 s5, s8, s9
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT: s_add_i32 s10, s10, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v10
-; GISEL-NEXT: s_add_i32 s8, s8, s10
+; GISEL-NEXT: s_add_u32 s8, s9, s8
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v6
+; GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT: s_add_i32 s9, s9, s5
+; GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v10
+; GISEL-NEXT: s_add_u32 s5, s6, s8
; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v11, vcc
-; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v11, v12, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v16, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v11, v12, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v16, v8, vcc
; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT: s_addc_u32 s6, s7, s9
; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
; GISEL-NEXT: v_cndmask_b32_e32 v1, v15, v1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; GISEL-NEXT: s_addc_u32 s6, s6, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: v_mov_b32_e32 v0, s7
; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
@@ -1417,27 +1341,24 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: s_add_u32 s4, s4, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GISEL-NEXT: s_mul_i32 s9, s5, s4
; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: s_mul_i32 s9, s5, s4
; GISEL-NEXT: s_mul_i32 s7, s6, s7
; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s7, s8, s7
-; GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GISEL-NEXT: s_add_i32 s9, s9, s7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v0
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: s_addc_u32 s7, s9, 0
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v7
; GISEL-NEXT: s_mul_i32 s4, s6, s4
-; GISEL-NEXT: s_add_u32 s7, s7, s8
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_u32 s4, s7, s4
-; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
-; GISEL-NEXT: s_add_i32 s7, s8, s7
-; GISEL-NEXT: s_add_u32 s4, s4, s9
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_i32 s7, s7, s8
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s8, s4
+; GISEL-NEXT: s_addc_u32 s8, s9, 0
+; GISEL-NEXT: s_add_u32 s4, s4, s7
+; GISEL-NEXT: s_addc_u32 s7, s8, 0
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_add_i32 s8, s8, s7
; GISEL-NEXT: s_add_u32 s4, s5, s4
@@ -1447,30 +1368,28 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
-; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v10, v7, s4
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v5, s5
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v2, v0
; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v11, 0
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v10, v0
+; GISEL-NEXT: v_xor_b32_e32 v12, v9, v4
; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v13, v[3:4]
; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v12, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v11, v[9:10]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v7, v3
@@ -1481,7 +1400,7 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v12
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v11
; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v13, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
@@ -1493,7 +1412,7 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v6, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
; GISEL-NEXT: v_xor_b32_e32 v2, v2, v8
; GISEL-NEXT: v_xor_b32_e32 v3, v3, v8
@@ -1504,254 +1423,236 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-LABEL: v_sdiv_v2i64_oddk_denom:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
+; CGP-NEXT: v_cvt_f32_u32_e32 v5, 0x12d8fb
; CGP-NEXT: v_mov_b32_e32 v6, 0x4f800000
-; CGP-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT: v_mac_f32_e32 v4, 0, v6
-; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT: v_mac_f32_e32 v5, 0, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; CGP-NEXT: v_xor_b32_e32 v10, v0, v4
+; CGP-NEXT: v_mov_b32_e32 v0, 0xffed2705
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
; CGP-NEXT: v_trunc_f32_e32 v6, v6
-; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
-; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
-; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
-; CGP-NEXT: v_mul_hi_u32 v8, v4, v7
-; CGP-NEXT: v_readfirstlane_b32 s6, v7
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v7, v0, v5
+; CGP-NEXT: v_readfirstlane_b32 s6, v5
; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
; CGP-NEXT: v_readfirstlane_b32 s7, v6
-; CGP-NEXT: v_readfirstlane_b32 s5, v8
-; CGP-NEXT: v_mul_hi_u32 v8, v7, s4
+; CGP-NEXT: v_readfirstlane_b32 s5, v7
+; CGP-NEXT: v_mul_hi_u32 v7, v5, s4
; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
; CGP-NEXT: s_add_u32 s5, s8, s5
; CGP-NEXT: s_mul_i32 s8, s6, -1
; CGP-NEXT: s_add_u32 s5, s8, s5
-; CGP-NEXT: v_readfirstlane_b32 s8, v8
+; CGP-NEXT: v_readfirstlane_b32 s8, v7
+; CGP-NEXT: v_mul_hi_u32 v7, v6, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v5, s5
; CGP-NEXT: s_mul_i32 s9, s6, s5
-; CGP-NEXT: v_mul_hi_u32 v8, v6, s4
-; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
; CGP-NEXT: s_mul_i32 s10, s7, s4
; CGP-NEXT: s_add_u32 s11, s8, s9
; CGP-NEXT: s_cselect_b32 s12, 1, 0
-; CGP-NEXT: s_add_u32 s11, s11, s10
-; CGP-NEXT: s_cselect_b32 s4, 1, 0
-; CGP-NEXT: s_add_i32 s12, s12, s4
-; CGP-NEXT: v_readfirstlane_b32 s11, v8
-; CGP-NEXT: v_readfirstlane_b32 s13, v7
-; CGP-NEXT: s_mul_i32 s14, s7, s5
-; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: s_add_u32 s4, s11, s10
+; CGP-NEXT: s_addc_u32 s4, s12, 0
+; CGP-NEXT: v_readfirstlane_b32 s11, v7
+; CGP-NEXT: v_readfirstlane_b32 s12, v5
+; CGP-NEXT: s_mul_i32 s13, s7, s5
+; CGP-NEXT: s_add_u32 s14, s11, s12
+; CGP-NEXT: v_mul_hi_u32 v5, v6, s5
; CGP-NEXT: s_cselect_b32 s15, 1, 0
-; CGP-NEXT: s_add_u32 s4, s4, s14
-; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v6, v6, s5
-; CGP-NEXT: s_add_i32 s15, s15, s16
-; CGP-NEXT: s_add_u32 s4, s4, s12
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
-; CGP-NEXT: s_add_i32 s15, s15, s5
-; CGP-NEXT: v_readfirstlane_b32 s12, v6
-; CGP-NEXT: s_add_i32 s5, s12, s15
+; CGP-NEXT: s_add_u32 s14, s14, s13
+; CGP-NEXT: s_addc_u32 s5, s15, 0
+; CGP-NEXT: s_add_u32 s4, s14, s4
+; CGP-NEXT: s_addc_u32 s5, s5, 0
+; CGP-NEXT: v_readfirstlane_b32 s14, v5
+; CGP-NEXT: s_add_i32 s5, s14, s5
; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_mul_hi_u32 v6, v4, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v0, s4
; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
; CGP-NEXT: s_addc_u32 s5, s7, s5
; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s16, v6
-; CGP-NEXT: v_mov_b32_e32 v6, s15
-; CGP-NEXT: v_mul_hi_u32 v7, s4, v6
+; CGP-NEXT: v_readfirstlane_b32 s16, v5
+; CGP-NEXT: v_mov_b32_e32 v5, s15
+; CGP-NEXT: v_mul_hi_u32 v6, s4, v5
; CGP-NEXT: s_add_u32 s16, s17, s16
; CGP-NEXT: s_mul_i32 s17, s4, -1
; CGP-NEXT: s_add_u32 s16, s17, s16
-; CGP-NEXT: v_readfirstlane_b32 s17, v7
-; CGP-NEXT: v_mov_b32_e32 v7, s16
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: v_mov_b32_e32 v6, s16
+; CGP-NEXT: v_mul_hi_u32 v5, s5, v5
+; CGP-NEXT: v_mul_hi_u32 v7, s4, v6
; CGP-NEXT: s_mul_i32 s18, s4, s16
-; CGP-NEXT: v_mul_hi_u32 v6, s5, v6
-; CGP-NEXT: v_mul_hi_u32 v8, s4, v7
; CGP-NEXT: s_mul_i32 s15, s5, s15
; CGP-NEXT: s_add_u32 s17, s17, s18
; CGP-NEXT: s_cselect_b32 s18, 1, 0
; CGP-NEXT: s_add_u32 s15, s17, s15
-; CGP-NEXT: s_cselect_b32 s15, 1, 0
-; CGP-NEXT: s_add_i32 s18, s18, s15
-; CGP-NEXT: v_readfirstlane_b32 s15, v6
-; CGP-NEXT: v_readfirstlane_b32 s17, v8
+; CGP-NEXT: s_addc_u32 s15, s18, 0
+; CGP-NEXT: v_readfirstlane_b32 s17, v5
+; CGP-NEXT: v_readfirstlane_b32 s18, v7
; CGP-NEXT: s_mul_i32 s16, s5, s16
-; CGP-NEXT: s_add_u32 s15, s15, s17
-; CGP-NEXT: s_cselect_b32 s17, 1, 0
-; CGP-NEXT: s_add_u32 s15, s15, s16
-; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v6, s5, v7
-; CGP-NEXT: s_add_i32 s16, s17, s16
-; CGP-NEXT: s_add_u32 s15, s15, s18
-; CGP-NEXT: s_cselect_b32 s17, 1, 0
-; CGP-NEXT: s_add_i32 s16, s16, s17
-; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: s_add_u32 s17, s17, s18
+; CGP-NEXT: v_mul_hi_u32 v5, s5, v6
+; CGP-NEXT: s_cselect_b32 s18, 1, 0
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: s_addc_u32 s17, s18, 0
+; CGP-NEXT: s_add_u32 s15, s16, s15
+; CGP-NEXT: s_addc_u32 s16, s17, 0
+; CGP-NEXT: v_readfirstlane_b32 s17, v5
; CGP-NEXT: s_add_i32 s17, s17, s16
; CGP-NEXT: s_add_u32 s4, s4, s15
; CGP-NEXT: s_addc_u32 s5, s5, s17
-; CGP-NEXT: v_mul_hi_u32 v7, v0, s4
-; CGP-NEXT: v_mul_lo_u32 v8, v0, s5
-; CGP-NEXT: v_mul_lo_u32 v9, v1, s4
-; CGP-NEXT: v_mul_hi_u32 v10, v0, s5
-; CGP-NEXT: v_mov_b32_e32 v6, 0x12d8fb
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v6, v10, s4
+; CGP-NEXT: v_mul_lo_u32 v7, v10, s5
+; CGP-NEXT: v_mul_lo_u32 v8, v1, s4
; CGP-NEXT: v_mul_hi_u32 v9, v1, s4
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_mul_lo_u32 v8, v1, s5
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_mul_hi_u32 v11, v10, s5
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_lo_u32 v7, v1, s5
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v11
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v8, v7
-; CGP-NEXT: v_mul_hi_u32 v12, v1, s5
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v11, 0
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v9
-; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v12, v[8:9]
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
-; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v9, vcc
-; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v9
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v6
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
+; CGP-NEXT: v_mov_b32_e32 v5, 0x12d8fb
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v7, v6
+; CGP-NEXT: v_mul_hi_u32 v9, v1, s5
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v11, 0
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v9, v8
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[7:8]
+; CGP-NEXT: v_sub_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v8, vcc
+; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v8
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v5
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
; CGP-NEXT: v_cndmask_b32_e64 v7, -1, v8, s[4:5]
-; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CGP-NEXT: s_add_u32 s4, s8, s9
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s10
+; CGP-NEXT: s_addc_u32 s4, s5, 0
; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: s_cselect_b32 s4, 1, 0
+; CGP-NEXT: s_add_u32 s5, s11, s12
+; CGP-NEXT: v_sub_i32_e32 v6, vcc, v6, v5
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s5, s5, s13
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT: s_addc_u32 s8, s8, 0
; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v11
-; CGP-NEXT: s_add_i32 s5, s5, s4
+; CGP-NEXT: s_add_u32 s4, s5, s4
; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
-; CGP-NEXT: s_add_u32 s4, s11, s13
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_u32 s4, s4, s14
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT: s_addc_u32 s5, s8, 0
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v6, v5
+; CGP-NEXT: s_add_i32 s14, s14, s5
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT: s_cselect_b32 s9, 1, 0
-; CGP-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v8
-; CGP-NEXT: s_add_i32 s8, s8, s9
+; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: v_cndmask_b32_e32 v1, -1, v6, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v8
+; CGP-NEXT: v_mul_hi_u32 v0, v0, s4
; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v9, vcc
-; CGP-NEXT: s_add_u32 s4, s4, s5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
-; CGP-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
-; CGP-NEXT: s_add_i32 s8, s8, s5
-; CGP-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
-; CGP-NEXT: s_add_i32 s12, s12, s8
-; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
-; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
+; CGP-NEXT: v_ashrrev_i32_e32 v8, 31, v3
; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_mov_b32_e32 v2, s6
-; CGP-NEXT: s_addc_u32 s5, s7, s12
-; CGP-NEXT: v_readfirstlane_b32 s7, v4
-; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v6, v12, v6, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT: s_addc_u32 s5, s7, s14
+; CGP-NEXT: v_readfirstlane_b32 s7, v0
+; CGP-NEXT: v_mov_b32_e32 v0, s6
+; CGP-NEXT: v_xor_b32_e32 v9, v2, v8
+; CGP-NEXT: v_mul_hi_u32 v2, s4, v0
; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CGP-NEXT: s_add_u32 s7, s8, s7
; CGP-NEXT: s_mul_i32 s8, s4, -1
; CGP-NEXT: s_add_u32 s7, s8, s7
-; CGP-NEXT: v_readfirstlane_b32 s8, v4
-; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: v_mov_b32_e32 v2, s7
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
+; CGP-NEXT: v_mul_hi_u32 v7, s4, v2
; CGP-NEXT: s_mul_i32 s9, s4, s7
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
-; CGP-NEXT: v_mul_hi_u32 v5, s4, v4
; CGP-NEXT: s_mul_i32 s6, s5, s6
; CGP-NEXT: s_add_u32 s8, s8, s9
; CGP-NEXT: s_cselect_b32 s9, 1, 0
; CGP-NEXT: s_add_u32 s6, s8, s6
-; CGP-NEXT: s_cselect_b32 s6, 1, 0
-; CGP-NEXT: s_add_i32 s9, s9, s6
-; CGP-NEXT: v_readfirstlane_b32 s6, v2
-; CGP-NEXT: v_readfirstlane_b32 s8, v5
+; CGP-NEXT: s_addc_u32 s6, s9, 0
+; CGP-NEXT: v_readfirstlane_b32 s8, v0
+; CGP-NEXT: v_readfirstlane_b32 s9, v7
; CGP-NEXT: s_mul_i32 s7, s5, s7
-; CGP-NEXT: s_add_u32 s6, s6, s8
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_u32 s6, s6, s7
-; CGP-NEXT: s_cselect_b32 s7, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
-; CGP-NEXT: s_add_i32 s7, s8, s7
-; CGP-NEXT: s_add_u32 s6, s6, s9
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_i32 s7, s7, s8
-; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: s_add_u32 s8, s8, s9
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v2
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: s_addc_u32 s8, s9, 0
+; CGP-NEXT: s_add_u32 s6, s7, s6
+; CGP-NEXT: s_addc_u32 s7, s8, 0
+; CGP-NEXT: v_readfirstlane_b32 s8, v0
; CGP-NEXT: s_add_i32 s8, s8, s7
; CGP-NEXT: s_add_u32 s4, s4, s6
-; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
; CGP-NEXT: s_addc_u32 s5, s5, s8
-; CGP-NEXT: v_mul_hi_u32 v2, v8, s4
-; CGP-NEXT: v_mul_lo_u32 v4, v8, s5
-; CGP-NEXT: v_xor_b32_e32 v9, v3, v7
-; CGP-NEXT: v_mul_lo_u32 v3, v9, s4
-; CGP-NEXT: v_mul_hi_u32 v5, v8, s5
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
+; CGP-NEXT: v_mul_lo_u32 v2, v9, s5
+; CGP-NEXT: v_xor_b32_e32 v10, v3, v8
+; CGP-NEXT: v_mul_lo_u32 v3, v10, s4
+; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v3, v9, s4
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CGP-NEXT: v_mul_lo_u32 v4, v9, s5
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v3, v2
; CGP-NEXT: v_mul_hi_u32 v11, v9, s5
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v10, 0
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v4
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v11, v[3:4]
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
-; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v9, v4, vcc
-; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v9, v4
-; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; CGP-NEXT: v_mul_lo_u32 v2, v10, s5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v7, v11
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v2, v0
+; CGP-NEXT: v_mul_hi_u32 v12, v10, s5
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v11, 0
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT: v_xor_b32_e32 v6, v6, v4
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v6, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v7
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v4, v[3:4]
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v9, v2
+; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v10, v6, vcc
+; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v10, v6
+; CGP-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v5
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v3, -1, v5, s[4:5]
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v10
-; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v3, -1, v7, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v11
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v4, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v5
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v7
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; CGP-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v3, v11, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v2, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v3, v3, v7
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v8
+; CGP-NEXT: v_xor_b32_e32 v3, v3, v8
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = sdiv <2 x i64> %num, <i64 1235195, i64 1235195>
ret <2 x i64> %result
@@ -1809,17 +1710,14 @@ define i64 @v_sdiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v15
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_mul_hi_u32 v8, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CHECK-NEXT: v_add_i32_e32 v12, vcc, v12, v5
; CHECK-NEXT: v_addc_u32_e32 v11, vcc, v11, v6, vcc
@@ -1835,21 +1733,18 @@ define i64 @v_sdiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v13, vcc
; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v8, v12, v9
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v6, v12, v9
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
-; CHECK-NEXT: v_mul_lo_u32 v7, v11, v9
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT: v_mul_lo_u32 v6, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v8
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_hi_u32 v7, v11, v9
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v12, v3
; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v11, v5, vcc
@@ -1859,22 +1754,19 @@ define i64 @v_sdiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v4, v9, v3
; CHECK-NEXT: v_mul_hi_u32 v3, v9, v3
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v8, v10, v5
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v6, v10, v5
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; CHECK-NEXT: v_mul_lo_u32 v7, v9, v5
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; CHECK-NEXT: v_mul_lo_u32 v6, v9, v5
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v7, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v8
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v4
; CHECK-NEXT: v_mul_hi_u32 v5, v9, v5
; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v11, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v12, vcc, v5, v6
; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v2, v12, v[4:5]
; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v10, v3
@@ -1976,213 +1868,195 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_hi_u32 v20, v16, v11
; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v9, v[13:14]
; GISEL-NEXT: v_mul_lo_u32 v12, v9, v11
-; GISEL-NEXT: v_mul_hi_u32 v14, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v14, v16, v11
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v18, v12
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
-; GISEL-NEXT: v_mul_lo_u32 v13, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v20, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v18, v14
+; GISEL-NEXT: v_mul_hi_u32 v12, v9, v11
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
; GISEL-NEXT: v_mul_hi_u32 v11, v16, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v20, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v18, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; GISEL-NEXT: v_add_i32_e32 v18, vcc, v9, v12
; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v16, v11, vcc
; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v15, v18, 0
+; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v15, v19, v[12:13]
-; GISEL-NEXT: v_mul_hi_u32 v9, v18, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v18, v11
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v17, v18, v[13:14]
; GISEL-NEXT: v_mul_lo_u32 v13, v19, v11
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v9
; GISEL-NEXT: v_mul_hi_u32 v11, v19, v11
-; GISEL-NEXT: v_mul_lo_u32 v12, v18, v15
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_mul_lo_u32 v14, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v14
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v1
-; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v9
-; GISEL-NEXT: v_xor_b32_e32 v17, v0, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v18, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_mul_hi_u32 v14, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v12, vcc
; GISEL-NEXT: v_mul_lo_u32 v12, v19, v15
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_mul_hi_u32 v13, v19, v15
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v18, v0
-; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], v1, v9, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v19, v11, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v13, v17, v11
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
-; GISEL-NEXT: v_mul_lo_u32 v14, v1, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v1, v0
+; GISEL-NEXT: v_mul_hi_u32 v12, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v13, v16, v11
+; GISEL-NEXT: v_xor_b32_e32 v15, v1, v9
+; GISEL-NEXT: v_mul_lo_u32 v1, v15, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_mul_hi_u32 v14, v16, v11
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v14, v17, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
-; GISEL-NEXT: v_mul_lo_u32 v13, v1, v11
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_mul_lo_u32 v12, v15, v11
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT: v_mul_hi_u32 v15, v1, v11
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v0, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v15, v13
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v10, v18, v[12:13]
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v17, v11
-; GISEL-NEXT: v_xor_b32_e32 v9, v9, v4
-; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v5, v0, v[13:14]
-; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v1, v15, vcc
-; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v15
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v11
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v17, 0
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v11, v12
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v18, v[1:2]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v5, v17, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], v15, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v15, v13
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v6
-; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v8, v6, s[4:5]
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v8, v6, s[4:5]
; GISEL-NEXT: v_xor_b32_e32 v8, v7, v6
-; GISEL-NEXT: v_xor_b32_e32 v7, v12, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v13, v7
+; GISEL-NEXT: v_xor_b32_e32 v7, v11, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v7
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v10
-; GISEL-NEXT: v_mac_f32_e32 v12, 0x4f800000, v13
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v12, v12
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v11, v10
-; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v10
-; GISEL-NEXT: v_mul_f32_e32 v10, 0x5f7ffffc, v12
-; GISEL-NEXT: v_mul_f32_e32 v11, 0x2f800000, v10
-; GISEL-NEXT: v_trunc_f32_e32 v11, v11
-; GISEL-NEXT: v_mac_f32_e32 v10, 0xcf800000, v11
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v10
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v10
+; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
+; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GISEL-NEXT: v_trunc_f32_e32 v1, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v0
; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v11
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v7, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v17, 0
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v1, v5
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v20, v19, v[11:12]
-; GISEL-NEXT: v_mul_hi_u32 v11, v17, v10
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[6:7], v21, v17, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v5
-; GISEL-NEXT: v_mul_lo_u32 v12, v17, v14
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[6:7]
-; GISEL-NEXT: v_mul_hi_u32 v10, v19, v10
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v18, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v15, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 1, v11
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v12, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v15, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v16, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v15, v5
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v19, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v16, v0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v21, v16, v[10:11]
+; GISEL-NEXT: v_mul_lo_u32 v10, v16, v12
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_mul_lo_u32 v10, v19, v0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_mul_hi_u32 v13, v17, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_mul_lo_u32 v13, v19, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v14, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v11
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v17, v10
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v11, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v13, 0
+; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v15, v5
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v14, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v13, v15, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v18, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, 1, v10
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v13, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v18, v12, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v14, v[11:12]
-; GISEL-NEXT: v_xor_b32_e32 v11, v5, v9
+; GISEL-NEXT: v_mul_hi_u32 v1, v16, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v11, s[6:7]
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_mul_lo_u32 v1, v19, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
-; GISEL-NEXT: v_xor_b32_e32 v12, v12, v9
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v13, v[0:1]
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v15
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v3, v15, vcc
-; GISEL-NEXT: v_mul_hi_u32 v2, v13, v10
-; GISEL-NEXT: v_mul_lo_u32 v3, v13, v4
-; GISEL-NEXT: v_xor_b32_e32 v16, v0, v15
-; GISEL-NEXT: v_mul_lo_u32 v0, v14, v10
-; GISEL-NEXT: v_mul_hi_u32 v5, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v2, v14, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0
-; GISEL-NEXT: v_mul_lo_u32 v3, v14, v4
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v1, v19, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v14, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v0
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v12, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v17, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v14, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v9, v4
+; GISEL-NEXT: v_xor_b32_e32 v9, v5, v1
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v12, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v5, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v11, v2, v15
+; GISEL-NEXT: v_mul_lo_u32 v2, v14, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v12, v4
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT: v_mul_hi_u32 v16, v12, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_mul_lo_u32 v5, v14, v4
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v16
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
; GISEL-NEXT: v_mul_hi_u32 v4, v14, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v14, v2, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v16, v0
-; GISEL-NEXT: v_mul_lo_u32 v4, v16, v2
-; GISEL-NEXT: v_xor_b32_e32 v13, v1, v15
-; GISEL-NEXT: v_mul_lo_u32 v1, v13, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v3, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v16, v2
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v4, v1
-; GISEL-NEXT: v_mul_lo_u32 v4, v13, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v11, v0
+; GISEL-NEXT: v_mul_lo_u32 v5, v11, v2
+; GISEL-NEXT: v_xor_b32_e32 v12, v3, v15
+; GISEL-NEXT: v_mul_lo_u32 v3, v12, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v12, v0
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_mul_hi_u32 v10, v11, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GISEL-NEXT: v_mul_lo_u32 v4, v12, v2
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v3, v4
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v0, v1
-; GISEL-NEXT: v_mul_hi_u32 v1, v13, v2
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v0, v3
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v12, v2
; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v14, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v1, v0
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v17, v[3:4]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v9
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v12, v9, vcc
+; GISEL-NEXT: v_xor_b32_e32 v10, v13, v1
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v4, v0
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v13, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v9, v1
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v10, v1, vcc
; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v7, v14, v[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v16, v2
-; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v13, v9, vcc
-; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v13, v9
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v12, v9, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v9
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v7
; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v7, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
@@ -2193,7 +2067,7 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, v9, s[4:5]
; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v14
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v17, vcc
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v13, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v4, v7
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v8
@@ -2207,7 +2081,7 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v7, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
; GISEL-NEXT: v_cndmask_b32_e32 v2, v14, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v17, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v4, vcc
; GISEL-NEXT: v_xor_b32_e32 v4, v15, v6
; GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
; GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
@@ -2262,17 +2136,14 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v15
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v20
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_mul_hi_u32 v13, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v10
; CGP-NEXT: v_addc_u32_e32 v16, vcc, v16, v11, vcc
@@ -2288,21 +2159,18 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_addc_u32_e32 v9, vcc, v9, v18, vcc
; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v17, v14
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v11, v17, v14
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v12, v8
-; CGP-NEXT: v_mul_lo_u32 v12, v16, v14
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_mul_lo_u32 v11, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v12, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_mul_hi_u32 v12, v16, v14
; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v12, v10
; CGP-NEXT: v_add_i32_e32 v8, vcc, v17, v8
; CGP-NEXT: v_addc_u32_e32 v10, vcc, v16, v10, vcc
@@ -2312,22 +2180,19 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v9, v14, v8
; CGP-NEXT: v_mul_hi_u32 v8, v14, v8
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v15, v10
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v11, v15, v10
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v12, v9
-; CGP-NEXT: v_mul_lo_u32 v12, v14, v10
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT: v_mul_lo_u32 v11, v14, v10
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v16, vcc, v8, v9
; CGP-NEXT: v_mul_hi_u32 v10, v14, v10
; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v16, 0
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v17, vcc, v10, v11
; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v17, v[9:10]
; CGP-NEXT: v_sub_i32_e32 v8, vcc, v15, v8
@@ -2437,17 +2302,14 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v17
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; CGP-NEXT: v_mul_hi_u32 v11, v14, v12
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v9, vcc
@@ -2463,21 +2325,18 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_addc_u32_e32 v7, vcc, v7, v15, vcc
; CGP-NEXT: v_mul_hi_u32 v8, v14, v8
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v13, v6, v12
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v9, v6, v12
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; CGP-NEXT: v_mul_lo_u32 v10, v14, v12
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT: v_mul_lo_u32 v9, v14, v12
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CGP-NEXT: v_mul_hi_u32 v10, v14, v12
; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; CGP-NEXT: v_addc_u32_e32 v6, vcc, v14, v8, vcc
@@ -2487,23 +2346,20 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v7, v12, v5
; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_mul_hi_u32 v10, v11, v6
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v11, v6
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; CGP-NEXT: v_mul_lo_u32 v9, v12, v6
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_mul_lo_u32 v8, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
-; CGP-NEXT: v_mul_hi_u32 v9, v12, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v12, v6
; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v4, v13, 0
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v9, v7
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v8, v7
; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v14, v[6:7]
; CGP-NEXT: v_sub_i32_e32 v5, vcc, v11, v5
; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v13, v[7:8]
@@ -2637,184 +2493,170 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
-; GISEL-NEXT: v_sub_i32_e32 v12, vcc, 0, v1
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v5
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v11, 0
+; GISEL-NEXT: v_sub_i32_e32 v10, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v11
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT: v_subb_u32_e64 v13, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; GISEL-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; GISEL-NEXT: v_trunc_f32_e32 v4, v4
; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v11, v3
-; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v4
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v11, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v14, v[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v4, v11, v3
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v11, v[7:8]
-; GISEL-NEXT: v_mul_lo_u32 v7, v14, v3
-; GISEL-NEXT: v_mul_hi_u32 v3, v14, v3
-; GISEL-NEXT: v_mul_lo_u32 v8, v11, v9
-; GISEL-NEXT: v_mul_hi_u32 v10, v11, v9
-; GISEL-NEXT: v_mul_lo_u32 v15, v14, v9
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v9, v3
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v13, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v9, v[7:8]
+; GISEL-NEXT: v_mul_hi_u32 v5, v9, v3
+; GISEL-NEXT: v_mul_lo_u32 v7, v13, v3
+; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
+; GISEL-NEXT: v_mul_lo_u32 v8, v9, v4
+; GISEL-NEXT: v_mul_hi_u32 v14, v9, v4
+; GISEL-NEXT: v_mul_lo_u32 v15, v13, v4
+; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v8, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v10
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v14
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v9
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v8, v4
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v3
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v10, 0
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v14, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v11, v[8:9]
-; GISEL-NEXT: v_and_b32_e32 v12, 0xffffff, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v10, v7
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v10, v[3:4]
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v9, v3
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v14, 0
; GISEL-NEXT: v_and_b32_e32 v3, 0xffffff, v6
-; GISEL-NEXT: v_mul_lo_u32 v6, v11, v7
-; GISEL-NEXT: v_and_b32_e32 v13, 0xffffff, v2
-; GISEL-NEXT: v_mul_lo_u32 v4, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v13, v[5:6]
+; GISEL-NEXT: v_mul_lo_u32 v6, v13, v4
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v14, v[7:8]
+; GISEL-NEXT: v_and_b32_e32 v12, 0xffffff, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v14, v4
+; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT: v_mul_lo_u32 v5, v14, v9
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v6, v11, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GISEL-NEXT: v_mul_lo_u32 v4, v11, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc
+; GISEL-NEXT: v_mul_lo_u32 v5, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_mul_hi_u32 v7, v11, v8
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GISEL-NEXT: v_mul_hi_u32 v6, v13, v9
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v11, v4, vcc
-; GISEL-NEXT: v_mul_hi_u32 v6, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v7, v12, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v13, v4, vcc
+; GISEL-NEXT: v_mul_hi_u32 v5, v12, v0
; GISEL-NEXT: v_mul_hi_u32 v0, 0, v0
-; GISEL-NEXT: v_mul_hi_u32 v8, v12, v4
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v4
-; GISEL-NEXT: v_mac_f32_e32 v10, 0x4f800000, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; GISEL-NEXT: v_mul_hi_u32 v6, v12, v4
+; GISEL-NEXT: v_mul_lo_u32 v8, 0, v4
+; GISEL-NEXT: v_mul_lo_u32 v7, v12, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v3
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v0, 0
-; GISEL-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v5
-; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v2
-; GISEL-NEXT: v_trunc_f32_e32 v5, v5
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v4, v[7:8]
-; GISEL-NEXT: v_mac_f32_e32 v2, 0xcf800000, v5
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v1, v0, 0
+; GISEL-NEXT: v_mac_f32_e32 v9, 0x4f800000, v11
+; GISEL-NEXT: v_and_b32_e32 v13, 0xffffff, v2
+; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v3
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v4, v[6:7]
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v6, v9
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v0, v[7:8]
+; GISEL-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v6
+; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v2
+; GISEL-NEXT: v_trunc_f32_e32 v6, v6
+; GISEL-NEXT: v_mac_f32_e32 v2, 0xcf800000, v6
; GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 0, v3
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], 0, v0, v[8:9]
-; GISEL-NEXT: v_cvt_u32_f32_e32 v11, v5
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v2, 0
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v12, v6
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v11, v[8:9]
-; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, v10, vcc
-; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], 0, v10
+; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v6
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v12, v5
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v2, 0
+; GISEL-NEXT: v_subb_u32_e64 v5, s[4:5], 0, v9, vcc
+; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], 0, v9
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v14, v[7:8]
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v1
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v2, v[5:6]
-; GISEL-NEXT: v_mul_hi_u32 v5, v2, v7
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GISEL-NEXT: v_mul_lo_u32 v16, v11, v7
-; GISEL-NEXT: v_mul_lo_u32 v6, v2, v8
-; GISEL-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_mul_hi_u32 v7, v11, v7
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v16, v2, v8
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; GISEL-NEXT: v_mul_lo_u32 v6, v11, v8
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v16, v7
-; GISEL-NEXT: v_mul_hi_u32 v8, v11, v8
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v16, v2, v[10:11]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v2, v6
+; GISEL-NEXT: v_mul_lo_u32 v11, v14, v6
+; GISEL-NEXT: v_mul_lo_u32 v8, v2, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v10, -1, v17, s[4:5]
+; GISEL-NEXT: v_mul_hi_u32 v6, v14, v6
+; GISEL-NEXT: v_mul_hi_u32 v17, v2, v7
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
+; GISEL-NEXT: v_mul_lo_u32 v8, v14, v7
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v2, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v9, -1, v17, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v11, v6, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v16, 0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v2, v5
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v6, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v15, v11, 0
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v1
-; GISEL-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v11, v[6:7]
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v14, v[6:7]
; GISEL-NEXT: v_add_i32_e32 v12, vcc, 1, v0
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v4, vcc
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v4, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1
-; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v15, v16, v[7:8]
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v16, v11, v[7:8]
; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
; GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
-; GISEL-NEXT: v_mul_hi_u32 v6, v16, v5
-; GISEL-NEXT: v_mul_lo_u32 v7, v16, v1
-; GISEL-NEXT: v_mul_lo_u32 v15, v11, v5
+; GISEL-NEXT: v_mul_hi_u32 v6, v11, v5
+; GISEL-NEXT: v_mul_lo_u32 v7, v11, v1
+; GISEL-NEXT: v_mul_lo_u32 v16, v14, v5
; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v12
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v14, vcc
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v5, v11, v5
-; GISEL-NEXT: v_mul_hi_u32 v15, v16, v1
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_mul_lo_u32 v7, v11, v1
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v5, v14, v5
+; GISEL-NEXT: v_mul_hi_u32 v17, v11, v1
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v16
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mul_lo_u32 v7, v14, v1
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v15, v7
-; GISEL-NEXT: v_mul_hi_u32 v1, v11, v1
+; GISEL-NEXT: v_mul_hi_u32 v1, v14, v1
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v6
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v16, v5
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v11, v1, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v11, v5
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v14, v1, vcc
; GISEL-NEXT: v_mul_hi_u32 v6, v13, v5
-; GISEL-NEXT: v_mul_lo_u32 v7, v13, v1
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_mul_hi_u32 v11, v13, v1
-; GISEL-NEXT: v_mul_hi_u32 v15, 0, v1
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v7, v13, v1
+; GISEL-NEXT: v_mul_lo_u32 v14, 0, v1
+; GISEL-NEXT: v_mul_lo_u32 v11, v13, v1
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v5, v6
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v14
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT: v_mul_hi_u32 v14, 0, v1
; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v3, v11, 0
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
; GISEL-NEXT: v_cndmask_b32_e32 v1, v12, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v14, v10, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v15, v[6:7]
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v15, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v14, v[6:7]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc
; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v11, v[7:8]
@@ -2829,7 +2671,7 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v6, s[4:5]
; GISEL-NEXT: v_add_i32_e32 v6, vcc, 1, v11
-; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v15, vcc
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v14, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5
@@ -2841,7 +2683,7 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GISEL-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v15, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v14, v3, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_sdiv_v2i64_24bit:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index 61bd7cb51f0d075..4ae5f8b4266af60 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -209,43 +209,40 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX8-NEXT: v_mul_hi_u32 v2, s12, v0
; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: s_mul_i32 s17, s12, s16
; GFX8-NEXT: v_readfirstlane_b32 s14, v1
-; GFX8-NEXT: v_mul_hi_u32 v3, v0, s17
+; GFX8-NEXT: s_mul_i32 s17, s12, s16
; GFX8-NEXT: s_mul_i32 s15, s12, s14
+; GFX8-NEXT: v_mul_hi_u32 v3, v0, s17
; GFX8-NEXT: v_readfirstlane_b32 s20, v2
; GFX8-NEXT: s_mul_i32 s18, s13, s16
; GFX8-NEXT: s_add_u32 s15, s15, s20
; GFX8-NEXT: s_add_u32 s15, s18, s15
-; GFX8-NEXT: v_readfirstlane_b32 s18, v3
-; GFX8-NEXT: s_mul_i32 s20, s16, s15
; GFX8-NEXT: v_mul_hi_u32 v2, v1, s17
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s15
+; GFX8-NEXT: v_readfirstlane_b32 s18, v3
+; GFX8-NEXT: s_mul_i32 s20, s16, s15
; GFX8-NEXT: s_mul_i32 s19, s14, s17
; GFX8-NEXT: s_add_u32 s18, s18, s20
; GFX8-NEXT: s_cselect_b32 s20, 1, 0
+; GFX8-NEXT: s_add_u32 s17, s18, s19
+; GFX8-NEXT: s_addc_u32 s17, s20, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: v_readfirstlane_b32 s19, v0
+; GFX8-NEXT: s_mul_i32 s20, s14, s15
; GFX8-NEXT: s_add_u32 s18, s18, s19
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
-; GFX8-NEXT: s_add_i32 s20, s20, s17
-; GFX8-NEXT: v_readfirstlane_b32 s17, v2
-; GFX8-NEXT: v_readfirstlane_b32 s18, v0
-; GFX8-NEXT: s_mul_i32 s19, s14, s15
-; GFX8-NEXT: s_add_u32 s17, s17, s18
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_u32 s17, s17, s19
-; GFX8-NEXT: s_cselect_b32 s19, 1, 0
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s15
-; GFX8-NEXT: s_add_i32 s18, s18, s19
-; GFX8-NEXT: s_add_u32 s15, s17, s20
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
-; GFX8-NEXT: s_add_i32 s18, s18, s17
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: s_add_i32 s17, s17, s18
-; GFX8-NEXT: s_add_u32 s15, s16, s15
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_add_u32 s18, s18, s20
+; GFX8-NEXT: s_addc_u32 s15, s19, 0
+; GFX8-NEXT: s_add_u32 s17, s18, s17
+; GFX8-NEXT: s_addc_u32 s15, s15, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v0
+; GFX8-NEXT: s_add_i32 s18, s18, s15
+; GFX8-NEXT: s_add_u32 s15, s16, s17
; GFX8-NEXT: v_mov_b32_e32 v0, s15
; GFX8-NEXT: v_mul_hi_u32 v0, s12, v0
; GFX8-NEXT: s_mul_i32 s16, s12, s15
-; GFX8-NEXT: s_addc_u32 s14, s14, s17
+; GFX8-NEXT: s_addc_u32 s14, s14, s18
; GFX8-NEXT: s_mul_i32 s12, s12, s14
; GFX8-NEXT: v_readfirstlane_b32 s17, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s16
@@ -255,27 +252,24 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: s_add_u32 s12, s13, s12
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: s_mul_i32 s17, s15, s12
; GFX8-NEXT: v_mul_hi_u32 v0, s14, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s15, v1
+; GFX8-NEXT: s_mul_i32 s17, s15, s12
; GFX8-NEXT: s_mul_i32 s16, s14, s16
; GFX8-NEXT: s_add_u32 s13, s13, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_u32 s13, s13, s16
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s17, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: v_readfirstlane_b32 s16, v2
+; GFX8-NEXT: s_addc_u32 s13, s17, 0
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: v_readfirstlane_b32 s17, v2
; GFX8-NEXT: s_mul_i32 s12, s14, s12
-; GFX8-NEXT: s_add_u32 s13, s13, s16
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_u32 s12, s13, s12
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_u32 s16, s16, s17
; GFX8-NEXT: v_mul_hi_u32 v0, s14, v1
-; GFX8-NEXT: s_add_i32 s13, s16, s13
-; GFX8-NEXT: s_add_u32 s12, s12, s17
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s13, s16
+; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_add_u32 s12, s16, s12
+; GFX8-NEXT: s_addc_u32 s16, s17, 0
+; GFX8-NEXT: s_add_u32 s12, s12, s13
+; GFX8-NEXT: s_addc_u32 s13, s16, 0
; GFX8-NEXT: v_readfirstlane_b32 s16, v0
; GFX8-NEXT: s_add_i32 s16, s16, s13
; GFX8-NEXT: s_add_u32 s12, s15, s12
@@ -283,31 +277,28 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
; GFX8-NEXT: s_addc_u32 s13, s14, s16
; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: s_mul_i32 s15, s6, s13
+; GFX8-NEXT: v_mul_hi_u32 v2, s6, v1
; GFX8-NEXT: v_readfirstlane_b32 s14, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s7
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s12
-; GFX8-NEXT: v_mul_hi_u32 v2, s6, v1
+; GFX8-NEXT: s_mul_i32 s15, s6, s13
; GFX8-NEXT: s_mul_i32 s16, s7, s12
; GFX8-NEXT: s_add_u32 s14, s14, s15
; GFX8-NEXT: s_cselect_b32 s15, 1, 0
-; GFX8-NEXT: s_add_u32 s14, s14, s16
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s15, s15, s12
-; GFX8-NEXT: v_readfirstlane_b32 s12, v0
-; GFX8-NEXT: v_readfirstlane_b32 s14, v2
+; GFX8-NEXT: s_add_u32 s12, s14, s16
+; GFX8-NEXT: s_addc_u32 s12, s15, 0
+; GFX8-NEXT: v_readfirstlane_b32 s14, v0
+; GFX8-NEXT: v_readfirstlane_b32 s15, v2
; GFX8-NEXT: s_mul_i32 s13, s7, s13
-; GFX8-NEXT: s_add_u32 s12, s12, s14
-; GFX8-NEXT: s_cselect_b32 s14, 1, 0
-; GFX8-NEXT: s_add_u32 s12, s12, s13
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s14, s13
-; GFX8-NEXT: s_add_u32 s12, s12, s15
+; GFX8-NEXT: s_add_u32 s14, s14, s15
+; GFX8-NEXT: s_cselect_b32 s15, 1, 0
+; GFX8-NEXT: s_add_u32 s13, s14, s13
+; GFX8-NEXT: s_addc_u32 s14, s15, 0
+; GFX8-NEXT: s_add_u32 s12, s13, s12
; GFX8-NEXT: v_mul_hi_u32 v0, s7, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s12
; GFX8-NEXT: v_mul_hi_u32 v1, s4, v1
-; GFX8-NEXT: s_cselect_b32 s14, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s13, s14
+; GFX8-NEXT: s_addc_u32 s13, s14, 0
; GFX8-NEXT: v_readfirstlane_b32 s14, v0
; GFX8-NEXT: s_add_i32 s13, s14, s13
; GFX8-NEXT: v_readfirstlane_b32 s15, v1
@@ -418,19 +409,16 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s18, s18, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
; GFX9-NEXT: s_add_u32 s18, s18, s20
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s17, s14, s17
-; GFX9-NEXT: s_mul_hi_u32 s18, s16, s15
+; GFX9-NEXT: s_mul_hi_u32 s19, s16, s15
; GFX9-NEXT: s_mul_i32 s20, s14, s15
-; GFX9-NEXT: s_add_u32 s17, s17, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_u32 s17, s17, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s20
; GFX9-NEXT: s_add_u32 s17, s17, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s19
+; GFX9-NEXT: s_add_u32 s17, s17, s20
+; GFX9-NEXT: s_addc_u32 s19, s19, 0
+; GFX9-NEXT: s_add_u32 s17, s17, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s15, s14, s15
; GFX9-NEXT: s_add_i32 s15, s15, s18
; GFX9-NEXT: s_add_u32 s16, s16, s17
@@ -447,19 +435,16 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s13, s13, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
; GFX9-NEXT: s_add_u32 s13, s13, s18
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s17, s17, s13
-; GFX9-NEXT: s_mul_hi_u32 s13, s14, s15
-; GFX9-NEXT: s_mul_hi_u32 s15, s16, s12
+; GFX9-NEXT: s_addc_u32 s13, s17, 0
+; GFX9-NEXT: s_mul_hi_u32 s15, s14, s15
+; GFX9-NEXT: s_mul_hi_u32 s17, s16, s12
; GFX9-NEXT: s_mul_i32 s18, s14, s12
-; GFX9-NEXT: s_add_u32 s13, s13, s15
-; GFX9-NEXT: s_cselect_b32 s15, 1, 0
-; GFX9-NEXT: s_add_u32 s13, s13, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s15, s15, s18
-; GFX9-NEXT: s_add_u32 s13, s13, s17
+; GFX9-NEXT: s_add_u32 s15, s15, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
-; GFX9-NEXT: s_add_i32 s15, s15, s17
+; GFX9-NEXT: s_add_u32 s15, s15, s18
+; GFX9-NEXT: s_addc_u32 s17, s17, 0
+; GFX9-NEXT: s_add_u32 s13, s15, s13
+; GFX9-NEXT: s_addc_u32 s15, s17, 0
; GFX9-NEXT: s_mul_hi_u32 s12, s14, s12
; GFX9-NEXT: s_add_i32 s12, s12, s15
; GFX9-NEXT: s_add_u32 s13, s16, s13
@@ -470,19 +455,16 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s12, s12, s15
; GFX9-NEXT: s_cselect_b32 s15, 1, 0
; GFX9-NEXT: s_add_u32 s12, s12, s16
-; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_i32 s15, s15, s12
-; GFX9-NEXT: s_mul_hi_u32 s12, s7, s13
-; GFX9-NEXT: s_mul_hi_u32 s13, s6, s14
+; GFX9-NEXT: s_addc_u32 s12, s15, 0
+; GFX9-NEXT: s_mul_hi_u32 s13, s7, s13
+; GFX9-NEXT: s_mul_hi_u32 s15, s6, s14
; GFX9-NEXT: s_mul_i32 s16, s7, s14
-; GFX9-NEXT: s_add_u32 s12, s12, s13
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s16
-; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_i32 s13, s13, s16
-; GFX9-NEXT: s_add_u32 s12, s12, s15
+; GFX9-NEXT: s_add_u32 s13, s13, s15
; GFX9-NEXT: s_cselect_b32 s15, 1, 0
-; GFX9-NEXT: s_add_i32 s13, s13, s15
+; GFX9-NEXT: s_add_u32 s13, s13, s16
+; GFX9-NEXT: s_addc_u32 s15, s15, 0
+; GFX9-NEXT: s_add_u32 s12, s13, s12
+; GFX9-NEXT: s_addc_u32 s13, s15, 0
; GFX9-NEXT: s_mul_hi_u32 s14, s7, s14
; GFX9-NEXT: s_add_i32 s13, s14, s13
; GFX9-NEXT: s_mul_hi_u32 s15, s4, s12
@@ -588,21 +570,18 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX10-NEXT: s_mul_i32 s20, s14, s17
; GFX10-NEXT: s_add_u32 s18, s18, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_add_u32 s18, s18, s20
-; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s17, s14, s17
; GFX10-NEXT: s_mul_hi_u32 s21, s15, s16
-; GFX10-NEXT: s_add_i32 s19, s19, s18
-; GFX10-NEXT: s_mul_i32 s20, s14, s16
+; GFX10-NEXT: s_add_u32 s18, s18, s20
+; GFX10-NEXT: s_addc_u32 s18, s19, 0
+; GFX10-NEXT: s_mul_i32 s22, s14, s16
; GFX10-NEXT: s_add_u32 s17, s17, s21
-; GFX10-NEXT: s_cselect_b32 s18, 1, 0
-; GFX10-NEXT: s_add_u32 s17, s17, s20
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s16, s14, s16
-; GFX10-NEXT: s_add_i32 s18, s18, s20
-; GFX10-NEXT: s_add_u32 s17, s17, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_add_i32 s18, s18, s19
+; GFX10-NEXT: s_add_u32 s17, s17, s22
+; GFX10-NEXT: s_addc_u32 s19, s19, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s18
+; GFX10-NEXT: s_mul_hi_u32 s16, s14, s16
+; GFX10-NEXT: s_addc_u32 s18, s19, 0
; GFX10-NEXT: s_add_i32 s16, s16, s18
; GFX10-NEXT: s_add_u32 s15, s15, s17
; GFX10-NEXT: s_addc_u32 s14, s14, s16
@@ -620,44 +599,38 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX10-NEXT: s_add_u32 s16, s18, s16
; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_add_u32 s16, s16, s17
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_mul_i32 s17, s14, s12
-; GFX10-NEXT: s_add_i32 s18, s18, s16
+; GFX10-NEXT: s_addc_u32 s16, s18, 0
+; GFX10-NEXT: s_mul_i32 s20, s14, s12
; GFX10-NEXT: s_add_u32 s13, s13, s19
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s13, s13, s17
; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s20
+; GFX10-NEXT: s_addc_u32 s17, s17, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s16
; GFX10-NEXT: s_mul_hi_u32 s12, s14, s12
-; GFX10-NEXT: s_add_i32 s16, s16, s17
-; GFX10-NEXT: s_add_u32 s13, s13, s18
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
-; GFX10-NEXT: s_add_i32 s16, s16, s17
+; GFX10-NEXT: s_addc_u32 s16, s17, 0
; GFX10-NEXT: s_add_i32 s12, s12, s16
; GFX10-NEXT: s_add_u32 s13, s15, s13
-; GFX10-NEXT: s_addc_u32 s14, s14, s12
-; GFX10-NEXT: s_mul_hi_u32 s12, s6, s13
-; GFX10-NEXT: s_mul_i32 s16, s6, s14
+; GFX10-NEXT: s_addc_u32 s12, s14, s12
+; GFX10-NEXT: s_mul_hi_u32 s14, s6, s13
+; GFX10-NEXT: s_mul_i32 s16, s6, s12
; GFX10-NEXT: s_mul_i32 s15, s7, s13
-; GFX10-NEXT: s_add_u32 s12, s12, s16
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s12, s12, s15
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s17, s6, s12
+; GFX10-NEXT: s_mul_i32 s18, s7, s12
+; GFX10-NEXT: s_mul_hi_u32 s19, s7, s12
+; GFX10-NEXT: s_add_u32 s12, s14, s16
+; GFX10-NEXT: s_cselect_b32 s14, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s13, s7, s13
-; GFX10-NEXT: s_mul_hi_u32 s17, s6, s14
-; GFX10-NEXT: s_add_i32 s16, s16, s12
-; GFX10-NEXT: s_mul_i32 s15, s7, s14
-; GFX10-NEXT: s_add_u32 s12, s13, s17
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
; GFX10-NEXT: s_add_u32 s12, s12, s15
-; GFX10-NEXT: s_cselect_b32 s15, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s14, s7, s14
-; GFX10-NEXT: s_add_i32 s13, s13, s15
-; GFX10-NEXT: s_add_u32 s12, s12, s16
-; GFX10-NEXT: s_cselect_b32 s15, 1, 0
-; GFX10-NEXT: s_mul_i32 s16, s5, s12
-; GFX10-NEXT: s_add_i32 s13, s13, s15
-; GFX10-NEXT: s_add_i32 s13, s14, s13
+; GFX10-NEXT: s_addc_u32 s12, s14, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s17
+; GFX10-NEXT: s_cselect_b32 s14, 1, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s18
+; GFX10-NEXT: s_addc_u32 s14, s14, 0
+; GFX10-NEXT: s_add_u32 s12, s13, s12
+; GFX10-NEXT: s_addc_u32 s13, s14, 0
; GFX10-NEXT: s_mul_hi_u32 s14, s4, s12
+; GFX10-NEXT: s_add_i32 s13, s19, s13
+; GFX10-NEXT: s_mul_i32 s16, s5, s12
; GFX10-NEXT: s_mul_i32 s15, s4, s13
; GFX10-NEXT: s_add_i32 s14, s14, s15
; GFX10-NEXT: s_mul_i32 s15, s4, s12
@@ -1603,43 +1576,40 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_mul_hi_u32 v2, s16, v0
; GFX8-NEXT: v_readfirstlane_b32 s20, v0
-; GFX8-NEXT: s_mul_i32 s21, s16, s20
; GFX8-NEXT: v_readfirstlane_b32 s18, v1
-; GFX8-NEXT: v_mul_hi_u32 v3, v0, s21
+; GFX8-NEXT: s_mul_i32 s21, s16, s20
; GFX8-NEXT: s_mul_i32 s19, s16, s18
+; GFX8-NEXT: v_mul_hi_u32 v3, v0, s21
; GFX8-NEXT: v_readfirstlane_b32 s23, v2
; GFX8-NEXT: s_mul_i32 s22, s17, s20
; GFX8-NEXT: s_add_u32 s19, s19, s23
; GFX8-NEXT: s_add_u32 s19, s22, s19
-; GFX8-NEXT: v_readfirstlane_b32 s22, v3
-; GFX8-NEXT: s_mul_i32 s23, s20, s19
; GFX8-NEXT: v_mul_hi_u32 v2, v1, s21
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s19
+; GFX8-NEXT: v_readfirstlane_b32 s22, v3
+; GFX8-NEXT: s_mul_i32 s23, s20, s19
; GFX8-NEXT: s_mul_i32 s24, s18, s21
; GFX8-NEXT: s_add_u32 s22, s22, s23
; GFX8-NEXT: s_cselect_b32 s23, 1, 0
-; GFX8-NEXT: s_add_u32 s22, s22, s24
-; GFX8-NEXT: s_cselect_b32 s21, 1, 0
-; GFX8-NEXT: s_add_i32 s23, s23, s21
-; GFX8-NEXT: v_readfirstlane_b32 s21, v2
-; GFX8-NEXT: v_readfirstlane_b32 s22, v0
+; GFX8-NEXT: s_add_u32 s21, s22, s24
+; GFX8-NEXT: s_addc_u32 s21, s23, 0
+; GFX8-NEXT: v_readfirstlane_b32 s22, v2
+; GFX8-NEXT: v_readfirstlane_b32 s23, v0
; GFX8-NEXT: s_mul_i32 s24, s18, s19
-; GFX8-NEXT: s_add_u32 s21, s21, s22
-; GFX8-NEXT: s_cselect_b32 s22, 1, 0
-; GFX8-NEXT: s_add_u32 s21, s21, s24
-; GFX8-NEXT: s_cselect_b32 s24, 1, 0
+; GFX8-NEXT: s_add_u32 s22, s22, s23
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s19
-; GFX8-NEXT: s_add_i32 s22, s22, s24
-; GFX8-NEXT: s_add_u32 s19, s21, s23
-; GFX8-NEXT: s_cselect_b32 s21, 1, 0
-; GFX8-NEXT: s_add_i32 s22, s22, s21
-; GFX8-NEXT: v_readfirstlane_b32 s21, v0
-; GFX8-NEXT: s_add_i32 s21, s21, s22
-; GFX8-NEXT: s_add_u32 s19, s20, s19
+; GFX8-NEXT: s_cselect_b32 s23, 1, 0
+; GFX8-NEXT: s_add_u32 s22, s22, s24
+; GFX8-NEXT: s_addc_u32 s19, s23, 0
+; GFX8-NEXT: s_add_u32 s21, s22, s21
+; GFX8-NEXT: s_addc_u32 s19, s19, 0
+; GFX8-NEXT: v_readfirstlane_b32 s22, v0
+; GFX8-NEXT: s_add_i32 s22, s22, s19
+; GFX8-NEXT: s_add_u32 s19, s20, s21
; GFX8-NEXT: v_mov_b32_e32 v0, s19
; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
; GFX8-NEXT: s_mul_i32 s20, s16, s19
-; GFX8-NEXT: s_addc_u32 s18, s18, s21
+; GFX8-NEXT: s_addc_u32 s18, s18, s22
; GFX8-NEXT: s_mul_i32 s16, s16, s18
; GFX8-NEXT: v_readfirstlane_b32 s21, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
@@ -1649,27 +1619,24 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_u32 s16, s17, s16
; GFX8-NEXT: v_readfirstlane_b32 s17, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: s_mul_i32 s21, s19, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s18, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s19, v1
+; GFX8-NEXT: s_mul_i32 s21, s19, s16
; GFX8-NEXT: s_mul_i32 s20, s18, s20
; GFX8-NEXT: s_add_u32 s17, s17, s21
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
; GFX8-NEXT: s_add_u32 s17, s17, s20
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
-; GFX8-NEXT: s_add_i32 s21, s21, s17
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: v_readfirstlane_b32 s20, v2
+; GFX8-NEXT: s_addc_u32 s17, s21, 0
+; GFX8-NEXT: v_readfirstlane_b32 s20, v0
+; GFX8-NEXT: v_readfirstlane_b32 s21, v2
; GFX8-NEXT: s_mul_i32 s16, s18, s16
-; GFX8-NEXT: s_add_u32 s17, s17, s20
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: s_add_u32 s16, s17, s16
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_add_u32 s20, s20, s21
; GFX8-NEXT: v_mul_hi_u32 v0, s18, v1
-; GFX8-NEXT: s_add_i32 s17, s20, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s21
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s17, s20
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_add_u32 s16, s20, s16
+; GFX8-NEXT: s_addc_u32 s20, s21, 0
+; GFX8-NEXT: s_add_u32 s16, s16, s17
+; GFX8-NEXT: s_addc_u32 s17, s20, 0
; GFX8-NEXT: v_readfirstlane_b32 s20, v0
; GFX8-NEXT: s_add_i32 s20, s20, s17
; GFX8-NEXT: s_add_u32 s16, s19, s16
@@ -1677,31 +1644,28 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
; GFX8-NEXT: s_addc_u32 s17, s18, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: s_mul_i32 s19, s8, s17
+; GFX8-NEXT: v_mul_hi_u32 v2, s8, v1
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s9
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v2, s8, v1
+; GFX8-NEXT: s_mul_i32 s19, s8, s17
; GFX8-NEXT: s_mul_i32 s20, s9, s16
; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
-; GFX8-NEXT: s_add_u32 s18, s18, s20
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_i32 s19, s19, s16
-; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: s_add_u32 s16, s18, s20
+; GFX8-NEXT: s_addc_u32 s16, s19, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v0
+; GFX8-NEXT: v_readfirstlane_b32 s19, v2
; GFX8-NEXT: s_mul_i32 s17, s9, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s18
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_u32 s16, s16, s17
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s18, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s19
+; GFX8-NEXT: s_add_u32 s18, s18, s19
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_add_u32 s17, s18, s17
+; GFX8-NEXT: s_addc_u32 s18, s19, 0
+; GFX8-NEXT: s_add_u32 s16, s17, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s9, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s16
; GFX8-NEXT: v_mul_hi_u32 v1, s4, v1
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s17, s18
+; GFX8-NEXT: s_addc_u32 s17, s18, 0
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: s_add_i32 s17, s18, s17
; GFX8-NEXT: v_readfirstlane_b32 s19, v1
@@ -1792,34 +1756,31 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_mul_i32 s22, s19, s16
; GFX8-NEXT: s_add_u32 s21, s22, s21
; GFX8-NEXT: v_readfirstlane_b32 s22, v2
-; GFX8-NEXT: s_mul_i32 s23, s16, s21
; GFX8-NEXT: v_mul_hi_u32 v2, v1, s20
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s21
+; GFX8-NEXT: s_mul_i32 s23, s16, s21
; GFX8-NEXT: s_mul_i32 s24, s18, s20
; GFX8-NEXT: s_add_u32 s22, s22, s23
; GFX8-NEXT: s_cselect_b32 s23, 1, 0
-; GFX8-NEXT: s_add_u32 s22, s22, s24
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: s_add_i32 s23, s23, s20
-; GFX8-NEXT: v_readfirstlane_b32 s20, v2
-; GFX8-NEXT: v_readfirstlane_b32 s22, v0
+; GFX8-NEXT: s_add_u32 s20, s22, s24
+; GFX8-NEXT: s_addc_u32 s20, s23, 0
+; GFX8-NEXT: v_readfirstlane_b32 s22, v2
+; GFX8-NEXT: v_readfirstlane_b32 s23, v0
; GFX8-NEXT: s_mul_i32 s24, s18, s21
-; GFX8-NEXT: s_add_u32 s20, s20, s22
-; GFX8-NEXT: s_cselect_b32 s22, 1, 0
-; GFX8-NEXT: s_add_u32 s20, s20, s24
-; GFX8-NEXT: s_cselect_b32 s24, 1, 0
+; GFX8-NEXT: s_add_u32 s22, s22, s23
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s21
-; GFX8-NEXT: s_add_i32 s22, s22, s24
-; GFX8-NEXT: s_add_u32 s20, s20, s23
-; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_cselect_b32 s23, 1, 0
+; GFX8-NEXT: s_add_u32 s22, s22, s24
+; GFX8-NEXT: s_addc_u32 s21, s23, 0
+; GFX8-NEXT: s_add_u32 s20, s22, s20
+; GFX8-NEXT: s_addc_u32 s21, s21, 0
+; GFX8-NEXT: v_readfirstlane_b32 s22, v0
; GFX8-NEXT: s_add_i32 s22, s22, s21
-; GFX8-NEXT: v_readfirstlane_b32 s21, v0
-; GFX8-NEXT: s_add_i32 s21, s21, s22
; GFX8-NEXT: s_add_u32 s16, s16, s20
; GFX8-NEXT: v_mov_b32_e32 v0, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s17, v0
; GFX8-NEXT: s_mul_i32 s20, s17, s16
-; GFX8-NEXT: s_addc_u32 s18, s18, s21
+; GFX8-NEXT: s_addc_u32 s18, s18, s22
; GFX8-NEXT: s_mul_i32 s17, s17, s18
; GFX8-NEXT: v_readfirstlane_b32 s21, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
@@ -1829,27 +1790,24 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_u32 s17, s19, s17
; GFX8-NEXT: v_readfirstlane_b32 s19, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: s_mul_i32 s21, s16, s17
; GFX8-NEXT: v_mul_hi_u32 v0, s18, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s16, v1
+; GFX8-NEXT: s_mul_i32 s21, s16, s17
; GFX8-NEXT: s_mul_i32 s20, s18, s20
; GFX8-NEXT: s_add_u32 s19, s19, s21
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
; GFX8-NEXT: s_add_u32 s19, s19, s20
-; GFX8-NEXT: s_cselect_b32 s19, 1, 0
-; GFX8-NEXT: s_add_i32 s21, s21, s19
-; GFX8-NEXT: v_readfirstlane_b32 s19, v0
-; GFX8-NEXT: v_readfirstlane_b32 s20, v2
+; GFX8-NEXT: s_addc_u32 s19, s21, 0
+; GFX8-NEXT: v_readfirstlane_b32 s20, v0
+; GFX8-NEXT: v_readfirstlane_b32 s21, v2
; GFX8-NEXT: s_mul_i32 s17, s18, s17
-; GFX8-NEXT: s_add_u32 s19, s19, s20
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: s_add_u32 s17, s19, s17
-; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_add_u32 s20, s20, s21
; GFX8-NEXT: v_mul_hi_u32 v0, s18, v1
-; GFX8-NEXT: s_add_i32 s19, s20, s19
-; GFX8-NEXT: s_add_u32 s17, s17, s21
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: s_add_i32 s19, s19, s20
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_add_u32 s17, s20, s17
+; GFX8-NEXT: s_addc_u32 s20, s21, 0
+; GFX8-NEXT: s_add_u32 s17, s17, s19
+; GFX8-NEXT: s_addc_u32 s19, s20, 0
; GFX8-NEXT: v_readfirstlane_b32 s20, v0
; GFX8-NEXT: s_add_i32 s20, s20, s19
; GFX8-NEXT: s_add_u32 s16, s16, s17
@@ -1857,31 +1815,28 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
; GFX8-NEXT: s_addc_u32 s17, s18, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s17
-; GFX8-NEXT: s_mul_i32 s19, s10, s17
+; GFX8-NEXT: v_mul_hi_u32 v2, s10, v1
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s11
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v2, s10, v1
+; GFX8-NEXT: s_mul_i32 s19, s10, s17
; GFX8-NEXT: s_mul_i32 s20, s11, s16
; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
-; GFX8-NEXT: s_add_u32 s18, s18, s20
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_i32 s19, s19, s16
-; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: s_add_u32 s16, s18, s20
+; GFX8-NEXT: s_addc_u32 s16, s19, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v0
+; GFX8-NEXT: v_readfirstlane_b32 s19, v2
; GFX8-NEXT: s_mul_i32 s17, s11, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s18
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_u32 s16, s16, s17
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s18, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s19
+; GFX8-NEXT: s_add_u32 s18, s18, s19
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_add_u32 s17, s18, s17
+; GFX8-NEXT: s_addc_u32 s18, s19, 0
+; GFX8-NEXT: s_add_u32 s16, s17, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s11, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s16
; GFX8-NEXT: v_mul_hi_u32 v1, s6, v1
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s17, s18
+; GFX8-NEXT: s_addc_u32 s17, s18, 0
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: s_add_i32 s17, s18, s17
; GFX8-NEXT: v_readfirstlane_b32 s19, v1
@@ -1995,19 +1950,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s22, s22, s23
; GFX9-NEXT: s_cselect_b32 s23, 1, 0
; GFX9-NEXT: s_add_u32 s22, s22, s24
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_i32 s23, s23, s22
+; GFX9-NEXT: s_addc_u32 s22, s23, 0
; GFX9-NEXT: s_mul_hi_u32 s21, s18, s21
-; GFX9-NEXT: s_mul_hi_u32 s22, s20, s19
+; GFX9-NEXT: s_mul_hi_u32 s23, s20, s19
; GFX9-NEXT: s_mul_i32 s24, s18, s19
-; GFX9-NEXT: s_add_u32 s21, s21, s22
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_u32 s21, s21, s24
-; GFX9-NEXT: s_cselect_b32 s24, 1, 0
-; GFX9-NEXT: s_add_i32 s22, s22, s24
; GFX9-NEXT: s_add_u32 s21, s21, s23
; GFX9-NEXT: s_cselect_b32 s23, 1, 0
-; GFX9-NEXT: s_add_i32 s22, s22, s23
+; GFX9-NEXT: s_add_u32 s21, s21, s24
+; GFX9-NEXT: s_addc_u32 s23, s23, 0
+; GFX9-NEXT: s_add_u32 s21, s21, s22
+; GFX9-NEXT: s_addc_u32 s22, s23, 0
; GFX9-NEXT: s_mul_hi_u32 s19, s18, s19
; GFX9-NEXT: s_add_i32 s19, s19, s22
; GFX9-NEXT: s_add_u32 s20, s20, s21
@@ -2024,19 +1976,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s17, s17, s21
; GFX9-NEXT: s_cselect_b32 s21, 1, 0
; GFX9-NEXT: s_add_u32 s17, s17, s22
-; GFX9-NEXT: s_cselect_b32 s17, 1, 0
-; GFX9-NEXT: s_add_i32 s21, s21, s17
-; GFX9-NEXT: s_mul_hi_u32 s17, s18, s19
-; GFX9-NEXT: s_mul_hi_u32 s19, s20, s16
+; GFX9-NEXT: s_addc_u32 s17, s21, 0
+; GFX9-NEXT: s_mul_hi_u32 s19, s18, s19
+; GFX9-NEXT: s_mul_hi_u32 s21, s20, s16
; GFX9-NEXT: s_mul_i32 s22, s18, s16
-; GFX9-NEXT: s_add_u32 s17, s17, s19
-; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_u32 s17, s17, s22
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s22
-; GFX9-NEXT: s_add_u32 s17, s17, s21
+; GFX9-NEXT: s_add_u32 s19, s19, s21
; GFX9-NEXT: s_cselect_b32 s21, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s21
+; GFX9-NEXT: s_add_u32 s19, s19, s22
+; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: s_add_u32 s17, s19, s17
+; GFX9-NEXT: s_addc_u32 s19, s21, 0
; GFX9-NEXT: s_mul_hi_u32 s16, s18, s16
; GFX9-NEXT: s_add_i32 s16, s16, s19
; GFX9-NEXT: s_add_u32 s17, s20, s17
@@ -2047,19 +1996,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s16, s16, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
; GFX9-NEXT: s_add_u32 s16, s16, s20
-; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s16
-; GFX9-NEXT: s_mul_hi_u32 s16, s13, s17
-; GFX9-NEXT: s_mul_hi_u32 s17, s12, s18
+; GFX9-NEXT: s_addc_u32 s16, s19, 0
+; GFX9-NEXT: s_mul_hi_u32 s17, s13, s17
+; GFX9-NEXT: s_mul_hi_u32 s19, s12, s18
; GFX9-NEXT: s_mul_i32 s20, s13, s18
-; GFX9-NEXT: s_add_u32 s16, s16, s17
-; GFX9-NEXT: s_cselect_b32 s17, 1, 0
-; GFX9-NEXT: s_add_u32 s16, s16, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_i32 s17, s17, s20
-; GFX9-NEXT: s_add_u32 s16, s16, s19
+; GFX9-NEXT: s_add_u32 s17, s17, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s17, s17, s19
+; GFX9-NEXT: s_add_u32 s17, s17, s20
+; GFX9-NEXT: s_addc_u32 s19, s19, 0
+; GFX9-NEXT: s_add_u32 s16, s17, s16
+; GFX9-NEXT: s_addc_u32 s17, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s18, s13, s18
; GFX9-NEXT: s_add_i32 s17, s18, s17
; GFX9-NEXT: s_mul_hi_u32 s19, s4, s16
@@ -2153,19 +2099,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s22, s22, s23
; GFX9-NEXT: s_cselect_b32 s23, 1, 0
; GFX9-NEXT: s_add_u32 s22, s22, s24
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_i32 s23, s23, s22
+; GFX9-NEXT: s_addc_u32 s22, s23, 0
; GFX9-NEXT: s_mul_hi_u32 s20, s17, s20
-; GFX9-NEXT: s_mul_hi_u32 s22, s16, s21
+; GFX9-NEXT: s_mul_hi_u32 s23, s16, s21
; GFX9-NEXT: s_mul_i32 s24, s17, s21
-; GFX9-NEXT: s_add_u32 s20, s20, s22
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_u32 s20, s20, s24
-; GFX9-NEXT: s_cselect_b32 s24, 1, 0
-; GFX9-NEXT: s_add_i32 s22, s22, s24
; GFX9-NEXT: s_add_u32 s20, s20, s23
; GFX9-NEXT: s_cselect_b32 s23, 1, 0
-; GFX9-NEXT: s_add_i32 s22, s22, s23
+; GFX9-NEXT: s_add_u32 s20, s20, s24
+; GFX9-NEXT: s_addc_u32 s23, s23, 0
+; GFX9-NEXT: s_add_u32 s20, s20, s22
+; GFX9-NEXT: s_addc_u32 s22, s23, 0
; GFX9-NEXT: s_mul_hi_u32 s21, s17, s21
; GFX9-NEXT: s_add_i32 s21, s21, s22
; GFX9-NEXT: s_add_u32 s16, s16, s20
@@ -2182,19 +2125,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s19, s19, s21
; GFX9-NEXT: s_cselect_b32 s21, 1, 0
; GFX9-NEXT: s_add_u32 s19, s19, s22
-; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s21, s21, s19
-; GFX9-NEXT: s_mul_hi_u32 s19, s17, s20
-; GFX9-NEXT: s_mul_hi_u32 s20, s16, s18
+; GFX9-NEXT: s_addc_u32 s19, s21, 0
+; GFX9-NEXT: s_mul_hi_u32 s20, s17, s20
+; GFX9-NEXT: s_mul_hi_u32 s21, s16, s18
; GFX9-NEXT: s_mul_i32 s22, s17, s18
-; GFX9-NEXT: s_add_u32 s19, s19, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_u32 s19, s19, s22
-; GFX9-NEXT: s_cselect_b32 s22, 1, 0
-; GFX9-NEXT: s_add_i32 s20, s20, s22
-; GFX9-NEXT: s_add_u32 s19, s19, s21
+; GFX9-NEXT: s_add_u32 s20, s20, s21
; GFX9-NEXT: s_cselect_b32 s21, 1, 0
-; GFX9-NEXT: s_add_i32 s20, s20, s21
+; GFX9-NEXT: s_add_u32 s20, s20, s22
+; GFX9-NEXT: s_addc_u32 s21, s21, 0
+; GFX9-NEXT: s_add_u32 s19, s20, s19
+; GFX9-NEXT: s_addc_u32 s20, s21, 0
; GFX9-NEXT: s_mul_hi_u32 s18, s17, s18
; GFX9-NEXT: s_add_i32 s18, s18, s20
; GFX9-NEXT: s_add_u32 s16, s16, s19
@@ -2205,19 +2145,16 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s18, s18, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
; GFX9-NEXT: s_add_u32 s18, s18, s20
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s16, s15, s16
-; GFX9-NEXT: s_mul_hi_u32 s18, s14, s17
+; GFX9-NEXT: s_mul_hi_u32 s19, s14, s17
; GFX9-NEXT: s_mul_i32 s20, s15, s17
-; GFX9-NEXT: s_add_u32 s16, s16, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_u32 s16, s16, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s20
; GFX9-NEXT: s_add_u32 s16, s16, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s19
+; GFX9-NEXT: s_add_u32 s16, s16, s20
+; GFX9-NEXT: s_addc_u32 s19, s19, 0
+; GFX9-NEXT: s_add_u32 s16, s16, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s17, s15, s17
; GFX9-NEXT: s_add_i32 s17, s17, s18
; GFX9-NEXT: s_mul_hi_u32 s19, s6, s16
@@ -2330,21 +2267,18 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_mul_i32 s24, s18, s21
; GFX10-NEXT: s_add_u32 s22, s22, s23
; GFX10-NEXT: s_cselect_b32 s23, 1, 0
-; GFX10-NEXT: s_add_u32 s22, s22, s24
-; GFX10-NEXT: s_cselect_b32 s22, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s21, s18, s21
; GFX10-NEXT: s_mul_hi_u32 s25, s19, s20
-; GFX10-NEXT: s_add_i32 s23, s23, s22
-; GFX10-NEXT: s_mul_i32 s24, s18, s20
+; GFX10-NEXT: s_add_u32 s22, s22, s24
+; GFX10-NEXT: s_addc_u32 s22, s23, 0
+; GFX10-NEXT: s_mul_i32 s26, s18, s20
; GFX10-NEXT: s_add_u32 s21, s21, s25
-; GFX10-NEXT: s_cselect_b32 s22, 1, 0
-; GFX10-NEXT: s_add_u32 s21, s21, s24
-; GFX10-NEXT: s_cselect_b32 s24, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s20, s18, s20
-; GFX10-NEXT: s_add_i32 s22, s22, s24
-; GFX10-NEXT: s_add_u32 s21, s21, s23
; GFX10-NEXT: s_cselect_b32 s23, 1, 0
-; GFX10-NEXT: s_add_i32 s22, s22, s23
+; GFX10-NEXT: s_add_u32 s21, s21, s26
+; GFX10-NEXT: s_addc_u32 s23, s23, 0
+; GFX10-NEXT: s_add_u32 s21, s21, s22
+; GFX10-NEXT: s_mul_hi_u32 s20, s18, s20
+; GFX10-NEXT: s_addc_u32 s22, s23, 0
; GFX10-NEXT: s_add_i32 s20, s20, s22
; GFX10-NEXT: s_add_u32 s19, s19, s21
; GFX10-NEXT: s_addc_u32 s18, s18, s20
@@ -2362,44 +2296,38 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_add_u32 s20, s22, s20
; GFX10-NEXT: s_cselect_b32 s22, 1, 0
; GFX10-NEXT: s_add_u32 s20, s20, s21
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_mul_i32 s21, s18, s16
-; GFX10-NEXT: s_add_i32 s22, s22, s20
+; GFX10-NEXT: s_addc_u32 s20, s22, 0
+; GFX10-NEXT: s_mul_i32 s24, s18, s16
; GFX10-NEXT: s_add_u32 s17, s17, s23
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_add_u32 s17, s17, s21
; GFX10-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s24
+; GFX10-NEXT: s_addc_u32 s21, s21, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s20
; GFX10-NEXT: s_mul_hi_u32 s16, s18, s16
-; GFX10-NEXT: s_add_i32 s20, s20, s21
-; GFX10-NEXT: s_add_u32 s17, s17, s22
-; GFX10-NEXT: s_cselect_b32 s21, 1, 0
-; GFX10-NEXT: s_add_i32 s20, s20, s21
+; GFX10-NEXT: s_addc_u32 s20, s21, 0
; GFX10-NEXT: s_add_i32 s16, s16, s20
; GFX10-NEXT: s_add_u32 s17, s19, s17
-; GFX10-NEXT: s_addc_u32 s18, s18, s16
-; GFX10-NEXT: s_mul_hi_u32 s16, s12, s17
-; GFX10-NEXT: s_mul_i32 s20, s12, s18
+; GFX10-NEXT: s_addc_u32 s16, s18, s16
+; GFX10-NEXT: s_mul_hi_u32 s18, s12, s17
+; GFX10-NEXT: s_mul_i32 s20, s12, s16
; GFX10-NEXT: s_mul_i32 s19, s13, s17
-; GFX10-NEXT: s_add_u32 s16, s16, s20
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_add_u32 s16, s16, s19
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s21, s12, s16
+; GFX10-NEXT: s_mul_i32 s22, s13, s16
+; GFX10-NEXT: s_mul_hi_u32 s23, s13, s16
+; GFX10-NEXT: s_add_u32 s16, s18, s20
+; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s17, s13, s17
-; GFX10-NEXT: s_mul_hi_u32 s21, s12, s18
-; GFX10-NEXT: s_add_i32 s20, s20, s16
-; GFX10-NEXT: s_mul_i32 s19, s13, s18
-; GFX10-NEXT: s_add_u32 s16, s17, s21
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
; GFX10-NEXT: s_add_u32 s16, s16, s19
-; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s18, s13, s18
-; GFX10-NEXT: s_add_i32 s17, s17, s19
-; GFX10-NEXT: s_add_u32 s16, s16, s20
-; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_mul_i32 s20, s5, s16
-; GFX10-NEXT: s_add_i32 s17, s17, s19
-; GFX10-NEXT: s_add_i32 s17, s18, s17
+; GFX10-NEXT: s_addc_u32 s16, s18, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s21
+; GFX10-NEXT: s_cselect_b32 s18, 1, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s22
+; GFX10-NEXT: s_addc_u32 s18, s18, 0
+; GFX10-NEXT: s_add_u32 s16, s17, s16
+; GFX10-NEXT: s_addc_u32 s17, s18, 0
; GFX10-NEXT: s_mul_hi_u32 s18, s4, s16
+; GFX10-NEXT: s_add_i32 s17, s23, s17
+; GFX10-NEXT: s_mul_i32 s20, s5, s16
; GFX10-NEXT: s_mul_i32 s19, s4, s17
; GFX10-NEXT: s_add_i32 s18, s18, s19
; GFX10-NEXT: s_mul_i32 s19, s4, s16
@@ -2488,21 +2416,18 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_mul_i32 s24, s18, s21
; GFX10-NEXT: s_add_u32 s22, s22, s23
; GFX10-NEXT: s_cselect_b32 s23, 1, 0
-; GFX10-NEXT: s_add_u32 s22, s22, s24
-; GFX10-NEXT: s_cselect_b32 s22, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s21, s18, s21
; GFX10-NEXT: s_mul_hi_u32 s25, s19, s20
-; GFX10-NEXT: s_add_i32 s23, s23, s22
-; GFX10-NEXT: s_mul_i32 s24, s18, s20
+; GFX10-NEXT: s_add_u32 s22, s22, s24
+; GFX10-NEXT: s_addc_u32 s22, s23, 0
+; GFX10-NEXT: s_mul_i32 s26, s18, s20
; GFX10-NEXT: s_add_u32 s21, s21, s25
-; GFX10-NEXT: s_cselect_b32 s22, 1, 0
-; GFX10-NEXT: s_add_u32 s21, s21, s24
-; GFX10-NEXT: s_cselect_b32 s24, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s20, s18, s20
-; GFX10-NEXT: s_add_i32 s22, s22, s24
-; GFX10-NEXT: s_add_u32 s21, s21, s23
; GFX10-NEXT: s_cselect_b32 s23, 1, 0
-; GFX10-NEXT: s_add_i32 s22, s22, s23
+; GFX10-NEXT: s_add_u32 s21, s21, s26
+; GFX10-NEXT: s_addc_u32 s23, s23, 0
+; GFX10-NEXT: s_add_u32 s21, s21, s22
+; GFX10-NEXT: s_mul_hi_u32 s20, s18, s20
+; GFX10-NEXT: s_addc_u32 s22, s23, 0
; GFX10-NEXT: s_add_i32 s20, s20, s22
; GFX10-NEXT: s_add_u32 s19, s19, s21
; GFX10-NEXT: s_addc_u32 s18, s18, s20
@@ -2520,44 +2445,38 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_add_u32 s20, s22, s20
; GFX10-NEXT: s_cselect_b32 s22, 1, 0
; GFX10-NEXT: s_add_u32 s20, s20, s21
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_mul_i32 s21, s18, s16
-; GFX10-NEXT: s_add_i32 s22, s22, s20
+; GFX10-NEXT: s_addc_u32 s20, s22, 0
+; GFX10-NEXT: s_mul_i32 s24, s18, s16
; GFX10-NEXT: s_add_u32 s17, s17, s23
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_add_u32 s17, s17, s21
; GFX10-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s24
+; GFX10-NEXT: s_addc_u32 s21, s21, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s20
; GFX10-NEXT: s_mul_hi_u32 s16, s18, s16
-; GFX10-NEXT: s_add_i32 s20, s20, s21
-; GFX10-NEXT: s_add_u32 s17, s17, s22
-; GFX10-NEXT: s_cselect_b32 s21, 1, 0
-; GFX10-NEXT: s_add_i32 s20, s20, s21
+; GFX10-NEXT: s_addc_u32 s20, s21, 0
; GFX10-NEXT: s_add_i32 s16, s16, s20
; GFX10-NEXT: s_add_u32 s17, s19, s17
-; GFX10-NEXT: s_addc_u32 s18, s18, s16
-; GFX10-NEXT: s_mul_hi_u32 s16, s14, s17
-; GFX10-NEXT: s_mul_i32 s20, s14, s18
+; GFX10-NEXT: s_addc_u32 s16, s18, s16
+; GFX10-NEXT: s_mul_hi_u32 s18, s14, s17
+; GFX10-NEXT: s_mul_i32 s20, s14, s16
; GFX10-NEXT: s_mul_i32 s19, s15, s17
-; GFX10-NEXT: s_add_u32 s16, s16, s20
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_add_u32 s16, s16, s19
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s21, s14, s16
+; GFX10-NEXT: s_mul_i32 s22, s15, s16
+; GFX10-NEXT: s_mul_hi_u32 s23, s15, s16
+; GFX10-NEXT: s_add_u32 s16, s18, s20
+; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s17, s15, s17
-; GFX10-NEXT: s_mul_hi_u32 s21, s14, s18
-; GFX10-NEXT: s_add_i32 s20, s20, s16
-; GFX10-NEXT: s_mul_i32 s19, s15, s18
-; GFX10-NEXT: s_add_u32 s16, s17, s21
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
; GFX10-NEXT: s_add_u32 s16, s16, s19
-; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s18, s15, s18
-; GFX10-NEXT: s_add_i32 s17, s17, s19
-; GFX10-NEXT: s_add_u32 s16, s16, s20
-; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_mul_i32 s20, s7, s16
-; GFX10-NEXT: s_add_i32 s17, s17, s19
-; GFX10-NEXT: s_add_i32 s17, s18, s17
+; GFX10-NEXT: s_addc_u32 s16, s18, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s21
+; GFX10-NEXT: s_cselect_b32 s18, 1, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s22
+; GFX10-NEXT: s_addc_u32 s18, s18, 0
+; GFX10-NEXT: s_add_u32 s16, s17, s16
+; GFX10-NEXT: s_addc_u32 s17, s18, 0
; GFX10-NEXT: s_mul_hi_u32 s18, s6, s16
+; GFX10-NEXT: s_add_i32 s17, s23, s17
+; GFX10-NEXT: s_mul_i32 s20, s7, s16
; GFX10-NEXT: s_mul_i32 s19, s6, s17
; GFX10-NEXT: s_add_i32 s18, s18, s19
; GFX10-NEXT: s_mul_i32 s19, s6, s16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 0d413f0070136d8..fc57fafa1d78de0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -53,17 +53,14 @@ define i64 @v_srem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v9
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v14
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CHECK-NEXT: v_mul_hi_u32 v7, v10, v8
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
; CHECK-NEXT: v_add_i32_e32 v11, vcc, v11, v2
; CHECK-NEXT: v_addc_u32_e32 v10, vcc, v10, v3, vcc
@@ -79,21 +76,18 @@ define i64 @v_srem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v3, v10, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v10, v2
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_mul_hi_u32 v7, v11, v8
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v5, v11, v8
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
-; CHECK-NEXT: v_mul_lo_u32 v6, v10, v8
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_mul_lo_u32 v5, v10, v8
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v6, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_mul_hi_u32 v6, v10, v8
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v11, v2
; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc
@@ -103,23 +97,20 @@ define i64 @v_srem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v4, v8, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v8, v2
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_mul_hi_u32 v7, v9, v3
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v5, v9, v3
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v6, v8, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_mul_lo_u32 v5, v8, v3
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v2, v4
-; CHECK-NEXT: v_mul_hi_u32 v6, v8, v3
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v8, v3
; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v10, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v4
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v5, v4
; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v6, v[3:4]
; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v9, v2
; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v10, v[4:5]
@@ -228,34 +219,31 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_mul_i32 s14, s5, s11
; CHECK-NEXT: s_add_u32 s12, s14, s12
; CHECK-NEXT: v_readfirstlane_b32 s14, v2
-; CHECK-NEXT: s_mul_i32 s15, s11, s12
; CHECK-NEXT: v_mul_hi_u32 v2, v1, s13
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s12
+; CHECK-NEXT: s_mul_i32 s15, s11, s12
; CHECK-NEXT: s_mul_i32 s16, s10, s13
; CHECK-NEXT: s_add_u32 s14, s14, s15
; CHECK-NEXT: s_cselect_b32 s15, 1, 0
-; CHECK-NEXT: s_add_u32 s14, s14, s16
-; CHECK-NEXT: s_cselect_b32 s13, 1, 0
-; CHECK-NEXT: s_add_i32 s15, s15, s13
-; CHECK-NEXT: v_readfirstlane_b32 s13, v2
-; CHECK-NEXT: v_readfirstlane_b32 s14, v0
+; CHECK-NEXT: s_add_u32 s13, s14, s16
+; CHECK-NEXT: s_addc_u32 s13, s15, 0
+; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: v_readfirstlane_b32 s15, v0
; CHECK-NEXT: s_mul_i32 s16, s10, s12
-; CHECK-NEXT: s_add_u32 s13, s13, s14
-; CHECK-NEXT: s_cselect_b32 s14, 1, 0
-; CHECK-NEXT: s_add_u32 s13, s13, s16
-; CHECK-NEXT: s_cselect_b32 s16, 1, 0
+; CHECK-NEXT: s_add_u32 s14, s14, s15
; CHECK-NEXT: v_mul_hi_u32 v0, v1, s12
-; CHECK-NEXT: s_add_i32 s14, s14, s16
-; CHECK-NEXT: s_add_u32 s12, s13, s15
-; CHECK-NEXT: s_cselect_b32 s13, 1, 0
-; CHECK-NEXT: s_add_i32 s14, s14, s13
-; CHECK-NEXT: v_readfirstlane_b32 s13, v0
-; CHECK-NEXT: s_add_i32 s13, s13, s14
-; CHECK-NEXT: s_add_u32 s11, s11, s12
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_u32 s14, s14, s16
+; CHECK-NEXT: s_addc_u32 s12, s15, 0
+; CHECK-NEXT: s_add_u32 s13, s14, s13
+; CHECK-NEXT: s_addc_u32 s12, s12, 0
+; CHECK-NEXT: v_readfirstlane_b32 s14, v0
+; CHECK-NEXT: s_add_i32 s14, s14, s12
+; CHECK-NEXT: s_add_u32 s11, s11, s13
; CHECK-NEXT: v_mov_b32_e32 v0, s11
; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
; CHECK-NEXT: s_mul_i32 s12, s3, s11
-; CHECK-NEXT: s_addc_u32 s10, s10, s13
+; CHECK-NEXT: s_addc_u32 s10, s10, s14
; CHECK-NEXT: s_mul_i32 s3, s3, s10
; CHECK-NEXT: v_readfirstlane_b32 s13, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s12
@@ -265,27 +253,24 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s3, s5, s3
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: s_mul_i32 s13, s11, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
; CHECK-NEXT: v_mul_hi_u32 v2, s11, v1
+; CHECK-NEXT: s_mul_i32 s13, s11, s3
; CHECK-NEXT: s_mul_i32 s12, s10, s12
; CHECK-NEXT: s_add_u32 s5, s5, s13
; CHECK-NEXT: s_cselect_b32 s13, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s12
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s13, s13, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: s_addc_u32 s5, s13, 0
+; CHECK-NEXT: v_readfirstlane_b32 s12, v0
+; CHECK-NEXT: v_readfirstlane_b32 s13, v2
; CHECK-NEXT: s_mul_i32 s3, s10, s3
-; CHECK-NEXT: s_add_u32 s5, s5, s12
-; CHECK-NEXT: s_cselect_b32 s12, 1, 0
-; CHECK-NEXT: s_add_u32 s3, s5, s3
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_u32 s12, s12, s13
; CHECK-NEXT: v_mul_hi_u32 v0, s10, v1
-; CHECK-NEXT: s_add_i32 s5, s12, s5
-; CHECK-NEXT: s_add_u32 s3, s3, s13
-; CHECK-NEXT: s_cselect_b32 s12, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s12
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s12, s3
+; CHECK-NEXT: s_addc_u32 s12, s13, 0
+; CHECK-NEXT: s_add_u32 s3, s3, s5
+; CHECK-NEXT: s_addc_u32 s5, s12, 0
; CHECK-NEXT: v_readfirstlane_b32 s12, v0
; CHECK-NEXT: s_add_i32 s12, s12, s5
; CHECK-NEXT: s_add_u32 s3, s11, s3
@@ -293,31 +278,28 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mul_hi_u32 v0, s8, v0
; CHECK-NEXT: s_addc_u32 s5, s10, s12
; CHECK-NEXT: v_mov_b32_e32 v1, s5
-; CHECK-NEXT: s_mul_i32 s11, s8, s5
+; CHECK-NEXT: v_mul_hi_u32 v2, s8, v1
; CHECK-NEXT: v_readfirstlane_b32 s10, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s9
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
-; CHECK-NEXT: v_mul_hi_u32 v2, s8, v1
+; CHECK-NEXT: s_mul_i32 s11, s8, s5
; CHECK-NEXT: s_mul_i32 s12, s9, s3
; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: s_cselect_b32 s11, 1, 0
-; CHECK-NEXT: s_add_u32 s10, s10, s12
-; CHECK-NEXT: s_cselect_b32 s3, 1, 0
-; CHECK-NEXT: s_add_i32 s11, s11, s3
-; CHECK-NEXT: v_readfirstlane_b32 s3, v0
-; CHECK-NEXT: v_readfirstlane_b32 s10, v2
+; CHECK-NEXT: s_add_u32 s3, s10, s12
+; CHECK-NEXT: s_addc_u32 s3, s11, 0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v0
+; CHECK-NEXT: v_readfirstlane_b32 s11, v2
; CHECK-NEXT: s_mul_i32 s5, s9, s5
-; CHECK-NEXT: s_add_u32 s3, s3, s10
-; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: s_add_u32 s3, s3, s5
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s10, s5
-; CHECK-NEXT: s_add_u32 s3, s3, s11
+; CHECK-NEXT: s_add_u32 s10, s10, s11
+; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s10, s5
+; CHECK-NEXT: s_addc_u32 s10, s11, 0
+; CHECK-NEXT: s_add_u32 s3, s5, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s9, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: v_mul_hi_u32 v1, s6, v1
-; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s10
+; CHECK-NEXT: s_addc_u32 s5, s10, 0
; CHECK-NEXT: v_readfirstlane_b32 s10, v0
; CHECK-NEXT: s_add_i32 s10, s10, s5
; CHECK-NEXT: v_readfirstlane_b32 s11, v1
@@ -439,22 +421,19 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v12, v15, v9
; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
; GISEL-NEXT: v_mul_lo_u32 v11, v4, v13
+; GISEL-NEXT: v_mul_hi_u32 v14, v4, v13
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v4, v13
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v14
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_mul_hi_u32 v12, v15, v13
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
; GISEL-NEXT: v_add_i32_e32 v18, vcc, v4, v9
; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v15, v10, vcc
@@ -469,22 +448,19 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v0, v15, v9
; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
; GISEL-NEXT: v_mul_lo_u32 v11, v18, v13
+; GISEL-NEXT: v_mul_hi_u32 v12, v18, v13
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v18, v13
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v10, v15, v13
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v18, v0
; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v15, v9, vcc
@@ -494,154 +470,142 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v1, v13, v0
; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, v9
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v1, vcc, v10, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, v9
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
-; GISEL-NEXT: v_mul_lo_u32 v11, v13, v9
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_mul_lo_u32 v10, v13, v9
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v0, v1
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v7
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v7, v11, vcc
+; GISEL-NEXT: v_xor_b32_e32 v6, v6, v11
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v11
; GISEL-NEXT: v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v7
; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v15, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v9, v10
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v11, v[1:2]
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v9, v10
+; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v16, v[1:2]
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v11
+; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v14, v0
; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v8, v15, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v14, v0
-; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], v13, v11, vcc
-; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v13, v11
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v5
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v12, v8
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v10, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v1, v9, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e64 v14, s[4:5], 0, v0, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v7
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v1
-; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v7, v1, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v1
-; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v7
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
-; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v15, v15, v16, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v0, v8, vcc
; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v8, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v8
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v7, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v20, v[1:2]
-; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
-; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v13, v5
-; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v16, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v15
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
-; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
-; GISEL-NEXT: v_mul_hi_u32 v9, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT: v_mul_lo_u32 v8, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v23, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v14, v9
-; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v20, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc
+; GISEL-NEXT: v_trunc_f32_e32 v1, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v0
+; GISEL-NEXT: v_sub_i32_e64 v17, s[4:5], 0, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v1
+; GISEL-NEXT: v_subb_u32_e64 v18, s[4:5], 0, v7, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_subb_u32_e64 v19, s[4:5], v13, v11, vcc
+; GISEL-NEXT: v_sub_i32_e64 v13, s[4:5], v13, v11
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v16, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v15, v0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v19, v8
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v18, v15, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v9, v15, v11
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v9
+; GISEL-NEXT: v_mul_lo_u32 v9, v16, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v14, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v19, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v1, v9, s[6:7]
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v13, v8, vcc
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v14, v5
+; GISEL-NEXT: v_subbrev_u32_e64 v13, s[6:7], 0, v1, vcc
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v12, v5
+; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v12, v5
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v13, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v21, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v13, v8
+; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v1, v15, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v20, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, v21, s[6:7]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v20
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_mul_lo_u32 v1, v16, v11
; GISEL-NEXT: v_cndmask_b32_e32 v5, v12, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v15, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v10, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v12, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v19, v14, v[8:9]
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v0
-; GISEL-NEXT: v_xor_b32_e32 v11, v2, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v11
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], 0, v10, s[4:5]
+; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v12, s[6:7]
+; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v1
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], 0, v10, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v11, v1
+; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v15, v0
+; GISEL-NEXT: v_addc_u32_e64 v15, s[4:5], v16, v1, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v12, 0
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v13, v8, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_ashrrev_i32_e32 v13, 31, v3
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v15, v[1:2]
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v19, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v13
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v18, v12, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v11, v2, v13
; GISEL-NEXT: v_mul_lo_u32 v2, v15, v0
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, v10
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v12, vcc
+; GISEL-NEXT: v_mul_lo_u32 v9, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v13, vcc
; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mul_hi_u32 v14, v12, v10
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v10
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_mul_lo_u32 v9, v15, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_mul_lo_u32 v8, v15, v10
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v14
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_mul_hi_u32 v9, v15, v10
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v8, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v15, v2, vcc
; GISEL-NEXT: v_mul_hi_u32 v8, v11, v0
; GISEL-NEXT: v_mul_lo_u32 v9, v11, v2
-; GISEL-NEXT: v_xor_b32_e32 v10, v3, v12
+; GISEL-NEXT: v_xor_b32_e32 v10, v3, v13
; GISEL-NEXT: v_mul_lo_u32 v3, v10, v0
; GISEL-NEXT: v_mul_hi_u32 v0, v10, v0
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mul_hi_u32 v12, v11, v2
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v8, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v11, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
-; GISEL-NEXT: v_mul_lo_u32 v9, v10, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_mul_lo_u32 v8, v10, v2
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v9, v10, v2
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v0, v3
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v8, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v10, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
+; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v0, v[3:4]
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v13, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v5, v4
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v12, v[8:9]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v3, vcc
; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v10, v3
@@ -669,10 +633,10 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v12
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v12
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v12
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v12, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v13
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v13
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v13
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v13, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64:
@@ -719,17 +683,14 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v18
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_mul_hi_u32 v5, v14, v12
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v2
; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v3, vcc
@@ -745,21 +706,18 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_addc_u32_e32 v4, vcc, v11, v16, vcc
; CGP-NEXT: v_mul_hi_u32 v2, v14, v2
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_mul_hi_u32 v11, v15, v12
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v5, v15, v12
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v10, v3
-; CGP-NEXT: v_mul_lo_u32 v10, v14, v12
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_mul_lo_u32 v5, v14, v12
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
; CGP-NEXT: v_mul_hi_u32 v10, v14, v12
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v10, v3
; CGP-NEXT: v_add_i32_e32 v2, vcc, v15, v2
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v14, v3, vcc
@@ -769,23 +727,20 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v4, v12, v2
; CGP-NEXT: v_mul_hi_u32 v2, v12, v2
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_mul_hi_u32 v11, v13, v3
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v5, v13, v3
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v10, v4
-; CGP-NEXT: v_mul_lo_u32 v10, v12, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_mul_lo_u32 v5, v12, v3
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v2, v4
-; CGP-NEXT: v_mul_hi_u32 v10, v12, v3
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v5, v12, v3
; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v14, 0
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v5, v4
; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v10, v[3:4]
; CGP-NEXT: v_sub_i32_e32 v2, vcc, v13, v2
; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v1, v14, v[4:5]
@@ -888,17 +843,14 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v16
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CGP-NEXT: v_mul_hi_u32 v7, v12, v10
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v4
; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v5, vcc
@@ -914,21 +866,18 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_addc_u32_e32 v6, vcc, v9, v14, vcc
; CGP-NEXT: v_mul_hi_u32 v4, v12, v4
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_mul_hi_u32 v9, v13, v10
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v7, v13, v10
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; CGP-NEXT: v_mul_lo_u32 v8, v12, v10
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v7, v12, v10
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CGP-NEXT: v_mul_hi_u32 v8, v12, v10
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v4
; CGP-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc
@@ -938,23 +887,20 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v6, v10, v4
; CGP-NEXT: v_mul_hi_u32 v4, v10, v4
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_mul_hi_u32 v9, v11, v5
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v7, v11, v5
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
-; CGP-NEXT: v_mul_lo_u32 v8, v10, v5
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v7, v10, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v6
-; CGP-NEXT: v_mul_hi_u32 v8, v10, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v10, v5
; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v12, 0
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v6
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v7, v6
; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v2, v8, v[5:6]
; CGP-NEXT: v_sub_i32_e32 v4, vcc, v11, v4
; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v12, v[6:7]
@@ -1076,24 +1022,24 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
; CHECK-NEXT: v_mov_b32_e32 v4, 0x4f800000
-; CHECK-NEXT: v_mov_b32_e32 v5, 0xffed2705
; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_mac_f32_e32 v3, 0, v4
; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; CHECK-NEXT: v_trunc_f32_e32 v4, v4
; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
-; CHECK-NEXT: v_mul_hi_u32 v6, v5, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
+; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
; CHECK-NEXT: v_readfirstlane_b32 s4, v3
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s5, v4
+; CHECK-NEXT: v_readfirstlane_b32 s5, v0
; CHECK-NEXT: v_readfirstlane_b32 s7, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
@@ -1101,113 +1047,104 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: v_readfirstlane_b32 s8, v6
-; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v6, v4, s6
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: s_mul_i32 s10, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_u32 s8, s8, s10
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v6
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_add_u32 s6, s8, s10
+; CHECK-NEXT: s_addc_u32 s6, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v6
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s10, s5, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s10
-; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v3, v4, s7
-; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: s_add_u32 s6, s6, s9
-; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s10
+; CHECK-NEXT: s_addc_u32 s7, s9, 0
+; CHECK-NEXT: s_add_u32 s6, s8, s6
+; CHECK-NEXT: s_addc_u32 s7, s7, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s7, v3
-; CHECK-NEXT: s_add_i32 s7, s7, s8
; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: v_mul_hi_u32 v3, v5, s4
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: s_addc_u32 s5, s5, s7
+; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
+; CHECK-NEXT: s_addc_u32 s5, s5, s8
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s7, v3
-; CHECK-NEXT: v_mov_b32_e32 v3, s6
-; CHECK-NEXT: v_mul_hi_u32 v4, s4, v3
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s8, v4
-; CHECK-NEXT: v_mov_b32_e32 v4, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
+; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v3, s5, v3
-; CHECK-NEXT: v_mul_hi_u32 v5, s4, v4
; CHECK-NEXT: s_mul_i32 s6, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s6, s8, s6
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v3
-; CHECK-NEXT: v_readfirstlane_b32 s8, v5
+; CHECK-NEXT: s_addc_u32 s6, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s7, s5, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s7
-; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v3, s5, v4
-; CHECK-NEXT: s_add_i32 s7, s8, s7
-; CHECK-NEXT: s_add_u32 s6, s6, s9
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_i32 s7, s7, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_addc_u32 s8, s9, 0
+; CHECK-NEXT: s_add_u32 s6, s7, s6
+; CHECK-NEXT: s_addc_u32 s7, s8, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s7
; CHECK-NEXT: s_add_u32 s4, s4, s6
; CHECK-NEXT: s_addc_u32 s5, s5, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v0, s4
-; CHECK-NEXT: v_mul_lo_u32 v4, v0, s5
-; CHECK-NEXT: v_mul_lo_u32 v5, v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v6, v0, s5
-; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
+; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
+; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v4, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v8, v5, s5
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v8
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v5, v1, s4
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_mul_lo_u32 v4, v1, s5
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v4, v3
-; CHECK-NEXT: v_mul_hi_u32 v9, v1, s5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v6, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v5
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[4:5]
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v1, v5, vcc
-; CHECK-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v5
-; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v1, v0
+; CHECK-NEXT: v_mul_hi_u32 v8, v6, s5
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v4, 0
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v3
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v8, v[1:2]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
+; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
+; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
; CHECK-NEXT: v_sub_i32_e32 v5, vcc, v0, v7
-; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
; CHECK-NEXT: v_sub_i32_e32 v7, vcc, v5, v7
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; CHECK-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v3, vcc
; CHECK-NEXT: v_cndmask_b32_e64 v4, -1, v4, s[4:5]
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
; CHECK-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1234,162 +1171,150 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_trunc_f32_e32 v6, v6
; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_xor_b32_e32 v14, v0, v4
; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v7
-; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
-; GISEL-NEXT: v_readfirstlane_b32 s6, v5
-; GISEL-NEXT: v_readfirstlane_b32 s9, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, v7
+; GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; GISEL-NEXT: s_mul_i32 s5, s6, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s7, v5
+; GISEL-NEXT: v_readfirstlane_b32 s9, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v7, s5
; GISEL-NEXT: s_subb_u32 s4, 0, 0
-; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
+; GISEL-NEXT: s_mul_i32 s10, s7, 0xffed2705
; GISEL-NEXT: s_add_u32 s9, s10, s9
-; GISEL-NEXT: s_mul_i32 s10, s4, s7
+; GISEL-NEXT: s_mul_i32 s10, s4, s6
; GISEL-NEXT: s_add_u32 s10, s10, s9
-; GISEL-NEXT: v_readfirstlane_b32 s11, v8
-; GISEL-NEXT: s_mul_i32 s12, s7, s10
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
-; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
-; GISEL-NEXT: s_mul_i32 s13, s6, s5
+; GISEL-NEXT: v_readfirstlane_b32 s11, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v8, v7, s10
+; GISEL-NEXT: s_mul_i32 s12, s6, s10
+; GISEL-NEXT: s_mul_i32 s13, s7, s5
; GISEL-NEXT: s_add_u32 s12, s11, s12
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: s_add_u32 s12, s12, s13
-; GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GISEL-NEXT: s_add_i32 s14, s14, s5
-; GISEL-NEXT: v_readfirstlane_b32 s12, v8
-; GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GISEL-NEXT: s_mul_i32 s15, s6, s10
-; GISEL-NEXT: s_add_u32 s5, s12, s5
+; GISEL-NEXT: s_add_u32 s5, s12, s13
+; GISEL-NEXT: s_addc_u32 s5, s14, 0
+; GISEL-NEXT: v_readfirstlane_b32 s12, v6
+; GISEL-NEXT: v_readfirstlane_b32 s14, v8
+; GISEL-NEXT: s_mul_i32 s15, s7, s10
+; GISEL-NEXT: s_add_u32 s14, s12, s14
+; GISEL-NEXT: v_mul_hi_u32 v6, v5, s10
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
-; GISEL-NEXT: s_add_u32 s5, s5, s15
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
-; GISEL-NEXT: s_add_i32 s15, s16, s15
-; GISEL-NEXT: s_add_u32 s5, s5, s14
-; GISEL-NEXT: s_cselect_b32 s10, 1, 0
-; GISEL-NEXT: s_add_i32 s15, s15, s10
-; GISEL-NEXT: v_readfirstlane_b32 s10, v8
-; GISEL-NEXT: s_add_i32 s10, s10, s15
-; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: s_add_u32 s14, s14, s15
+; GISEL-NEXT: s_addc_u32 s10, s16, 0
+; GISEL-NEXT: s_add_u32 s5, s14, s5
+; GISEL-NEXT: s_addc_u32 s10, s10, 0
+; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: s_add_i32 s14, s14, s10
+; GISEL-NEXT: s_add_u32 s5, s6, s5
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, s5
+; GISEL-NEXT: s_addc_u32 s10, s7, s14
; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
-; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
+; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
; GISEL-NEXT: v_mov_b32_e32 v6, s14
-; GISEL-NEXT: s_addc_u32 s10, s6, s10
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
-; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
; GISEL-NEXT: s_add_u32 s15, s16, s15
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s15
; GISEL-NEXT: v_readfirstlane_b32 s15, v8
; GISEL-NEXT: v_mov_b32_e32 v8, s4
-; GISEL-NEXT: s_mul_i32 s16, s5, s4
; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: s_mul_i32 s16, s5, s4
; GISEL-NEXT: s_mul_i32 s14, s10, s14
; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s14, s15, s14
-; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: s_add_i32 s16, s16, s14
-; GISEL-NEXT: v_readfirstlane_b32 s14, v6
-; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: s_addc_u32 s14, s16, 0
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: v_readfirstlane_b32 s16, v9
; GISEL-NEXT: s_mul_i32 s4, s10, s4
-; GISEL-NEXT: s_add_u32 s14, s14, s15
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: s_add_u32 s4, s14, s4
-; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
-; GISEL-NEXT: s_add_i32 s14, s15, s14
-; GISEL-NEXT: s_add_u32 s4, s4, s16
-; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: s_add_i32 s14, s14, s15
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s15, s4
+; GISEL-NEXT: s_addc_u32 s15, s16, 0
+; GISEL-NEXT: s_add_u32 s4, s4, s14
+; GISEL-NEXT: s_addc_u32 s14, s15, 0
; GISEL-NEXT: v_readfirstlane_b32 s15, v6
; GISEL-NEXT: s_add_i32 s15, s15, s14
; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
; GISEL-NEXT: s_addc_u32 s5, s10, s15
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
-; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
-; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, s4
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
+; GISEL-NEXT: v_mul_lo_u32 v10, v1, s4
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s4
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
+; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
-; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v11, v10
; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v12, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v14, v8
; GISEL-NEXT: s_cmp_lg_u32 s8, 0
; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
-; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v1, v12, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v12
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v6
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9
; GISEL-NEXT: v_cndmask_b32_e64 v10, -1, v10, s[4:5]
; GISEL-NEXT: s_subb_u32 s4, 0, 0
-; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: s_mul_i32 s5, s4, s6
; GISEL-NEXT: s_add_u32 s5, s5, s9
-; GISEL-NEXT: s_mul_i32 s8, s7, s5
; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: s_mul_i32 s8, s6, s5
; GISEL-NEXT: s_add_u32 s8, s11, s8
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s13
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_i32 s9, s9, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
-; GISEL-NEXT: s_mul_i32 s10, s6, s5
-; GISEL-NEXT: s_add_u32 s8, s12, s8
-; GISEL-NEXT: s_cselect_b32 s11, 1, 0
-; GISEL-NEXT: s_add_u32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: s_addc_u32 s8, s9, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v7
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: s_mul_i32 s10, s7, s5
+; GISEL-NEXT: s_add_u32 s9, s12, s9
; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v1, v6
-; GISEL-NEXT: s_add_i32 s10, s11, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: s_add_u32 s5, s8, s9
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v8, v6
+; GISEL-NEXT: s_cselect_b32 s11, 1, 0
+; GISEL-NEXT: s_add_u32 s9, s9, s10
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: s_addc_u32 s5, s11, 0
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v6
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s9, s8
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT: s_add_i32 s10, s10, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v5
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v5
; GISEL-NEXT: v_cndmask_b32_e32 v12, -1, v12, vcc
; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v11, v6
-; GISEL-NEXT: s_add_i32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v9, vcc
-; GISEL-NEXT: s_add_u32 s5, s7, s5
+; GISEL-NEXT: s_add_i32 s9, s9, s5
+; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v1, vcc
+; GISEL-NEXT: s_add_u32 s5, s6, s8
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v14, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT: s_addc_u32 s6, s7, s9
; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; GISEL-NEXT: s_addc_u32 s6, s6, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: v_mov_b32_e32 v0, s7
; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
@@ -1400,27 +1325,24 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: s_add_u32 s4, s4, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GISEL-NEXT: s_mul_i32 s9, s5, s4
; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: s_mul_i32 s9, s5, s4
; GISEL-NEXT: s_mul_i32 s7, s6, s7
; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s7, s8, s7
-; GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GISEL-NEXT: s_add_i32 s9, s9, s7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v0
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: s_addc_u32 s7, s9, 0
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v7
; GISEL-NEXT: s_mul_i32 s4, s6, s4
-; GISEL-NEXT: s_add_u32 s7, s7, s8
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_u32 s4, s7, s4
-; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
-; GISEL-NEXT: s_add_i32 s7, s8, s7
-; GISEL-NEXT: s_add_u32 s4, s4, s9
-; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: s_add_i32 s7, s7, s8
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s8, s4
+; GISEL-NEXT: s_addc_u32 s8, s9, 0
+; GISEL-NEXT: s_add_u32 s4, s4, s7
+; GISEL-NEXT: s_addc_u32 s7, s8, 0
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_add_i32 s8, s8, s7
; GISEL-NEXT: s_add_u32 s4, s5, s4
@@ -1430,29 +1352,26 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
-; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, s4
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v5, s5
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v10
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, s5
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v4
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
@@ -1485,223 +1404,205 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-LABEL: v_srem_v2i64_oddk_denom:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
-; CGP-NEXT: v_mov_b32_e32 v5, 0x4f800000
-; CGP-NEXT: v_ashrrev_i32_e32 v6, 31, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v6
-; CGP-NEXT: v_mac_f32_e32 v4, 0, v5
-; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v6
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
-; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT: v_cvt_f32_u32_e32 v5, 0x12d8fb
+; CGP-NEXT: v_mov_b32_e32 v6, 0x4f800000
+; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT: v_mac_f32_e32 v5, 0, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; CGP-NEXT: v_xor_b32_e32 v10, v0, v4
+; CGP-NEXT: v_mov_b32_e32 v0, 0xffed2705
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; CGP-NEXT: v_trunc_f32_e32 v6, v6
+; CGP-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v6
-; CGP-NEXT: v_mul_hi_u32 v8, v4, v7
-; CGP-NEXT: v_readfirstlane_b32 s6, v7
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v7, v0, v5
+; CGP-NEXT: v_readfirstlane_b32 s6, v5
; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s7, v5
-; CGP-NEXT: v_readfirstlane_b32 s5, v8
-; CGP-NEXT: v_mul_hi_u32 v8, v7, s4
+; CGP-NEXT: v_readfirstlane_b32 s7, v6
+; CGP-NEXT: v_readfirstlane_b32 s5, v7
+; CGP-NEXT: v_mul_hi_u32 v7, v5, s4
; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
; CGP-NEXT: s_add_u32 s5, s8, s5
; CGP-NEXT: s_mul_i32 s8, s6, -1
; CGP-NEXT: s_add_u32 s5, s8, s5
-; CGP-NEXT: v_readfirstlane_b32 s8, v8
+; CGP-NEXT: v_readfirstlane_b32 s8, v7
+; CGP-NEXT: v_mul_hi_u32 v7, v6, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v5, s5
; CGP-NEXT: s_mul_i32 s9, s6, s5
-; CGP-NEXT: v_mul_hi_u32 v8, v5, s4
-; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
; CGP-NEXT: s_mul_i32 s10, s7, s4
; CGP-NEXT: s_add_u32 s11, s8, s9
; CGP-NEXT: s_cselect_b32 s12, 1, 0
-; CGP-NEXT: s_add_u32 s11, s11, s10
-; CGP-NEXT: s_cselect_b32 s4, 1, 0
-; CGP-NEXT: s_add_i32 s12, s12, s4
-; CGP-NEXT: v_readfirstlane_b32 s11, v8
-; CGP-NEXT: v_readfirstlane_b32 s13, v7
-; CGP-NEXT: s_mul_i32 s14, s7, s5
-; CGP-NEXT: s_add_u32 s4, s11, s13
-; CGP-NEXT: s_cselect_b32 s15, 1, 0
-; CGP-NEXT: s_add_u32 s4, s4, s14
-; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v5, v5, s5
-; CGP-NEXT: s_add_i32 s15, s15, s16
-; CGP-NEXT: s_add_u32 s4, s4, s12
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
-; CGP-NEXT: s_add_i32 s15, s15, s5
+; CGP-NEXT: s_add_u32 s4, s11, s10
+; CGP-NEXT: s_addc_u32 s4, s12, 0
+; CGP-NEXT: v_readfirstlane_b32 s11, v7
; CGP-NEXT: v_readfirstlane_b32 s12, v5
-; CGP-NEXT: s_add_i32 s5, s12, s15
+; CGP-NEXT: s_mul_i32 s13, s7, s5
+; CGP-NEXT: s_add_u32 s14, s11, s12
+; CGP-NEXT: v_mul_hi_u32 v5, v6, s5
+; CGP-NEXT: s_cselect_b32 s15, 1, 0
+; CGP-NEXT: s_add_u32 s14, s14, s13
+; CGP-NEXT: s_addc_u32 s5, s15, 0
+; CGP-NEXT: s_add_u32 s4, s14, s4
+; CGP-NEXT: s_addc_u32 s5, s5, 0
+; CGP-NEXT: v_readfirstlane_b32 s14, v5
+; CGP-NEXT: s_add_i32 s5, s14, s5
; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_mul_hi_u32 v5, v4, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v0, s4
; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
; CGP-NEXT: s_addc_u32 s5, s7, s5
; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
; CGP-NEXT: v_readfirstlane_b32 s16, v5
; CGP-NEXT: v_mov_b32_e32 v5, s15
-; CGP-NEXT: v_mul_hi_u32 v7, s4, v5
+; CGP-NEXT: v_mul_hi_u32 v6, s4, v5
; CGP-NEXT: s_add_u32 s16, s17, s16
; CGP-NEXT: s_mul_i32 s17, s4, -1
; CGP-NEXT: s_add_u32 s16, s17, s16
-; CGP-NEXT: v_readfirstlane_b32 s17, v7
-; CGP-NEXT: v_mov_b32_e32 v7, s16
-; CGP-NEXT: s_mul_i32 s18, s4, s16
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: v_mov_b32_e32 v6, s16
; CGP-NEXT: v_mul_hi_u32 v5, s5, v5
-; CGP-NEXT: v_mul_hi_u32 v8, s4, v7
+; CGP-NEXT: v_mul_hi_u32 v7, s4, v6
+; CGP-NEXT: s_mul_i32 s18, s4, s16
; CGP-NEXT: s_mul_i32 s15, s5, s15
; CGP-NEXT: s_add_u32 s17, s17, s18
; CGP-NEXT: s_cselect_b32 s18, 1, 0
; CGP-NEXT: s_add_u32 s15, s17, s15
-; CGP-NEXT: s_cselect_b32 s15, 1, 0
-; CGP-NEXT: s_add_i32 s18, s18, s15
-; CGP-NEXT: v_readfirstlane_b32 s15, v5
-; CGP-NEXT: v_readfirstlane_b32 s17, v8
+; CGP-NEXT: s_addc_u32 s15, s18, 0
+; CGP-NEXT: v_readfirstlane_b32 s17, v5
+; CGP-NEXT: v_readfirstlane_b32 s18, v7
; CGP-NEXT: s_mul_i32 s16, s5, s16
-; CGP-NEXT: s_add_u32 s15, s15, s17
-; CGP-NEXT: s_cselect_b32 s17, 1, 0
-; CGP-NEXT: s_add_u32 s15, s15, s16
-; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v5, s5, v7
-; CGP-NEXT: s_add_i32 s16, s17, s16
-; CGP-NEXT: s_add_u32 s15, s15, s18
-; CGP-NEXT: s_cselect_b32 s17, 1, 0
-; CGP-NEXT: s_add_i32 s16, s16, s17
+; CGP-NEXT: s_add_u32 s17, s17, s18
+; CGP-NEXT: v_mul_hi_u32 v5, s5, v6
+; CGP-NEXT: s_cselect_b32 s18, 1, 0
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: s_addc_u32 s17, s18, 0
+; CGP-NEXT: s_add_u32 s15, s16, s15
+; CGP-NEXT: s_addc_u32 s16, s17, 0
; CGP-NEXT: v_readfirstlane_b32 s17, v5
; CGP-NEXT: s_add_i32 s17, s17, s16
; CGP-NEXT: s_add_u32 s4, s4, s15
; CGP-NEXT: s_addc_u32 s5, s5, s17
-; CGP-NEXT: v_mul_hi_u32 v7, v0, s4
-; CGP-NEXT: v_mul_lo_u32 v8, v0, s5
-; CGP-NEXT: v_mul_lo_u32 v9, v1, s4
-; CGP-NEXT: v_mul_hi_u32 v10, v0, s5
-; CGP-NEXT: v_mov_b32_e32 v5, 0x12d8fb
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v6, v10, s4
+; CGP-NEXT: v_mul_lo_u32 v7, v10, s5
+; CGP-NEXT: v_mul_lo_u32 v8, v1, s4
; CGP-NEXT: v_mul_hi_u32 v9, v1, s4
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_mul_lo_u32 v8, v1, s5
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_mul_hi_u32 v11, v10, s5
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_lo_u32 v7, v1, s5
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v11
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v8, v7
-; CGP-NEXT: v_mul_hi_u32 v12, v1, s5
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v5, v10, 0
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v9
-; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v11, v[8:9]
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
-; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v9, vcc
-; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v9
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
+; CGP-NEXT: v_mov_b32_e32 v5, 0x12d8fb
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v7, v6
+; CGP-NEXT: v_mul_hi_u32 v11, v1, s5
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v9, 0
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v8
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v11, v[7:8]
+; CGP-NEXT: v_sub_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v8, vcc
+; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v8
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v5
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
; CGP-NEXT: v_cndmask_b32_e64 v8, -1, v8, s[4:5]
; CGP-NEXT: s_add_u32 s4, s8, s9
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s10
+; CGP-NEXT: s_addc_u32 s4, s5, 0
+; CGP-NEXT: s_add_u32 s5, s11, s12
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s5, s5, s13
; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: s_cselect_b32 s4, 1, 0
-; CGP-NEXT: v_sub_i32_e32 v9, vcc, v0, v5
-; CGP-NEXT: s_add_i32 s5, s5, s4
+; CGP-NEXT: s_addc_u32 s8, s8, 0
+; CGP-NEXT: v_sub_i32_e32 v9, vcc, v6, v5
+; CGP-NEXT: s_add_u32 s4, s5, s4
; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: s_addc_u32 s5, s8, 0
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v9, v5
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_u32 s4, s4, s14
+; CGP-NEXT: s_add_i32 s14, s14, s5
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: s_add_u32 s4, s6, s4
; CGP-NEXT: v_cndmask_b32_e32 v10, -1, v10, vcc
; CGP-NEXT: v_sub_i32_e32 v11, vcc, v9, v5
-; CGP-NEXT: s_add_i32 s8, s8, s9
+; CGP-NEXT: v_mul_hi_u32 v0, v0, s4
; CGP-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v1, vcc
-; CGP-NEXT: s_add_u32 s4, s4, s5
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; CGP-NEXT: s_add_i32 s8, s8, s5
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; CGP-NEXT: s_add_i32 s12, s12, s8
-; CGP-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v6
-; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v6
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
-; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc
; CGP-NEXT: v_ashrrev_i32_e32 v8, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v8
; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT: s_addc_u32 s5, s7, s14
+; CGP-NEXT: v_readfirstlane_b32 s7, v0
+; CGP-NEXT: v_mov_b32_e32 v0, s6
; CGP-NEXT: v_xor_b32_e32 v9, v2, v8
-; CGP-NEXT: v_mov_b32_e32 v2, s6
-; CGP-NEXT: s_addc_u32 s5, s7, s12
-; CGP-NEXT: v_readfirstlane_b32 s7, v4
-; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: v_mul_hi_u32 v2, s4, v0
; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CGP-NEXT: s_add_u32 s7, s8, s7
; CGP-NEXT: s_mul_i32 s8, s4, -1
; CGP-NEXT: s_add_u32 s7, s8, s7
-; CGP-NEXT: v_readfirstlane_b32 s8, v4
-; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: v_mov_b32_e32 v2, s7
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
+; CGP-NEXT: v_mul_hi_u32 v7, s4, v2
; CGP-NEXT: s_mul_i32 s9, s4, s7
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
-; CGP-NEXT: v_mul_hi_u32 v6, s4, v4
; CGP-NEXT: s_mul_i32 s6, s5, s6
; CGP-NEXT: s_add_u32 s8, s8, s9
; CGP-NEXT: s_cselect_b32 s9, 1, 0
; CGP-NEXT: s_add_u32 s6, s8, s6
-; CGP-NEXT: s_cselect_b32 s6, 1, 0
-; CGP-NEXT: s_add_i32 s9, s9, s6
-; CGP-NEXT: v_readfirstlane_b32 s6, v2
-; CGP-NEXT: v_readfirstlane_b32 s8, v6
+; CGP-NEXT: s_addc_u32 s6, s9, 0
+; CGP-NEXT: v_readfirstlane_b32 s8, v0
+; CGP-NEXT: v_readfirstlane_b32 s9, v7
; CGP-NEXT: s_mul_i32 s7, s5, s7
-; CGP-NEXT: s_add_u32 s6, s6, s8
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_u32 s6, s6, s7
-; CGP-NEXT: s_cselect_b32 s7, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
-; CGP-NEXT: s_add_i32 s7, s8, s7
-; CGP-NEXT: s_add_u32 s6, s6, s9
-; CGP-NEXT: s_cselect_b32 s8, 1, 0
-; CGP-NEXT: s_add_i32 s7, s7, s8
-; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: s_add_u32 s8, s8, s9
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v2
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: s_addc_u32 s8, s9, 0
+; CGP-NEXT: s_add_u32 s6, s7, s6
+; CGP-NEXT: s_addc_u32 s7, s8, 0
+; CGP-NEXT: v_readfirstlane_b32 s8, v0
; CGP-NEXT: s_add_i32 s8, s8, s7
; CGP-NEXT: s_add_u32 s4, s4, s6
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
; CGP-NEXT: s_addc_u32 s5, s5, s8
-; CGP-NEXT: v_mul_hi_u32 v2, v9, s4
-; CGP-NEXT: v_mul_lo_u32 v4, v9, s5
+; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
+; CGP-NEXT: v_mul_lo_u32 v2, v9, s5
; CGP-NEXT: v_xor_b32_e32 v10, v3, v8
; CGP-NEXT: v_mul_lo_u32 v3, v10, s4
-; CGP-NEXT: v_mul_hi_u32 v6, v9, s5
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v3, v10, s4
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CGP-NEXT: v_mul_lo_u32 v4, v10, s5
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v3, v2
-; CGP-NEXT: v_mul_hi_u32 v11, v10, s5
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v6, 0
+; CGP-NEXT: v_mul_hi_u32 v11, v9, s5
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; CGP-NEXT: v_mul_lo_u32 v2, v10, s5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v7, v11
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; CGP-NEXT: v_mul_hi_u32 v11, v10, s5
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v0, 0
+; CGP-NEXT: v_xor_b32_e32 v6, v6, v4
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v6, v4
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v7
; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v4, v[3:4]
; CGP-NEXT: v_sub_i32_e32 v2, vcc, v9, v2
; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v10, v6
@@ -1786,17 +1687,14 @@ define i64 @v_srem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v14
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_mul_hi_u32 v8, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
; CHECK-NEXT: v_addc_u32_e32 v11, vcc, v11, v6, vcc
@@ -1812,21 +1710,18 @@ define i64 @v_srem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v12, vcc
; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v10, v2, v9
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v6, v2, v9
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
-; CHECK-NEXT: v_mul_lo_u32 v7, v11, v9
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT: v_mul_lo_u32 v6, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_hi_u32 v7, v11, v9
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v11, v5, vcc
@@ -1836,23 +1731,20 @@ define i64 @v_srem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v4, v9, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v9, v2
; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_mul_hi_u32 v7, v8, v3
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v5, v8, v3
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v6, v9, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_mul_lo_u32 v5, v9, v3
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v2, v4
-; CHECK-NEXT: v_mul_hi_u32 v6, v9, v3
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v9, v3
; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v10, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v4
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v5, v4
; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v6, v[3:4]
; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v10, v[4:5]
@@ -1920,268 +1812,250 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_srem_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v7, 0x1000
-; GISEL-NEXT: v_mov_b32_e32 v8, 0
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[7:8], v4
-; GISEL-NEXT: v_lshl_b64 v[6:7], v[7:8], v6
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v5, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v4, v9
-; GISEL-NEXT: v_xor_b32_e32 v9, v10, v9
+; GISEL-NEXT: v_mov_b32_e32 v10, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v11, 0
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[10:11], v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v5, v7, vcc
+; GISEL-NEXT: v_xor_b32_e32 v5, v4, v7
+; GISEL-NEXT: v_xor_b32_e32 v7, v8, v7
; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v5
-; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v9
+; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v5
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v9, vcc
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v10
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v8
; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4
; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GISEL-NEXT: v_mul_f32_e32 v10, 0x2f800000, v4
-; GISEL-NEXT: v_trunc_f32_e32 v10, v10
-; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v10
+; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v4
+; GISEL-NEXT: v_trunc_f32_e32 v8, v8
+; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v8
; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v4, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[11:12]
-; GISEL-NEXT: v_mul_hi_u32 v11, v4, v10
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v8
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v4, 0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v4, v8
; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v16, v10
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
; GISEL-NEXT: v_mul_lo_u32 v13, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_mul_hi_u32 v15, v4, v14
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_mul_lo_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v14
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v13, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v15
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_mul_hi_u32 v13, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v4, v10
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v11, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v19, 0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v19, vcc, v4, v8
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v19, 0
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[11:12]
-; GISEL-NEXT: v_mul_hi_u32 v11, v19, v10
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
+; GISEL-NEXT: v_lshl_b64 v[9:10], v[10:11], v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v19, v8
; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v19, v[12:13]
-; GISEL-NEXT: v_xor_b32_e32 v15, v0, v4
-; GISEL-NEXT: v_mul_lo_u32 v0, v16, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v16, v10
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v19, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_xor_b32_e32 v13, v0, v4
+; GISEL-NEXT: v_mul_lo_u32 v0, v16, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
+; GISEL-NEXT: v_mul_lo_u32 v11, v19, v14
+; GISEL-NEXT: v_mul_hi_u32 v12, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v16, v14
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v6, v0
+; GISEL-NEXT: v_mul_lo_u32 v6, v16, v14
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v14
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v6, v0
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v11, v6
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v16, v10, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v15, v0
-; GISEL-NEXT: v_mul_lo_u32 v12, v15, v10
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v16, v6, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v0
+; GISEL-NEXT: v_mul_lo_u32 v11, v13, v6
; GISEL-NEXT: v_xor_b32_e32 v14, v1, v4
; GISEL-NEXT: v_mul_lo_u32 v1, v14, v0
; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v15, v10
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_mul_lo_u32 v12, v14, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v13, v6
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
+; GISEL-NEXT: v_mul_lo_u32 v8, v14, v6
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v0, v1
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, v10
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v16, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v10, v11
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v5, v12, v[1:2]
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v9, v16, v[10:11]
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v15, v0
-; GISEL-NEXT: v_subb_u32_e64 v11, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v5
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v9, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v9
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v10, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v1, v8, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e64 v14, s[4:5], 0, v0, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v7
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v1
-; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v7, v1, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v1
-; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
-; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v8
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v15, v15, v16, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v0, v1
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v8, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v10
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, v6
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v10, v8, vcc
+; GISEL-NEXT: v_xor_b32_e32 v6, v6, v8
+; GISEL-NEXT: v_xor_b32_e32 v8, v9, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v16, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v17, v8
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v15, 0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_mac_f32_e32 v16, 0x4f800000, v17
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v11, v[1:2]
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v16
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v13, v0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v7, v15, v[9:10]
; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v8, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v8
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v7, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v20, v[1:2]
-; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
-; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v13, v5
-; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v16, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v15
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
-; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
-; GISEL-NEXT: v_mul_hi_u32 v9, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT: v_mul_lo_u32 v8, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v23, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v14, v9
-; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v20, v0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v15, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v10, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v12, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v19, v14, v[8:9]
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v0
-; GISEL-NEXT: v_xor_b32_e32 v11, v2, v12
-; GISEL-NEXT: v_mul_lo_u32 v2, v15, v0
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, v10
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v12, vcc
-; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v14, v10
+; GISEL-NEXT: v_trunc_f32_e32 v1, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v0
+; GISEL-NEXT: v_sub_i32_e64 v17, s[4:5], 0, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v1
+; GISEL-NEXT: v_subb_u32_e64 v18, s[4:5], 0, v8, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_subb_u32_e64 v19, s[4:5], v14, v11, vcc
+; GISEL-NEXT: v_sub_i32_e64 v14, s[4:5], v14, v11
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v16, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v15, v0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v19, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v18, v15, v[9:10]
+; GISEL-NEXT: v_mul_lo_u32 v9, v15, v11
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v9
+; GISEL-NEXT: v_mul_lo_u32 v9, v16, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v13, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v19, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v1, v9, s[6:7]
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v14, v7, vcc
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v13, v5
+; GISEL-NEXT: v_subbrev_u32_e64 v14, s[6:7], 0, v1, vcc
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v7, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v12, v5
+; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v12, v5
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v14, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v21, 0, -1, s[6:7]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v14, v7
+; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v1, v15, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v20, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, v21, s[6:7]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v20
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_mul_lo_u32 v1, v16, v11
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v12, v5, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v11
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], 0, v10, s[4:5]
+; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v12, s[6:7]
+; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v1
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], 0, v10, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v11, v1
+; GISEL-NEXT: v_add_i32_e64 v15, s[4:5], v15, v0
+; GISEL-NEXT: v_addc_u32_e64 v16, s[4:5], v16, v1, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v14, v7, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v16, v[1:2]
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v19, v7, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v18, v15, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v0
+; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
+; GISEL-NEXT: v_mul_lo_u32 v2, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v15, v11
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v13, v15, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_mul_lo_u32 v9, v15, v10
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v9, v16, v11
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v10
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v16, v11
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v15, v2, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v11, v0
-; GISEL-NEXT: v_mul_lo_u32 v9, v11, v2
-; GISEL-NEXT: v_xor_b32_e32 v10, v3, v12
-; GISEL-NEXT: v_mul_lo_u32 v3, v10, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v10, v0
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v8, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v11, v2
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v15, v0
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v16, v2, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v12, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v12, v2
+; GISEL-NEXT: v_xor_b32_e32 v11, v3, v7
+; GISEL-NEXT: v_mul_lo_u32 v3, v11, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v11, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v13, v12, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
-; GISEL-NEXT: v_mul_lo_u32 v9, v10, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v9, v11, v2
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v9, v10, v2
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v9, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v11, v2
; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v0, v[3:4]
; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v13, v[8:9]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v10, v3
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v7
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v5, v4
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v13, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v11, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v11, v3
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v7
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v2, v6
-; GISEL-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v3, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v6
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v9, vcc, v2, v6
+; GISEL-NEXT: v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v8
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v7
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v8, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v10, v8
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[4:5]
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v12
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v12
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v12
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v12, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64_pow2_shl_denom:
@@ -2231,17 +2105,14 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v15
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v19
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_mul_hi_u32 v13, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
; CGP-NEXT: v_addc_u32_e32 v16, vcc, v16, v11, vcc
@@ -2257,21 +2128,18 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_addc_u32_e32 v9, vcc, v9, v17, vcc
; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v4, v14
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v14
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v12, v8
-; CGP-NEXT: v_mul_lo_u32 v12, v16, v14
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_mul_lo_u32 v11, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v12, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_mul_hi_u32 v12, v16, v14
; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v12, v10
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; CGP-NEXT: v_addc_u32_e32 v8, vcc, v16, v10, vcc
@@ -2281,23 +2149,20 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v9, v14, v4
; CGP-NEXT: v_mul_hi_u32 v4, v14, v4
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_mul_hi_u32 v12, v15, v8
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v10, v15, v8
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; CGP-NEXT: v_mul_lo_u32 v11, v14, v8
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_mul_lo_u32 v10, v14, v8
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v9
-; CGP-NEXT: v_mul_hi_u32 v12, v14, v8
+; CGP-NEXT: v_mul_hi_u32 v11, v14, v8
; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v4, 0
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v11, v10
; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v0, v12, v[9:10]
; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v1, v4, v[10:11]
; CGP-NEXT: v_sub_i32_e32 v4, vcc, v15, v8
@@ -2402,17 +2267,14 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v16
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; CGP-NEXT: v_mul_hi_u32 v11, v6, v12
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v9, vcc
@@ -2428,21 +2290,18 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_addc_u32_e32 v7, vcc, v7, v14, vcc
; CGP-NEXT: v_mul_hi_u32 v8, v6, v8
; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v13, v4, v12
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v9, v4, v12
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; CGP-NEXT: v_mul_lo_u32 v10, v6, v12
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT: v_mul_lo_u32 v9, v6, v12
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CGP-NEXT: v_mul_hi_u32 v10, v6, v12
; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_addc_u32_e32 v5, vcc, v6, v8, vcc
@@ -2452,23 +2311,20 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v7, v10, v4
; CGP-NEXT: v_mul_hi_u32 v4, v10, v4
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT: v_mul_hi_u32 v9, v11, v5
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v7, v11, v5
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
-; CGP-NEXT: v_mul_lo_u32 v8, v10, v5
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v7, v10, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v6
-; CGP-NEXT: v_mul_hi_u32 v8, v10, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v10, v5
; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v12, 0
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v6
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v7, v6
; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v2, v8, v[5:6]
; CGP-NEXT: v_sub_i32_e32 v4, vcc, v11, v4
; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v12, v[6:7]
@@ -2594,133 +2450,123 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, 0, v1
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v11, 0
+; GISEL-NEXT: v_sub_i32_e32 v10, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v11
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; GISEL-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
-; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v3
-; GISEL-NEXT: v_trunc_f32_e32 v5, v5
-; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
+; GISEL-NEXT: v_trunc_f32_e32 v4, v4
+; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
+; GISEL-NEXT: v_cvt_u32_f32_e32 v9, v3
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
; GISEL-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v3, 0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v5, v[8:9]
-; GISEL-NEXT: v_mul_hi_u32 v8, v3, v7
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v3, v[9:10]
-; GISEL-NEXT: v_mul_lo_u32 v9, v5, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v11
-; GISEL-NEXT: v_mul_hi_u32 v12, v3, v11
-; GISEL-NEXT: v_mul_lo_u32 v15, v5, v11
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v5, v11
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v13, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v9, v[7:8]
+; GISEL-NEXT: v_mul_hi_u32 v5, v9, v3
+; GISEL-NEXT: v_mul_lo_u32 v7, v13, v3
+; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
+; GISEL-NEXT: v_mul_lo_u32 v8, v9, v4
+; GISEL-NEXT: v_mul_hi_u32 v14, v9, v4
+; GISEL-NEXT: v_mul_lo_u32 v15, v13, v4
+; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v3, v7
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v15, 0
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v15
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v9, v3
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v14, 0
; GISEL-NEXT: v_and_b32_e32 v3, 0xffffff, v6
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v5, v[8:9]
-; GISEL-NEXT: v_mul_hi_u32 v8, v15, v7
-; GISEL-NEXT: v_mul_lo_u32 v6, v5, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v15, v[9:10]
-; GISEL-NEXT: v_mul_lo_u32 v9, v15, v11
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v15, v11
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_mul_lo_u32 v9, v5, v11
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_mul_hi_u32 v9, v5, v11
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v13, v[5:6]
+; GISEL-NEXT: v_mul_hi_u32 v5, v14, v4
+; GISEL-NEXT: v_mul_lo_u32 v6, v13, v4
+; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v14, v[7:8]
+; GISEL-NEXT: v_mul_lo_u32 v7, v14, v9
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, v9
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v15, v6
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
-; GISEL-NEXT: v_mul_hi_u32 v7, v0, v6
-; GISEL-NEXT: v_mul_lo_u32 v8, v0, v5
-; GISEL-NEXT: v_mul_hi_u32 v6, 0, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, v0, v5
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_lo_u32 v8, 0, v5
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT: v_mul_lo_u32 v6, v13, v9
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mul_hi_u32 v7, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v14, v4
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v13, v5, vcc
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, v4
+; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
+; GISEL-NEXT: v_mul_hi_u32 v7, v0, v5
+; GISEL-NEXT: v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v3
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v6, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, 0, v5
-; GISEL-NEXT: v_mac_f32_e32 v8, 0x4f800000, v4
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v8
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v1, v12, 0
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v4, vcc
+; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v11
+; GISEL-NEXT: v_mul_hi_u32 v6, 0, v5
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v12, 0
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v9, v7
; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v3
-; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v7, v[6:7]
-; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v6, v[5:6]
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v9
+; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
; GISEL-NEXT: v_trunc_f32_e32 v6, v6
-; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
+; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v5
; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v6
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], 0, v12, v[8:9]
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v12, v[7:8]
; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v13, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT: v_subb_u32_e64 v11, s[4:5], 0, v10, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v14, v[7:8]
-; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], 0, v10
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, v6
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v13, v[8:9]
-; GISEL-NEXT: v_mul_hi_u32 v5, v13, v6
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v14, v[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v4, v13, v5
+; GISEL-NEXT: v_sub_i32_e64 v6, s[4:5], 0, v9
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v13, v[7:8]
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, v5
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; GISEL-NEXT: v_mul_lo_u32 v8, v13, v4
-; GISEL-NEXT: v_mul_hi_u32 v6, v14, v6
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v10, v13, v4
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v8, v5
-; GISEL-NEXT: v_mul_lo_u32 v8, v14, v4
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v10, v8
-; GISEL-NEXT: v_mul_hi_u32 v4, v14, v4
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v8, v6
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v6
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v13, v5
-; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v14, v4, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_mul_lo_u32 v7, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v5, v14, v5
+; GISEL-NEXT: v_mul_hi_u32 v11, v13, v10
+; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT: v_addc_u32_e64 v4, s[4:5], 0, v7, s[4:5]
+; GISEL-NEXT: v_mul_lo_u32 v7, v14, v10
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v7
+; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], 0, v9, s[4:5]
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v10
+; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v5, v4
+; GISEL-NEXT: v_addc_u32_e64 v5, s[4:5], 0, v7, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v9, v5
+; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v13, v4
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v14, v5, s[4:5]
; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v10, 0
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v13, -1, v9, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v7, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v12, v[5:6]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v13, -1, v8, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v6, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v11, v[5:6]
; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v0, v1
; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v8, vcc
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v16, v10, v[6:7]
@@ -2730,39 +2576,35 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v5, vcc
; GISEL-NEXT: v_mul_hi_u32 v5, v10, v4
; GISEL-NEXT: v_mul_lo_u32 v7, v10, v8
-; GISEL-NEXT: v_mul_lo_u32 v16, v12, v4
+; GISEL-NEXT: v_mul_lo_u32 v16, v11, v4
; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v14, v1
; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v11, v4
+; GISEL-NEXT: v_mul_hi_u32 v17, v10, v8
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v4, v12, v4
-; GISEL-NEXT: v_mul_hi_u32 v16, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; GISEL-NEXT: v_mul_lo_u32 v7, v12, v8
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v16
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mul_lo_u32 v7, v11, v8
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v17
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v16, v7
-; GISEL-NEXT: v_mul_hi_u32 v8, v12, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v8
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v10, v4
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v11, v5, vcc
; GISEL-NEXT: v_mul_hi_u32 v7, v2, v4
-; GISEL-NEXT: v_mul_lo_u32 v8, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
-; GISEL-NEXT: v_mul_hi_u32 v10, v2, v5
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_lo_u32 v8, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT: v_mul_hi_u32 v8, v2, v5
+; GISEL-NEXT: v_mul_lo_u32 v11, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v10, v2, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v4, v7
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v4, vcc
; GISEL-NEXT: v_mul_hi_u32 v8, 0, v5
; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, v10, 0
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
@@ -2771,7 +2613,7 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v8, v[5:6]
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v11, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v12, v9, vcc
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], 0, v10, v[6:7]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], 0, v8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/subo.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/subo.ll
index 7dddac67f0f842e..70f50e3c443d002 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/subo.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/subo.ll
@@ -8,24 +8,21 @@ define i32 @v_usubo_i32(i32 %a, i32 %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubo_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v1
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubo_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
%usubo = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
%sub = extractvalue {i32, i1} %usubo, 0
@@ -163,33 +160,27 @@ define <2 x i32> @v_usubo_v2i32(<2 x i32> %a, <2 x i32> %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX7-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
+; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubo_v2i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubo_v2i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_sub_u32_e32 v1, v1, v3
+; GFX9-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
%usubo = call {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b)
%sub = extractvalue {<2 x i32>, <2 x i1>} %usubo, 0
@@ -448,22 +439,19 @@ define amdgpu_ps i32 @s_usubo_i32(i32 inreg %a, i32 inreg %b) {
; GFX7-LABEL: s_usubo_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_sub_u32 s0, s0, s1
-; GFX7-NEXT: s_cselect_b32 s1, 1, 0
-; GFX7-NEXT: s_sub_i32 s0, s0, s1
+; GFX7-NEXT: s_subb_u32 s0, s0, 0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_usubo_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_sub_u32 s0, s0, s1
-; GFX8-NEXT: s_cselect_b32 s1, 1, 0
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: s_subb_u32 s0, s0, 0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubo_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_sub_u32 s0, s0, s1
-; GFX9-NEXT: s_cselect_b32 s1, 1, 0
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_subb_u32 s0, s0, 0
; GFX9-NEXT: ; return to shader part epilog
%usubo = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
%sub = extractvalue {i32, i1} %usubo, 0
@@ -516,11 +504,9 @@ define amdgpu_ps <2 x i32> @s_usubo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX7-NEXT: s_sub_u32 s1, s1, s3
; GFX7-NEXT: s_cselect_b32 s3, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s2, 0
-; GFX7-NEXT: s_cselect_b32 s2, 1, 0
+; GFX7-NEXT: s_subb_u32 s0, s0, 0
; GFX7-NEXT: s_cmp_lg_u32 s3, 0
-; GFX7-NEXT: s_cselect_b32 s3, 1, 0
-; GFX7-NEXT: s_sub_i32 s0, s0, s2
-; GFX7-NEXT: s_sub_i32 s1, s1, s3
+; GFX7-NEXT: s_subb_u32 s1, s1, 0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_usubo_v2i32:
@@ -530,11 +516,9 @@ define amdgpu_ps <2 x i32> @s_usubo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX8-NEXT: s_sub_u32 s1, s1, s3
; GFX8-NEXT: s_cselect_b32 s3, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s2, 0
-; GFX8-NEXT: s_cselect_b32 s2, 1, 0
+; GFX8-NEXT: s_subb_u32 s0, s0, 0
; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_sub_i32 s0, s0, s2
-; GFX8-NEXT: s_sub_i32 s1, s1, s3
+; GFX8-NEXT: s_subb_u32 s1, s1, 0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubo_v2i32:
@@ -544,11 +528,9 @@ define amdgpu_ps <2 x i32> @s_usubo_v2i32(<2 x i32> inreg %a, <2 x i32> inreg %b
; GFX9-NEXT: s_sub_u32 s1, s1, s3
; GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s2, 0
-; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_subb_u32 s0, s0, 0
; GFX9-NEXT: s_cmp_lg_u32 s3, 0
-; GFX9-NEXT: s_cselect_b32 s3, 1, 0
-; GFX9-NEXT: s_sub_i32 s0, s0, s2
-; GFX9-NEXT: s_sub_i32 s1, s1, s3
+; GFX9-NEXT: s_subb_u32 s1, s1, 0
; GFX9-NEXT: ; return to shader part epilog
%usubo = call {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b)
%sub = extractvalue {<2 x i32>, <2 x i1>} %usubo, 0
@@ -931,24 +913,21 @@ define amdgpu_ps i32 @usubo_i32_sv(i32 inreg %a, i32 %b) {
; GFX7-LABEL: usubo_i32_sv:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubo_i32_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: usubo_i32_sv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
%usubo = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index cfe86e2b7081aa5..c92d9c42054b0d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -46,22 +46,19 @@ define i64 @v_udiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v13, v0, v8
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
@@ -75,22 +72,19 @@ define i64 @v_udiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
; CHECK-NEXT: v_mul_lo_u32 v10, v1, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v11, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
@@ -98,22 +92,19 @@ define i64 @v_udiv_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
; CHECK-NEXT: v_mul_lo_u32 v8, v5, v0
; CHECK-NEXT: v_mul_hi_u32 v0, v5, v0
+; CHECK-NEXT: v_mul_hi_u32 v9, v4, v1
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v8, v4, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_lo_u32 v7, v5, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; CHECK-NEXT: v_mul_hi_u32 v1, v5, v1
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v2, v0
; CHECK-NEXT: v_mul_lo_u32 v7, v2, v1
@@ -207,43 +198,40 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
; CHECK-NEXT: v_readfirstlane_b32 s7, v1
-; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: v_readfirstlane_b32 s6, v2
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: s_mul_i32 s8, s4, s6
-; CHECK-NEXT: v_readfirstlane_b32 s12, v3
+; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: s_mul_i32 s11, s6, s9
+; CHECK-NEXT: v_mul_hi_u32 v5, v2, s9
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s10, s5, s7
-; CHECK-NEXT: s_add_i32 s8, s12, s8
+; CHECK-NEXT: s_add_i32 s8, s9, s8
; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: v_readfirstlane_b32 s10, v4
-; CHECK-NEXT: s_mul_i32 s12, s7, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v2, s9
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
-; CHECK-NEXT: s_mul_i32 s11, s6, s9
-; CHECK-NEXT: s_add_u32 s10, s10, s12
-; CHECK-NEXT: s_cselect_b32 s12, 1, 0
-; CHECK-NEXT: s_add_u32 s10, s10, s11
-; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_i32 s12, s12, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v3
-; CHECK-NEXT: v_readfirstlane_b32 s10, v1
-; CHECK-NEXT: s_mul_i32 s11, s6, s8
+; CHECK-NEXT: v_readfirstlane_b32 s9, v4
+; CHECK-NEXT: s_mul_i32 s10, s7, s8
; CHECK-NEXT: s_add_u32 s9, s9, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_u32 s9, s9, s11
-; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_addc_u32 s9, s10, 0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v5
+; CHECK-NEXT: v_readfirstlane_b32 s11, v1
+; CHECK-NEXT: s_mul_i32 s12, s6, s8
+; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: v_mul_hi_u32 v1, v2, s8
-; CHECK-NEXT: s_add_i32 s10, s10, s11
-; CHECK-NEXT: s_add_u32 s8, s9, s12
-; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_i32 s10, s10, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v1
-; CHECK-NEXT: s_add_i32 s9, s9, s10
-; CHECK-NEXT: s_add_u32 s7, s7, s8
+; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_add_u32 s10, s10, s12
+; CHECK-NEXT: s_addc_u32 s8, s11, 0
+; CHECK-NEXT: s_add_u32 s9, s10, s9
+; CHECK-NEXT: s_addc_u32 s8, s8, 0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v1
+; CHECK-NEXT: s_add_i32 s10, s10, s8
+; CHECK-NEXT: s_add_u32 s7, s7, s9
; CHECK-NEXT: v_mov_b32_e32 v1, s7
; CHECK-NEXT: v_mul_hi_u32 v1, s4, v1
; CHECK-NEXT: s_mul_i32 s8, s4, s7
-; CHECK-NEXT: s_addc_u32 s6, s6, s9
+; CHECK-NEXT: s_addc_u32 s6, s6, s10
; CHECK-NEXT: s_mul_i32 s4, s4, s6
; CHECK-NEXT: v_readfirstlane_b32 s9, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s8
@@ -254,27 +242,24 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s6
; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
; CHECK-NEXT: v_mul_hi_u32 v3, s7, v2
+; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: s_mul_i32 s10, s6, s8
; CHECK-NEXT: s_add_u32 s5, s5, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s10
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_addc_u32 s5, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s4, s6, s4
-; CHECK-NEXT: s_add_u32 s5, s5, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s4, s5, s4
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: v_mul_hi_u32 v1, s6, v2
-; CHECK-NEXT: s_add_i32 s5, s8, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s9
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s8
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s4, s8, s4
+; CHECK-NEXT: s_addc_u32 s8, s9, 0
+; CHECK-NEXT: s_add_u32 s4, s4, s5
+; CHECK-NEXT: s_addc_u32 s5, s8, 0
; CHECK-NEXT: v_readfirstlane_b32 s8, v1
; CHECK-NEXT: s_add_i32 s8, s8, s5
; CHECK-NEXT: s_add_u32 s4, s7, s4
@@ -282,31 +267,28 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mul_hi_u32 v1, s0, v1
; CHECK-NEXT: s_addc_u32 s5, s6, s8
; CHECK-NEXT: v_mov_b32_e32 v2, s5
-; CHECK-NEXT: s_mul_i32 s7, s0, s5
+; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
; CHECK-NEXT: v_readfirstlane_b32 s6, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s1
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
+; CHECK-NEXT: s_mul_i32 s7, s0, s5
; CHECK-NEXT: s_mul_i32 s8, s1, s4
; CHECK-NEXT: s_add_u32 s6, s6, s7
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s4, 1, 0
-; CHECK-NEXT: s_add_i32 s7, s7, s4
-; CHECK-NEXT: v_readfirstlane_b32 s4, v1
-; CHECK-NEXT: v_readfirstlane_b32 s6, v3
+; CHECK-NEXT: s_add_u32 s4, s6, s8
+; CHECK-NEXT: s_addc_u32 s4, s7, 0
+; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
; CHECK-NEXT: s_mul_i32 s5, s1, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_u32 s4, s4, s5
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s6, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s7
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s6, s5
+; CHECK-NEXT: s_addc_u32 s6, s7, 0
+; CHECK-NEXT: s_add_u32 s4, s5, s4
; CHECK-NEXT: v_mul_hi_u32 v1, s1, v2
; CHECK-NEXT: v_mov_b32_e32 v2, s4
; CHECK-NEXT: v_mul_hi_u32 v2, s2, v2
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_addc_u32 s5, s6, 0
; CHECK-NEXT: v_readfirstlane_b32 s6, v1
; CHECK-NEXT: s_add_i32 s5, s6, s5
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
@@ -425,22 +407,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v15, v8, v12
; GISEL-NEXT: v_mul_lo_u32 v16, v9, v14
; GISEL-NEXT: v_mul_hi_u32 v14, v9, v14
+; GISEL-NEXT: v_mul_hi_u32 v17, v8, v12
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v16, v8, v12
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
; GISEL-NEXT: v_mul_lo_u32 v15, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v17
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v12, vcc
@@ -448,29 +427,25 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v13, v10, v9
; GISEL-NEXT: v_mul_lo_u32 v11, v11, v8
; GISEL-NEXT: v_mul_lo_u32 v10, v10, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v7
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; GISEL-NEXT: v_mul_hi_u32 v12, v8, v10
; GISEL-NEXT: v_mul_lo_u32 v13, v8, v11
; GISEL-NEXT: v_mul_lo_u32 v14, v9, v10
; GISEL-NEXT: v_mul_hi_u32 v10, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v15, v8, v11
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v14, v8, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_mul_lo_u32 v13, v9, v11
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v15
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v11, vcc
@@ -478,29 +453,25 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v11, v0, v9
; GISEL-NEXT: v_mul_lo_u32 v12, v1, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v1, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v14, v6
+; GISEL-NEXT: v_mul_hi_u32 v13, v0, v9
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v0, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_lo_u32 v11, v1, v9
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; GISEL-NEXT: v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
; GISEL-NEXT: v_mul_hi_u32 v10, v4, v8
; GISEL-NEXT: v_mul_lo_u32 v11, v4, v9
; GISEL-NEXT: v_mul_lo_u32 v13, v5, v8
; GISEL-NEXT: v_mul_lo_u32 v12, v4, v8
-; GISEL-NEXT: v_mac_f32_e32 v14, 0x4f800000, v15
+; GISEL-NEXT: v_cvt_f32_u32_e32 v14, v6
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v12
@@ -512,6 +483,7 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v4
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v7
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v5
; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v8
@@ -520,6 +492,7 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
+; GISEL-NEXT: v_mac_f32_e32 v14, 0x4f800000, v15
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v14
@@ -546,23 +519,20 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_hi_u32 v15, v1, v16
; GISEL-NEXT: v_mul_lo_u32 v17, v1, v14
; GISEL-NEXT: v_mul_lo_u32 v12, v4, v16
+; GISEL-NEXT: v_mul_hi_u32 v16, v4, v16
+; GISEL-NEXT: v_mul_hi_u32 v18, v1, v14
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v15, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v4, v16
-; GISEL-NEXT: v_mul_hi_u32 v16, v1, v14
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v17, v12
-; GISEL-NEXT: v_mul_lo_u32 v17, v4, v14
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_mul_lo_u32 v15, v4, v14
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v14, v4, v14
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v15, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12
; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v4, v14, vcc
@@ -578,22 +548,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
; GISEL-NEXT: v_mul_lo_u32 v8, v4, v11
; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v11, v4, v11
+; GISEL-NEXT: v_mul_hi_u32 v14, v1, v12
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v10, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v10, v4, v11
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, v12
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v13, v8
-; GISEL-NEXT: v_mul_lo_u32 v13, v4, v12
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT: v_mul_lo_u32 v10, v4, v12
+; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], 0, v13, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v11, s[4:5], v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v11, v10
; GISEL-NEXT: v_mul_hi_u32 v12, v4, v12
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v13, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v10, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v11, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v12, v10
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v1, v8
; GISEL-NEXT: v_addc_u32_e64 v4, s[4:5], v4, v10, s[4:5]
@@ -603,21 +570,18 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v5, v3, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v3, v8
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v4
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v2, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v4
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v4
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v6, v5
; GISEL-NEXT: v_mul_lo_u32 v9, v6, v4
@@ -692,22 +656,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v15, v0, v12
; CGP-NEXT: v_mul_lo_u32 v16, v1, v14
; CGP-NEXT: v_mul_hi_u32 v14, v1, v14
+; CGP-NEXT: v_mul_hi_u32 v17, v0, v12
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
; CGP-NEXT: v_mul_lo_u32 v15, v1, v12
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v17
; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v15
-; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
@@ -721,22 +682,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v13, v0, v3
; CGP-NEXT: v_mul_lo_u32 v14, v1, v2
; CGP-NEXT: v_mul_hi_u32 v2, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v15, v0, v3
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v0, v3
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v1, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
@@ -744,22 +702,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v3, v10, v1
; CGP-NEXT: v_mul_lo_u32 v12, v11, v0
; CGP-NEXT: v_mul_hi_u32 v0, v11, v0
+; CGP-NEXT: v_mul_hi_u32 v13, v10, v1
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v10, v1
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; CGP-NEXT: v_mul_lo_u32 v3, v11, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
; CGP-NEXT: v_mul_hi_u32 v1, v11, v1
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CGP-NEXT: v_mul_hi_u32 v2, v4, v0
; CGP-NEXT: v_mul_lo_u32 v3, v4, v1
@@ -859,22 +814,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v13, v2, v10
; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v15, v2, v10
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v2, v10
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v3, v10
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v10, vcc
@@ -888,22 +840,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v11, v2, v5
; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v5
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v2, v5
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_mul_lo_u32 v11, v3, v5
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CGP-NEXT: v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc
@@ -911,22 +860,19 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v5, v8, v3
; CGP-NEXT: v_mul_lo_u32 v10, v9, v2
; CGP-NEXT: v_mul_hi_u32 v2, v9, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v8, v3
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v10, v8, v3
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; CGP-NEXT: v_mul_lo_u32 v5, v9, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
; CGP-NEXT: v_mul_hi_u32 v3, v9, v3
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v6, v2
; CGP-NEXT: v_mul_lo_u32 v5, v6, v3
@@ -1028,18 +974,15 @@ define i64 @v_udiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CHECK-NEXT: v_mul_lo_u32 v5, v1, v4
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v2, v0
; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
; CHECK-NEXT: v_mul_hi_u32 v1, v1, v4
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CHECK-NEXT: v_lshr_b64 v[0:1], v[0:1], 20
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -1056,23 +999,20 @@ define <2 x i64> @v_udiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_mul_hi_u32 v5, v0, v4
; GISEL-NEXT: v_mul_lo_u32 v7, v0, v6
; GISEL-NEXT: v_mul_lo_u32 v8, v1, v4
+; GISEL-NEXT: v_mul_hi_u32 v9, v1, v4
; GISEL-NEXT: v_mul_hi_u32 v0, v0, v6
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; GISEL-NEXT: v_mul_lo_u32 v7, v1, v6
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; GISEL-NEXT: v_mul_hi_u32 v1, v1, v6
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; GISEL-NEXT: v_mul_hi_u32 v7, v2, v4
; GISEL-NEXT: v_mul_lo_u32 v8, v2, v6
; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v5
@@ -1083,17 +1023,14 @@ define <2 x i64> @v_udiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; GISEL-NEXT: v_mul_lo_u32 v7, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_mul_hi_u32 v3, v3, v6
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; GISEL-NEXT: v_lshr_b64 v[0:1], v[0:1], 20
; GISEL-NEXT: v_lshr_b64 v[2:3], v[2:3], 20
@@ -1107,23 +1044,20 @@ define <2 x i64> @v_udiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: v_mul_hi_u32 v5, v0, v4
; CGP-NEXT: v_mul_lo_u32 v7, v0, v6
; CGP-NEXT: v_mul_lo_u32 v8, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v9, v1, v4
; CGP-NEXT: v_mul_hi_u32 v0, v0, v6
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v1, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; CGP-NEXT: v_mul_lo_u32 v7, v1, v6
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v9, v0
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; CGP-NEXT: v_mul_hi_u32 v1, v1, v6
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v5
; CGP-NEXT: v_mul_hi_u32 v5, v2, v4
; CGP-NEXT: v_mul_lo_u32 v7, v2, v6
@@ -1133,18 +1067,15 @@ define <2 x i64> @v_udiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; CGP-NEXT: v_mul_lo_u32 v7, v3, v6
; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2
; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; CGP-NEXT: v_mul_hi_u32 v3, v3, v6
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_lshr_b64 v[0:1], v[0:1], 20
; CGP-NEXT: v_lshr_b64 v[2:3], v[2:3], 20
@@ -1198,22 +1129,19 @@ define i64 @v_udiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v13, v0, v8
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
@@ -1227,22 +1155,19 @@ define i64 @v_udiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
; CHECK-NEXT: v_mul_lo_u32 v10, v1, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT: v_mul_hi_u32 v11, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
@@ -1250,22 +1175,19 @@ define i64 @v_udiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v7, v3, v1
; CHECK-NEXT: v_mul_lo_u32 v8, v4, v0
; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mul_hi_u32 v9, v3, v1
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v8, v3, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; CHECK-NEXT: v_mul_hi_u32 v1, v4, v1
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v5, v0
; CHECK-NEXT: v_mul_lo_u32 v7, v5, v1
@@ -1362,22 +1284,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v16, v4, v13
; GISEL-NEXT: v_mul_lo_u32 v17, v5, v15
; GISEL-NEXT: v_mul_hi_u32 v15, v5, v15
+; GISEL-NEXT: v_mul_hi_u32 v18, v4, v13
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v16
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v17
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v17, v4, v13
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v16, vcc
; GISEL-NEXT: v_mul_lo_u32 v16, v5, v13
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v17, v16
; GISEL-NEXT: v_mul_hi_u32 v13, v5, v13
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v14
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
@@ -1391,22 +1310,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v14, v4, v12
; GISEL-NEXT: v_mul_lo_u32 v15, v5, v11
; GISEL-NEXT: v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v16, v4, v12
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v4, v12
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_mul_lo_u32 v14, v5, v12
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v15
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v16
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
; GISEL-NEXT: v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v4, v11
; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v5, v12, vcc
@@ -1414,24 +1330,20 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v14, v0, v12
; GISEL-NEXT: v_lshl_b64 v[4:5], v[9:10], v6
; GISEL-NEXT: v_mul_lo_u32 v6, v1, v11
-; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v5
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, v11
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v13, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v1, v11
-; GISEL-NEXT: v_mul_hi_u32 v11, v0, v12
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v6
-; GISEL-NEXT: v_mul_lo_u32 v10, v1, v12
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, v12
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v13
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_hi_u32 v11, v1, v12
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
; GISEL-NEXT: v_mul_hi_u32 v10, v7, v6
; GISEL-NEXT: v_mul_lo_u32 v11, v7, v9
@@ -1449,6 +1361,7 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v5
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v8
; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v6
@@ -1484,23 +1397,20 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_hi_u32 v15, v1, v16
; GISEL-NEXT: v_mul_lo_u32 v17, v1, v14
; GISEL-NEXT: v_mul_lo_u32 v12, v7, v16
+; GISEL-NEXT: v_mul_hi_u32 v16, v7, v16
+; GISEL-NEXT: v_mul_hi_u32 v18, v1, v14
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v15, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v7, v16
-; GISEL-NEXT: v_mul_hi_u32 v16, v1, v14
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v17, v12
-; GISEL-NEXT: v_mul_lo_u32 v17, v7, v14
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_mul_lo_u32 v15, v7, v14
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v14, v7, v14
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v15, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12
; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v7, v14, vcc
@@ -1516,22 +1426,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; GISEL-NEXT: v_mul_lo_u32 v6, v7, v11
; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v11, v7, v11
+; GISEL-NEXT: v_mul_hi_u32 v14, v1, v12
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v10, v7, v11
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, v12
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v13, v6
-; GISEL-NEXT: v_mul_lo_u32 v13, v7, v12
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT: v_mul_lo_u32 v10, v7, v12
+; GISEL-NEXT: v_addc_u32_e64 v6, s[4:5], 0, v13, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v11, s[4:5], v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v11, v10
; GISEL-NEXT: v_mul_hi_u32 v12, v7, v12
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v13, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v11, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v12, v10
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v1, v6
; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v7, v10, s[4:5]
@@ -1541,21 +1448,18 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v8, v3, v6
; GISEL-NEXT: v_mul_hi_u32 v6, v3, v6
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v7
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v2, v7
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v7
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
; GISEL-NEXT: v_mul_hi_u32 v7, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v4, v6
; GISEL-NEXT: v_mul_lo_u32 v9, v4, v7
@@ -1633,22 +1537,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v16, v0, v13
; CGP-NEXT: v_mul_lo_u32 v17, v1, v15
; CGP-NEXT: v_mul_hi_u32 v15, v1, v15
+; CGP-NEXT: v_mul_hi_u32 v18, v0, v13
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v17
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v17, v0, v13
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v16, vcc
; CGP-NEXT: v_mul_lo_u32 v16, v1, v13
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v18
; CGP-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v17, v16
; CGP-NEXT: v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
-; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v14
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
@@ -1662,22 +1563,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v14, v0, v12
; CGP-NEXT: v_mul_lo_u32 v15, v1, v4
; CGP-NEXT: v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v14
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v15, v0, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_mul_lo_u32 v14, v1, v12
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v15
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v16
; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v14
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
@@ -1685,22 +1583,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v12, v8, v1
; CGP-NEXT: v_mul_lo_u32 v13, v9, v0
; CGP-NEXT: v_mul_hi_u32 v0, v9, v0
+; CGP-NEXT: v_mul_hi_u32 v14, v8, v1
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v13, v8, v1
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v12, vcc
; CGP-NEXT: v_mul_lo_u32 v12, v9, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v14
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; CGP-NEXT: v_mul_hi_u32 v1, v9, v1
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
; CGP-NEXT: v_mul_hi_u32 v4, v2, v0
; CGP-NEXT: v_mul_lo_u32 v12, v2, v1
@@ -1802,22 +1697,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v13, v2, v8
; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v15, v2, v8
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v2, v8
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v3, v8
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v8, v3, v8
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
@@ -1831,22 +1723,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v11, v2, v6
; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v6
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v2, v6
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
; CGP-NEXT: v_mul_lo_u32 v11, v3, v6
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CGP-NEXT: v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc
@@ -1854,22 +1743,19 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v6, v5, v3
; CGP-NEXT: v_mul_lo_u32 v8, v7, v2
; CGP-NEXT: v_mul_hi_u32 v2, v7, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v5, v3
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v6
; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v5, v3
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
; CGP-NEXT: v_mul_lo_u32 v6, v7, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CGP-NEXT: v_mul_hi_u32 v3, v7, v3
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v9, v2
; CGP-NEXT: v_mul_lo_u32 v6, v9, v3
@@ -2001,11 +1887,11 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-LABEL: v_udiv_v2i64_24bit:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v4
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v1, 0
-; GISEL-NEXT: v_sub_i32_e32 v7, vcc, 0, v4
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; GISEL-NEXT: v_sub_i32_e32 v7, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0
@@ -2026,22 +1912,19 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v12, v3, v9
; GISEL-NEXT: v_mul_lo_u32 v13, v5, v11
; GISEL-NEXT: v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v14, v3, v9
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v3, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v12, vcc
; GISEL-NEXT: v_mul_lo_u32 v12, v5, v9
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; GISEL-NEXT: v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v3, v10
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
@@ -2057,60 +1940,56 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v6, v5, v7
; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v10, v8
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v11, v6
-; GISEL-NEXT: v_mul_lo_u32 v11, v5, v8
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; GISEL-NEXT: v_mul_lo_u32 v9, v5, v8
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; GISEL-NEXT: v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v6
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
; GISEL-NEXT: v_mul_hi_u32 v7, v0, v6
-; GISEL-NEXT: v_mul_lo_u32 v8, v0, v5
; GISEL-NEXT: v_mul_hi_u32 v6, 0, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, v0, v5
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_lo_u32 v8, 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT: v_mul_lo_u32 v10, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v9, v0, v5
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v4, v6
-; GISEL-NEXT: v_mul_lo_u32 v8, v4, v5
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT: v_mul_hi_u32 v7, v1, v6
+; GISEL-NEXT: v_mul_lo_u32 v8, v1, v5
; GISEL-NEXT: v_mul_lo_u32 v10, 0, v6
-; GISEL-NEXT: v_mul_lo_u32 v9, v4, v6
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, v6
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v9
; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], 0, v7, vcc
; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], 0, v7
; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v1
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v8, -1, v9, s[4:5]
; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v6
; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v5, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v3
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v3
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
; GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v1
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v4
+; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v4
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GISEL-NEXT: v_add_i32_e32 v7, vcc, 1, v9
; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v10, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
@@ -2136,22 +2015,19 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; GISEL-NEXT: v_mul_lo_u32 v6, v7, v13
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v12, v14
+; GISEL-NEXT: v_mul_hi_u32 v13, v7, v13
+; GISEL-NEXT: v_mul_hi_u32 v14, v1, v11
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v8, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v8, v7, v13
-; GISEL-NEXT: v_mul_hi_u32 v13, v1, v11
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v12, v6
-; GISEL-NEXT: v_mul_lo_u32 v12, v7, v11
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v8, v13
+; GISEL-NEXT: v_mul_lo_u32 v8, v7, v11
+; GISEL-NEXT: v_addc_u32_e64 v6, s[4:5], 0, v12, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v13, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v8, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v12, v8
; GISEL-NEXT: v_mul_hi_u32 v11, v7, v11
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v13, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v8, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v12, v8
+; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], 0, v12, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v11, v8
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v1, v6
; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v7, v8, s[4:5]
@@ -2165,37 +2041,33 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v11, v6, v8
; GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
; GISEL-NEXT: v_mul_lo_u32 v4, v7, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, v9
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v6, v8
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v5, v7, v9
-; GISEL-NEXT: v_mul_hi_u32 v9, v6, v8
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v10, v4
-; GISEL-NEXT: v_mul_lo_u32 v10, v7, v8
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT: v_mul_lo_u32 v5, v7, v8
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v5
; GISEL-NEXT: v_mul_hi_u32 v8, v7, v8
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v6, v4
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc
; GISEL-NEXT: v_mul_hi_u32 v6, v2, v4
-; GISEL-NEXT: v_mul_lo_u32 v7, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
-; GISEL-NEXT: v_mul_hi_u32 v8, v2, v5
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v7, v2, v5
+; GISEL-NEXT: v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v8, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; GISEL-NEXT: v_mul_hi_u32 v6, v3, v4
; GISEL-NEXT: v_mul_lo_u32 v7, v3, v5
; GISEL-NEXT: v_mul_lo_u32 v9, 0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index 2ec45c8f5e7ff08..3c8f7356a49d068 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -165,43 +165,40 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX8-NEXT: v_mul_hi_u32 v2, s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
-; GFX8-NEXT: s_mul_i32 s13, s8, s12
; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_mul_hi_u32 v3, v0, s13
+; GFX8-NEXT: s_mul_i32 s13, s8, s12
; GFX8-NEXT: s_mul_i32 s11, s8, s10
+; GFX8-NEXT: v_mul_hi_u32 v3, v0, s13
; GFX8-NEXT: s_mul_i32 s15, s10, s13
; GFX8-NEXT: v_mul_hi_u32 v4, v1, s13
; GFX8-NEXT: v_readfirstlane_b32 s13, v2
; GFX8-NEXT: s_mul_i32 s14, s9, s12
; GFX8-NEXT: s_add_u32 s11, s11, s13
; GFX8-NEXT: s_add_u32 s11, s14, s11
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s11
; GFX8-NEXT: v_readfirstlane_b32 s13, v3
; GFX8-NEXT: s_mul_i32 s16, s12, s11
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s11
; GFX8-NEXT: s_add_u32 s13, s13, s16
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
; GFX8-NEXT: s_add_u32 s13, s13, s15
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: v_readfirstlane_b32 s14, v4
-; GFX8-NEXT: s_add_i32 s16, s16, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: s_mul_i32 s15, s10, s11
-; GFX8-NEXT: s_add_u32 s13, s14, s13
-; GFX8-NEXT: s_cselect_b32 s14, 1, 0
-; GFX8-NEXT: s_add_u32 s13, s13, s15
-; GFX8-NEXT: s_cselect_b32 s15, 1, 0
+; GFX8-NEXT: s_addc_u32 s13, s16, 0
+; GFX8-NEXT: v_readfirstlane_b32 s15, v0
+; GFX8-NEXT: s_mul_i32 s16, s10, s11
+; GFX8-NEXT: s_add_u32 s14, s14, s15
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s11
-; GFX8-NEXT: s_add_i32 s14, s14, s15
-; GFX8-NEXT: s_add_u32 s11, s13, s16
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s14, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: s_add_i32 s13, s13, s14
-; GFX8-NEXT: s_add_u32 s11, s12, s11
+; GFX8-NEXT: s_cselect_b32 s15, 1, 0
+; GFX8-NEXT: s_add_u32 s14, s14, s16
+; GFX8-NEXT: s_addc_u32 s11, s15, 0
+; GFX8-NEXT: s_add_u32 s13, s14, s13
+; GFX8-NEXT: s_addc_u32 s11, s11, 0
+; GFX8-NEXT: v_readfirstlane_b32 s14, v0
+; GFX8-NEXT: s_add_i32 s14, s14, s11
+; GFX8-NEXT: s_add_u32 s11, s12, s13
; GFX8-NEXT: v_mov_b32_e32 v0, s11
; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
; GFX8-NEXT: s_mul_i32 s12, s8, s11
-; GFX8-NEXT: s_addc_u32 s10, s10, s13
+; GFX8-NEXT: s_addc_u32 s10, s10, s14
; GFX8-NEXT: s_mul_i32 s8, s8, s10
; GFX8-NEXT: v_readfirstlane_b32 s13, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s12
@@ -211,27 +208,24 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: s_add_u32 s8, s9, s8
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: s_mul_i32 s13, s11, s8
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s11, v1
+; GFX8-NEXT: s_mul_i32 s13, s11, s8
; GFX8-NEXT: s_mul_i32 s12, s10, s12
; GFX8-NEXT: s_add_u32 s9, s9, s13
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: s_add_u32 s9, s9, s12
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s13, s9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_readfirstlane_b32 s12, v2
+; GFX8-NEXT: s_addc_u32 s9, s13, 0
+; GFX8-NEXT: v_readfirstlane_b32 s12, v0
+; GFX8-NEXT: v_readfirstlane_b32 s13, v2
; GFX8-NEXT: s_mul_i32 s8, s10, s8
-; GFX8-NEXT: s_add_u32 s9, s9, s12
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_u32 s8, s9, s8
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: s_add_u32 s12, s12, s13
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v1
-; GFX8-NEXT: s_add_i32 s9, s12, s9
-; GFX8-NEXT: s_add_u32 s8, s8, s13
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s9, s9, s12
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_u32 s8, s12, s8
+; GFX8-NEXT: s_addc_u32 s12, s13, 0
+; GFX8-NEXT: s_add_u32 s8, s8, s9
+; GFX8-NEXT: s_addc_u32 s9, s12, 0
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
; GFX8-NEXT: s_add_i32 s12, s12, s9
; GFX8-NEXT: s_add_u32 s8, s11, s8
@@ -239,31 +233,28 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
; GFX8-NEXT: s_addc_u32 s9, s10, s12
; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_mul_i32 s11, s4, s9
+; GFX8-NEXT: v_mul_hi_u32 v2, s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
-; GFX8-NEXT: v_mul_hi_u32 v2, s4, v1
+; GFX8-NEXT: s_mul_i32 s11, s4, s9
; GFX8-NEXT: s_mul_i32 s12, s5, s8
; GFX8-NEXT: s_add_u32 s10, s10, s11
; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_u32 s10, s10, s12
-; GFX8-NEXT: s_cselect_b32 s8, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s11, s8
-; GFX8-NEXT: v_readfirstlane_b32 s8, v0
-; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: s_add_u32 s8, s10, s12
+; GFX8-NEXT: s_addc_u32 s8, s11, 0
+; GFX8-NEXT: v_readfirstlane_b32 s10, v0
+; GFX8-NEXT: v_readfirstlane_b32 s11, v2
; GFX8-NEXT: s_mul_i32 s9, s5, s9
-; GFX8-NEXT: s_add_u32 s8, s8, s10
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_u32 s8, s8, s9
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
-; GFX8-NEXT: s_add_i32 s9, s10, s9
-; GFX8-NEXT: s_add_u32 s8, s8, s11
+; GFX8-NEXT: s_add_u32 s10, s10, s11
+; GFX8-NEXT: s_cselect_b32 s11, 1, 0
+; GFX8-NEXT: s_add_u32 s9, s10, s9
+; GFX8-NEXT: s_addc_u32 s10, s11, 0
+; GFX8-NEXT: s_add_u32 s8, s9, s8
; GFX8-NEXT: v_mul_hi_u32 v0, s5, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: v_mul_hi_u32 v1, s6, v1
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s9, s9, s10
+; GFX8-NEXT: s_addc_u32 s9, s10, 0
; GFX8-NEXT: v_readfirstlane_b32 s10, v0
; GFX8-NEXT: s_add_i32 s9, s10, s9
; GFX8-NEXT: v_readfirstlane_b32 s11, v1
@@ -357,19 +348,16 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s14, s14, s15
; GFX9-NEXT: s_cselect_b32 s15, 1, 0
; GFX9-NEXT: s_add_u32 s14, s14, s16
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
; GFX9-NEXT: s_mul_hi_u32 s13, s10, s13
; GFX9-NEXT: s_mul_hi_u32 s17, s12, s11
-; GFX9-NEXT: s_add_i32 s15, s15, s14
-; GFX9-NEXT: s_mul_i32 s14, s10, s11
+; GFX9-NEXT: s_addc_u32 s14, s15, 0
+; GFX9-NEXT: s_mul_i32 s15, s10, s11
; GFX9-NEXT: s_add_u32 s13, s13, s17
; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_u32 s13, s13, s14
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s16, s14
; GFX9-NEXT: s_add_u32 s13, s13, s15
-; GFX9-NEXT: s_cselect_b32 s15, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s14, s15
+; GFX9-NEXT: s_addc_u32 s15, s16, 0
+; GFX9-NEXT: s_add_u32 s13, s13, s14
+; GFX9-NEXT: s_addc_u32 s14, s15, 0
; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
; GFX9-NEXT: s_add_i32 s11, s11, s14
; GFX9-NEXT: s_add_u32 s12, s12, s13
@@ -386,19 +374,16 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s9, s9, s13
; GFX9-NEXT: s_cselect_b32 s13, 1, 0
; GFX9-NEXT: s_add_u32 s9, s9, s14
-; GFX9-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-NEXT: s_add_i32 s13, s13, s9
-; GFX9-NEXT: s_mul_hi_u32 s9, s10, s11
-; GFX9-NEXT: s_mul_hi_u32 s11, s12, s8
+; GFX9-NEXT: s_addc_u32 s9, s13, 0
+; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
+; GFX9-NEXT: s_mul_hi_u32 s13, s12, s8
; GFX9-NEXT: s_mul_i32 s14, s10, s8
-; GFX9-NEXT: s_add_u32 s9, s9, s11
-; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_u32 s9, s9, s14
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s14
-; GFX9-NEXT: s_add_u32 s9, s9, s13
+; GFX9-NEXT: s_add_u32 s11, s11, s13
; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s13
+; GFX9-NEXT: s_add_u32 s11, s11, s14
+; GFX9-NEXT: s_addc_u32 s13, s13, 0
+; GFX9-NEXT: s_add_u32 s9, s11, s9
+; GFX9-NEXT: s_addc_u32 s11, s13, 0
; GFX9-NEXT: s_mul_hi_u32 s8, s10, s8
; GFX9-NEXT: s_add_i32 s8, s8, s11
; GFX9-NEXT: s_add_u32 s9, s12, s9
@@ -409,19 +394,16 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: s_add_u32 s8, s8, s11
; GFX9-NEXT: s_cselect_b32 s11, 1, 0
; GFX9-NEXT: s_add_u32 s8, s8, s12
-; GFX9-NEXT: s_cselect_b32 s8, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s8
-; GFX9-NEXT: s_mul_hi_u32 s8, s5, s9
-; GFX9-NEXT: s_mul_hi_u32 s9, s4, s10
+; GFX9-NEXT: s_addc_u32 s8, s11, 0
+; GFX9-NEXT: s_mul_hi_u32 s9, s5, s9
+; GFX9-NEXT: s_mul_hi_u32 s11, s4, s10
; GFX9-NEXT: s_mul_i32 s12, s5, s10
-; GFX9-NEXT: s_add_u32 s8, s8, s9
-; GFX9-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-NEXT: s_add_u32 s8, s8, s12
-; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_i32 s9, s9, s12
-; GFX9-NEXT: s_add_u32 s8, s8, s11
+; GFX9-NEXT: s_add_u32 s9, s9, s11
; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_i32 s9, s9, s11
+; GFX9-NEXT: s_add_u32 s9, s9, s12
+; GFX9-NEXT: s_addc_u32 s11, s11, 0
+; GFX9-NEXT: s_add_u32 s8, s9, s8
+; GFX9-NEXT: s_addc_u32 s9, s11, 0
; GFX9-NEXT: s_mul_hi_u32 s10, s5, s10
; GFX9-NEXT: s_add_i32 s9, s10, s9
; GFX9-NEXT: s_mul_hi_u32 s11, s6, s8
@@ -510,21 +492,18 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX10-NEXT: s_mul_i32 s16, s8, s13
; GFX10-NEXT: s_add_u32 s14, s14, s15
; GFX10-NEXT: s_cselect_b32 s15, 1, 0
-; GFX10-NEXT: s_add_u32 s14, s14, s16
-; GFX10-NEXT: s_cselect_b32 s14, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s13, s8, s13
; GFX10-NEXT: s_mul_hi_u32 s17, s9, s12
-; GFX10-NEXT: s_add_i32 s15, s15, s14
-; GFX10-NEXT: s_mul_i32 s16, s8, s12
+; GFX10-NEXT: s_add_u32 s14, s14, s16
+; GFX10-NEXT: s_addc_u32 s14, s15, 0
+; GFX10-NEXT: s_mul_i32 s18, s8, s12
; GFX10-NEXT: s_add_u32 s13, s13, s17
-; GFX10-NEXT: s_cselect_b32 s14, 1, 0
-; GFX10-NEXT: s_add_u32 s13, s13, s16
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s12, s8, s12
-; GFX10-NEXT: s_add_i32 s14, s14, s16
-; GFX10-NEXT: s_add_u32 s13, s13, s15
; GFX10-NEXT: s_cselect_b32 s15, 1, 0
-; GFX10-NEXT: s_add_i32 s14, s14, s15
+; GFX10-NEXT: s_add_u32 s13, s13, s18
+; GFX10-NEXT: s_addc_u32 s15, s15, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s14
+; GFX10-NEXT: s_mul_hi_u32 s12, s8, s12
+; GFX10-NEXT: s_addc_u32 s14, s15, 0
; GFX10-NEXT: s_add_i32 s12, s12, s14
; GFX10-NEXT: s_add_u32 s9, s9, s13
; GFX10-NEXT: s_addc_u32 s8, s8, s12
@@ -542,44 +521,38 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX10-NEXT: s_add_u32 s12, s14, s12
; GFX10-NEXT: s_cselect_b32 s14, 1, 0
; GFX10-NEXT: s_add_u32 s12, s12, s13
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_mul_i32 s13, s8, s10
-; GFX10-NEXT: s_add_i32 s14, s14, s12
+; GFX10-NEXT: s_addc_u32 s12, s14, 0
+; GFX10-NEXT: s_mul_i32 s16, s8, s10
; GFX10-NEXT: s_add_u32 s11, s11, s15
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_u32 s11, s11, s13
; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s16
+; GFX10-NEXT: s_addc_u32 s13, s13, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s12
; GFX10-NEXT: s_mul_hi_u32 s10, s8, s10
-; GFX10-NEXT: s_add_i32 s12, s12, s13
-; GFX10-NEXT: s_add_u32 s11, s11, s14
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_add_i32 s12, s12, s13
+; GFX10-NEXT: s_addc_u32 s12, s13, 0
; GFX10-NEXT: s_add_i32 s10, s10, s12
; GFX10-NEXT: s_add_u32 s9, s9, s11
-; GFX10-NEXT: s_addc_u32 s10, s8, s10
-; GFX10-NEXT: s_mul_hi_u32 s8, s4, s9
-; GFX10-NEXT: s_mul_i32 s12, s4, s10
+; GFX10-NEXT: s_addc_u32 s8, s8, s10
+; GFX10-NEXT: s_mul_hi_u32 s10, s4, s9
+; GFX10-NEXT: s_mul_i32 s12, s4, s8
; GFX10-NEXT: s_mul_i32 s11, s5, s9
-; GFX10-NEXT: s_add_u32 s8, s8, s12
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_u32 s8, s8, s11
-; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s13, s4, s8
+; GFX10-NEXT: s_mul_i32 s14, s5, s8
+; GFX10-NEXT: s_mul_hi_u32 s15, s5, s8
+; GFX10-NEXT: s_add_u32 s8, s10, s12
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s9, s5, s9
-; GFX10-NEXT: s_mul_hi_u32 s13, s4, s10
-; GFX10-NEXT: s_add_i32 s12, s12, s8
-; GFX10-NEXT: s_mul_i32 s11, s5, s10
-; GFX10-NEXT: s_add_u32 s8, s9, s13
-; GFX10-NEXT: s_cselect_b32 s9, 1, 0
; GFX10-NEXT: s_add_u32 s8, s8, s11
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s10, s5, s10
-; GFX10-NEXT: s_add_i32 s9, s9, s11
-; GFX10-NEXT: s_add_u32 s8, s8, s12
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_mul_i32 s12, s7, s8
-; GFX10-NEXT: s_add_i32 s9, s9, s11
-; GFX10-NEXT: s_add_i32 s9, s10, s9
+; GFX10-NEXT: s_addc_u32 s8, s10, 0
+; GFX10-NEXT: s_add_u32 s9, s9, s13
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_add_u32 s9, s9, s14
+; GFX10-NEXT: s_addc_u32 s10, s10, 0
+; GFX10-NEXT: s_add_u32 s8, s9, s8
+; GFX10-NEXT: s_addc_u32 s9, s10, 0
; GFX10-NEXT: s_mul_hi_u32 s10, s6, s8
+; GFX10-NEXT: s_add_i32 s9, s15, s9
+; GFX10-NEXT: s_mul_i32 s12, s7, s8
; GFX10-NEXT: s_mul_i32 s11, s6, s9
; GFX10-NEXT: s_add_i32 s10, s10, s11
; GFX10-NEXT: s_mul_i32 s11, s6, s8
@@ -1306,39 +1279,36 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX8-NEXT: v_mul_hi_u32 v2, s10, v0
; GFX8-NEXT: v_readfirstlane_b32 s1, v0
-; GFX8-NEXT: s_mul_i32 s2, s10, s1
; GFX8-NEXT: v_readfirstlane_b32 s16, v1
-; GFX8-NEXT: v_mul_hi_u32 v3, v0, s2
+; GFX8-NEXT: s_mul_i32 s2, s10, s1
; GFX8-NEXT: s_mul_i32 s0, s10, s16
+; GFX8-NEXT: v_mul_hi_u32 v3, v0, s2
; GFX8-NEXT: s_mul_i32 s4, s16, s2
; GFX8-NEXT: v_mul_hi_u32 v4, v1, s2
; GFX8-NEXT: v_readfirstlane_b32 s2, v2
; GFX8-NEXT: s_mul_i32 s3, s11, s1
; GFX8-NEXT: s_add_u32 s0, s0, s2
; GFX8-NEXT: s_add_u32 s0, s3, s0
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s0
; GFX8-NEXT: v_readfirstlane_b32 s2, v3
; GFX8-NEXT: s_mul_i32 s5, s1, s0
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s0
; GFX8-NEXT: s_add_u32 s2, s2, s5
; GFX8-NEXT: s_cselect_b32 s5, 1, 0
; GFX8-NEXT: s_add_u32 s2, s2, s4
-; GFX8-NEXT: s_cselect_b32 s2, 1, 0
; GFX8-NEXT: v_readfirstlane_b32 s3, v4
-; GFX8-NEXT: s_add_i32 s5, s5, s2
-; GFX8-NEXT: v_readfirstlane_b32 s2, v0
-; GFX8-NEXT: s_mul_i32 s4, s16, s0
-; GFX8-NEXT: s_add_u32 s2, s3, s2
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_add_u32 s2, s2, s4
-; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: s_addc_u32 s2, s5, 0
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_mul_i32 s5, s16, s0
+; GFX8-NEXT: s_add_u32 s3, s3, s4
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s0
-; GFX8-NEXT: s_add_i32 s3, s3, s4
-; GFX8-NEXT: s_add_u32 s0, s2, s5
-; GFX8-NEXT: s_cselect_b32 s2, 1, 0
-; GFX8-NEXT: s_add_i32 s3, s3, s2
-; GFX8-NEXT: v_readfirstlane_b32 s2, v0
-; GFX8-NEXT: s_add_i32 s17, s2, s3
-; GFX8-NEXT: s_add_u32 s18, s1, s0
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: s_add_u32 s3, s3, s5
+; GFX8-NEXT: s_addc_u32 s0, s4, 0
+; GFX8-NEXT: s_add_u32 s2, s3, s2
+; GFX8-NEXT: s_addc_u32 s0, s0, 0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_add_i32 s17, s3, s0
+; GFX8-NEXT: s_add_u32 s18, s1, s2
; GFX8-NEXT: v_mov_b32_e32 v0, s18
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
@@ -1353,27 +1323,24 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_u32 s10, s11, s10
; GFX8-NEXT: v_readfirstlane_b32 s11, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_mul_i32 s16, s18, s10
; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s18, v1
+; GFX8-NEXT: s_mul_i32 s16, s18, s10
; GFX8-NEXT: s_mul_i32 s9, s8, s9
; GFX8-NEXT: s_add_u32 s11, s11, s16
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
; GFX8-NEXT: s_add_u32 s9, s11, s9
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
-; GFX8-NEXT: s_add_i32 s16, s16, s9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_readfirstlane_b32 s11, v2
+; GFX8-NEXT: s_addc_u32 s9, s16, 0
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: v_readfirstlane_b32 s16, v2
; GFX8-NEXT: s_mul_i32 s10, s8, s10
-; GFX8-NEXT: s_add_u32 s9, s9, s11
-; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_u32 s9, s9, s10
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
+; GFX8-NEXT: s_add_u32 s11, s11, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s8, v1
-; GFX8-NEXT: s_add_i32 s10, s11, s10
-; GFX8-NEXT: s_add_u32 s9, s9, s16
-; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s10, s10, s11
+; GFX8-NEXT: s_cselect_b32 s16, 1, 0
+; GFX8-NEXT: s_add_u32 s10, s11, s10
+; GFX8-NEXT: s_addc_u32 s11, s16, 0
+; GFX8-NEXT: s_add_u32 s9, s10, s9
+; GFX8-NEXT: s_addc_u32 s10, s11, 0
; GFX8-NEXT: v_readfirstlane_b32 s11, v0
; GFX8-NEXT: s_add_i32 s11, s11, s10
; GFX8-NEXT: s_add_u32 s9, s18, s9
@@ -1382,31 +1349,28 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
; GFX8-NEXT: s_addc_u32 s8, s8, s11
; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: s_mul_i32 s11, s4, s8
+; GFX8-NEXT: v_mul_hi_u32 v2, s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s10, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s9
-; GFX8-NEXT: v_mul_hi_u32 v2, s4, v1
+; GFX8-NEXT: s_mul_i32 s11, s4, s8
; GFX8-NEXT: s_mul_i32 s16, s5, s9
; GFX8-NEXT: s_add_u32 s10, s10, s11
; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_u32 s10, s10, s16
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s11, s9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: s_add_u32 s9, s10, s16
+; GFX8-NEXT: s_addc_u32 s9, s11, 0
+; GFX8-NEXT: v_readfirstlane_b32 s10, v0
+; GFX8-NEXT: v_readfirstlane_b32 s11, v2
; GFX8-NEXT: s_mul_i32 s8, s5, s8
-; GFX8-NEXT: s_add_u32 s9, s9, s10
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_u32 s8, s9, s8
-; GFX8-NEXT: s_cselect_b32 s9, 1, 0
-; GFX8-NEXT: s_add_i32 s9, s10, s9
-; GFX8-NEXT: s_add_u32 s8, s8, s11
+; GFX8-NEXT: s_add_u32 s10, s10, s11
+; GFX8-NEXT: s_cselect_b32 s11, 1, 0
+; GFX8-NEXT: s_add_u32 s8, s10, s8
+; GFX8-NEXT: s_addc_u32 s10, s11, 0
+; GFX8-NEXT: s_add_u32 s8, s8, s9
; GFX8-NEXT: v_mul_hi_u32 v0, s5, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: v_mul_hi_u32 v1, s12, v1
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s9, s9, s10
+; GFX8-NEXT: s_addc_u32 s9, s10, 0
; GFX8-NEXT: v_readfirstlane_b32 s10, v0
; GFX8-NEXT: s_add_i32 s9, s10, s9
; GFX8-NEXT: v_readfirstlane_b32 s11, v1
@@ -1480,34 +1444,31 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_mul_i32 s18, s13, s10
; GFX8-NEXT: s_add_u32 s17, s18, s17
; GFX8-NEXT: v_readfirstlane_b32 s18, v2
-; GFX8-NEXT: s_mul_i32 s19, s10, s17
; GFX8-NEXT: v_mul_hi_u32 v2, v1, s16
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s17
+; GFX8-NEXT: s_mul_i32 s19, s10, s17
; GFX8-NEXT: s_mul_i32 s20, s12, s16
; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
-; GFX8-NEXT: s_add_u32 s18, s18, s20
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_i32 s19, s19, s16
-; GFX8-NEXT: v_readfirstlane_b32 s16, v2
-; GFX8-NEXT: v_readfirstlane_b32 s18, v0
+; GFX8-NEXT: s_add_u32 s16, s18, s20
+; GFX8-NEXT: s_addc_u32 s16, s19, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: v_readfirstlane_b32 s19, v0
; GFX8-NEXT: s_mul_i32 s20, s12, s17
-; GFX8-NEXT: s_add_u32 s16, s16, s18
-; GFX8-NEXT: s_cselect_b32 s18, 1, 0
-; GFX8-NEXT: s_add_u32 s16, s16, s20
-; GFX8-NEXT: s_cselect_b32 s20, 1, 0
+; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: v_mul_hi_u32 v0, v1, s17
-; GFX8-NEXT: s_add_i32 s18, s18, s20
-; GFX8-NEXT: s_add_u32 s16, s16, s19
-; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_add_u32 s18, s18, s20
+; GFX8-NEXT: s_addc_u32 s17, s19, 0
+; GFX8-NEXT: s_add_u32 s16, s18, s16
+; GFX8-NEXT: s_addc_u32 s17, s17, 0
+; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: s_add_i32 s18, s18, s17
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: s_add_i32 s17, s17, s18
; GFX8-NEXT: s_add_u32 s10, s10, s16
; GFX8-NEXT: v_mov_b32_e32 v0, s10
; GFX8-NEXT: v_mul_hi_u32 v0, s11, v0
; GFX8-NEXT: s_mul_i32 s16, s11, s10
-; GFX8-NEXT: s_addc_u32 s12, s12, s17
+; GFX8-NEXT: s_addc_u32 s12, s12, s18
; GFX8-NEXT: s_mul_i32 s11, s11, s12
; GFX8-NEXT: v_readfirstlane_b32 s17, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s16
@@ -1517,27 +1478,24 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_u32 s11, s13, s11
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_mul_i32 s17, s10, s11
; GFX8-NEXT: v_mul_hi_u32 v0, s12, v0
; GFX8-NEXT: v_mul_hi_u32 v2, s10, v1
+; GFX8-NEXT: s_mul_i32 s17, s10, s11
; GFX8-NEXT: s_mul_i32 s16, s12, s16
; GFX8-NEXT: s_add_u32 s13, s13, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_u32 s13, s13, s16
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s17, s17, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: v_readfirstlane_b32 s16, v2
+; GFX8-NEXT: s_addc_u32 s13, s17, 0
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: v_readfirstlane_b32 s17, v2
; GFX8-NEXT: s_mul_i32 s11, s12, s11
-; GFX8-NEXT: s_add_u32 s13, s13, s16
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_u32 s11, s13, s11
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_u32 s16, s16, s17
; GFX8-NEXT: v_mul_hi_u32 v0, s12, v1
-; GFX8-NEXT: s_add_i32 s13, s16, s13
-; GFX8-NEXT: s_add_u32 s11, s11, s17
-; GFX8-NEXT: s_cselect_b32 s16, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s13, s16
+; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_add_u32 s11, s16, s11
+; GFX8-NEXT: s_addc_u32 s16, s17, 0
+; GFX8-NEXT: s_add_u32 s11, s11, s13
+; GFX8-NEXT: s_addc_u32 s13, s16, 0
; GFX8-NEXT: v_readfirstlane_b32 s16, v0
; GFX8-NEXT: s_add_i32 s16, s16, s13
; GFX8-NEXT: s_add_u32 s10, s10, s11
@@ -1545,31 +1503,28 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
; GFX8-NEXT: s_addc_u32 s11, s12, s16
; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_mul_i32 s13, s6, s11
+; GFX8-NEXT: v_mul_hi_u32 v2, s6, v1
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s7
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s10
-; GFX8-NEXT: v_mul_hi_u32 v2, s6, v1
+; GFX8-NEXT: s_mul_i32 s13, s6, s11
; GFX8-NEXT: s_mul_i32 s16, s7, s10
; GFX8-NEXT: s_add_u32 s12, s12, s13
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_u32 s12, s12, s16
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s13, s10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v0
-; GFX8-NEXT: v_readfirstlane_b32 s12, v2
+; GFX8-NEXT: s_add_u32 s10, s12, s16
+; GFX8-NEXT: s_addc_u32 s10, s13, 0
+; GFX8-NEXT: v_readfirstlane_b32 s12, v0
+; GFX8-NEXT: v_readfirstlane_b32 s13, v2
; GFX8-NEXT: s_mul_i32 s11, s7, s11
-; GFX8-NEXT: s_add_u32 s10, s10, s12
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_u32 s10, s10, s11
-; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s12, s11
-; GFX8-NEXT: s_add_u32 s10, s10, s13
+; GFX8-NEXT: s_add_u32 s12, s12, s13
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_u32 s11, s12, s11
+; GFX8-NEXT: s_addc_u32 s12, s13, 0
+; GFX8-NEXT: s_add_u32 s10, s11, s10
; GFX8-NEXT: v_mul_hi_u32 v0, s7, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: v_mul_hi_u32 v1, s14, v1
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s11, s12
+; GFX8-NEXT: s_addc_u32 s11, s12, 0
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
; GFX8-NEXT: s_add_i32 s11, s12, s11
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
@@ -1668,19 +1623,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s18, s18, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
; GFX9-NEXT: s_add_u32 s18, s18, s20
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
; GFX9-NEXT: s_mul_hi_u32 s17, s10, s17
-; GFX9-NEXT: s_add_i32 s19, s19, s18
-; GFX9-NEXT: s_mul_hi_u32 s18, s16, s11
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
+; GFX9-NEXT: s_mul_hi_u32 s19, s16, s11
; GFX9-NEXT: s_mul_i32 s20, s10, s11
-; GFX9-NEXT: s_add_u32 s17, s17, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_u32 s17, s17, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s20
; GFX9-NEXT: s_add_u32 s17, s17, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s19
+; GFX9-NEXT: s_add_u32 s17, s17, s20
+; GFX9-NEXT: s_addc_u32 s19, s19, 0
+; GFX9-NEXT: s_add_u32 s17, s17, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
; GFX9-NEXT: s_add_i32 s11, s11, s18
; GFX9-NEXT: s_add_u32 s16, s16, s17
@@ -1697,19 +1649,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s9, s9, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
; GFX9-NEXT: s_add_u32 s9, s9, s18
-; GFX9-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-NEXT: s_add_i32 s17, s17, s9
-; GFX9-NEXT: s_mul_hi_u32 s9, s10, s11
-; GFX9-NEXT: s_mul_hi_u32 s11, s16, s8
+; GFX9-NEXT: s_addc_u32 s9, s17, 0
+; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
+; GFX9-NEXT: s_mul_hi_u32 s17, s16, s8
; GFX9-NEXT: s_mul_i32 s18, s10, s8
-; GFX9-NEXT: s_add_u32 s9, s9, s11
-; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_u32 s9, s9, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s18
-; GFX9-NEXT: s_add_u32 s9, s9, s17
+; GFX9-NEXT: s_add_u32 s11, s11, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s17
+; GFX9-NEXT: s_add_u32 s11, s11, s18
+; GFX9-NEXT: s_addc_u32 s17, s17, 0
+; GFX9-NEXT: s_add_u32 s9, s11, s9
+; GFX9-NEXT: s_addc_u32 s11, s17, 0
; GFX9-NEXT: s_mul_hi_u32 s8, s10, s8
; GFX9-NEXT: s_add_i32 s8, s8, s11
; GFX9-NEXT: s_add_u32 s9, s16, s9
@@ -1720,19 +1669,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s8, s8, s11
; GFX9-NEXT: s_cselect_b32 s11, 1, 0
; GFX9-NEXT: s_add_u32 s8, s8, s16
-; GFX9-NEXT: s_cselect_b32 s8, 1, 0
-; GFX9-NEXT: s_add_i32 s11, s11, s8
-; GFX9-NEXT: s_mul_hi_u32 s8, s5, s9
-; GFX9-NEXT: s_mul_hi_u32 s9, s4, s10
+; GFX9-NEXT: s_addc_u32 s8, s11, 0
+; GFX9-NEXT: s_mul_hi_u32 s9, s5, s9
+; GFX9-NEXT: s_mul_hi_u32 s11, s4, s10
; GFX9-NEXT: s_mul_i32 s16, s5, s10
-; GFX9-NEXT: s_add_u32 s8, s8, s9
-; GFX9-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-NEXT: s_add_u32 s8, s8, s16
-; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_i32 s9, s9, s16
-; GFX9-NEXT: s_add_u32 s8, s8, s11
+; GFX9-NEXT: s_add_u32 s9, s9, s11
; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_i32 s9, s9, s11
+; GFX9-NEXT: s_add_u32 s9, s9, s16
+; GFX9-NEXT: s_addc_u32 s11, s11, 0
+; GFX9-NEXT: s_add_u32 s8, s9, s8
+; GFX9-NEXT: s_addc_u32 s9, s11, 0
; GFX9-NEXT: s_mul_hi_u32 s10, s5, s10
; GFX9-NEXT: s_add_i32 s9, s10, s9
; GFX9-NEXT: s_mul_hi_u32 s11, s12, s8
@@ -1809,19 +1755,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s18, s18, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
; GFX9-NEXT: s_add_u32 s18, s18, s20
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s19, s19, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s16, s11, s16
-; GFX9-NEXT: s_mul_hi_u32 s18, s10, s17
+; GFX9-NEXT: s_mul_hi_u32 s19, s10, s17
; GFX9-NEXT: s_mul_i32 s20, s11, s17
-; GFX9-NEXT: s_add_u32 s16, s16, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_u32 s16, s16, s20
-; GFX9-NEXT: s_cselect_b32 s20, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s20
; GFX9-NEXT: s_add_u32 s16, s16, s19
; GFX9-NEXT: s_cselect_b32 s19, 1, 0
-; GFX9-NEXT: s_add_i32 s18, s18, s19
+; GFX9-NEXT: s_add_u32 s16, s16, s20
+; GFX9-NEXT: s_addc_u32 s19, s19, 0
+; GFX9-NEXT: s_add_u32 s16, s16, s18
+; GFX9-NEXT: s_addc_u32 s18, s19, 0
; GFX9-NEXT: s_mul_hi_u32 s17, s11, s17
; GFX9-NEXT: s_add_i32 s17, s17, s18
; GFX9-NEXT: s_add_u32 s10, s10, s16
@@ -1838,19 +1781,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s13, s13, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
; GFX9-NEXT: s_add_u32 s13, s13, s18
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s17, s17, s13
-; GFX9-NEXT: s_mul_hi_u32 s13, s11, s16
-; GFX9-NEXT: s_mul_hi_u32 s16, s10, s12
+; GFX9-NEXT: s_addc_u32 s13, s17, 0
+; GFX9-NEXT: s_mul_hi_u32 s16, s11, s16
+; GFX9-NEXT: s_mul_hi_u32 s17, s10, s12
; GFX9-NEXT: s_mul_i32 s18, s11, s12
-; GFX9-NEXT: s_add_u32 s13, s13, s16
-; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_u32 s13, s13, s18
-; GFX9-NEXT: s_cselect_b32 s18, 1, 0
-; GFX9-NEXT: s_add_i32 s16, s16, s18
-; GFX9-NEXT: s_add_u32 s13, s13, s17
+; GFX9-NEXT: s_add_u32 s16, s16, s17
; GFX9-NEXT: s_cselect_b32 s17, 1, 0
-; GFX9-NEXT: s_add_i32 s16, s16, s17
+; GFX9-NEXT: s_add_u32 s16, s16, s18
+; GFX9-NEXT: s_addc_u32 s17, s17, 0
+; GFX9-NEXT: s_add_u32 s13, s16, s13
+; GFX9-NEXT: s_addc_u32 s16, s17, 0
; GFX9-NEXT: s_mul_hi_u32 s12, s11, s12
; GFX9-NEXT: s_add_i32 s12, s12, s16
; GFX9-NEXT: s_add_u32 s10, s10, s13
@@ -1861,19 +1801,16 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: s_add_u32 s12, s12, s13
; GFX9-NEXT: s_cselect_b32 s13, 1, 0
; GFX9-NEXT: s_add_u32 s12, s12, s16
-; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_i32 s13, s13, s12
+; GFX9-NEXT: s_addc_u32 s12, s13, 0
; GFX9-NEXT: s_mul_hi_u32 s10, s7, s10
-; GFX9-NEXT: s_mul_hi_u32 s12, s6, s11
+; GFX9-NEXT: s_mul_hi_u32 s13, s6, s11
; GFX9-NEXT: s_mul_i32 s16, s7, s11
-; GFX9-NEXT: s_add_u32 s10, s10, s12
-; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_u32 s10, s10, s16
-; GFX9-NEXT: s_cselect_b32 s16, 1, 0
-; GFX9-NEXT: s_add_i32 s12, s12, s16
; GFX9-NEXT: s_add_u32 s10, s10, s13
; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s12, s12, s13
+; GFX9-NEXT: s_add_u32 s10, s10, s16
+; GFX9-NEXT: s_addc_u32 s13, s13, 0
+; GFX9-NEXT: s_add_u32 s10, s10, s12
+; GFX9-NEXT: s_addc_u32 s12, s13, 0
; GFX9-NEXT: s_mul_hi_u32 s11, s7, s11
; GFX9-NEXT: s_add_i32 s11, s11, s12
; GFX9-NEXT: s_mul_hi_u32 s13, s14, s10
@@ -1944,7 +1881,7 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s13
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s12
-; GFX10-NEXT: s_sub_u32 s10, 0, s12
+; GFX10-NEXT: s_sub_u32 s8, 0, s12
; GFX10-NEXT: s_subb_u32 s11, 0, s13
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0
; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
@@ -1955,97 +1892,88 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: v_mul_f32_e32 v2, 0xcf800000, v1
; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX10-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s15
+; GFX10-NEXT: v_readfirstlane_b32 s9, v1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s15
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: s_mul_i32 s16, s10, s8
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v0
+; GFX10-NEXT: s_mul_i32 s16, s8, s9
+; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v2
+; GFX10-NEXT: v_readfirstlane_b32 s10, v0
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s14
-; GFX10-NEXT: s_mul_hi_u32 s18, s10, s9
-; GFX10-NEXT: s_mul_i32 s19, s11, s9
+; GFX10-NEXT: s_mul_hi_u32 s18, s8, s10
+; GFX10-NEXT: s_mul_i32 s19, s11, s10
; GFX10-NEXT: s_add_i32 s16, s18, s16
-; GFX10-NEXT: s_mul_i32 s17, s10, s9
+; GFX10-NEXT: s_mul_i32 s17, s8, s10
; GFX10-NEXT: s_add_i32 s16, s16, s19
-; GFX10-NEXT: s_mul_hi_u32 s18, s9, s17
-; GFX10-NEXT: s_mul_i32 s19, s9, s16
-; GFX10-NEXT: s_mul_i32 s20, s8, s17
+; GFX10-NEXT: s_mul_hi_u32 s18, s10, s17
+; GFX10-NEXT: s_mul_i32 s19, s10, s16
+; GFX10-NEXT: s_mul_i32 s20, s9, s17
; GFX10-NEXT: s_add_u32 s18, s18, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s17, s9, s17
+; GFX10-NEXT: s_mul_hi_u32 s21, s10, s16
; GFX10-NEXT: s_add_u32 s18, s18, s20
-; GFX10-NEXT: s_cselect_b32 s18, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s17, s8, s17
-; GFX10-NEXT: s_mul_hi_u32 s21, s9, s16
-; GFX10-NEXT: s_add_i32 s19, s19, s18
-; GFX10-NEXT: s_mul_i32 s20, s8, s16
+; GFX10-NEXT: s_addc_u32 s18, s19, 0
+; GFX10-NEXT: s_mul_i32 s22, s9, s16
; GFX10-NEXT: s_add_u32 s17, s17, s21
-; GFX10-NEXT: s_cselect_b32 s18, 1, 0
-; GFX10-NEXT: s_add_u32 s17, s17, s20
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s16, s8, s16
-; GFX10-NEXT: s_add_i32 s18, s18, s20
-; GFX10-NEXT: s_add_u32 s17, s17, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s22
+; GFX10-NEXT: s_addc_u32 s19, s19, 0
+; GFX10-NEXT: s_add_u32 s17, s17, s18
+; GFX10-NEXT: s_mul_hi_u32 s16, s9, s16
+; GFX10-NEXT: s_addc_u32 s18, s19, 0
; GFX10-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX10-NEXT: s_add_i32 s18, s18, s19
; GFX10-NEXT: s_add_i32 s16, s16, s18
-; GFX10-NEXT: s_add_u32 s9, s9, s17
-; GFX10-NEXT: s_addc_u32 s8, s8, s16
-; GFX10-NEXT: s_mul_i32 s16, s10, s9
-; GFX10-NEXT: s_mul_hi_u32 s17, s10, s9
-; GFX10-NEXT: s_mul_i32 s10, s10, s8
-; GFX10-NEXT: s_mul_i32 s11, s11, s9
-; GFX10-NEXT: s_add_i32 s10, s17, s10
-; GFX10-NEXT: s_mul_hi_u32 s18, s9, s16
-; GFX10-NEXT: s_add_i32 s10, s10, s11
-; GFX10-NEXT: s_mul_i32 s17, s8, s16
-; GFX10-NEXT: s_mul_hi_u32 s11, s8, s16
-; GFX10-NEXT: s_mul_i32 s16, s9, s10
-; GFX10-NEXT: s_mul_hi_u32 s19, s9, s10
+; GFX10-NEXT: s_add_u32 s10, s10, s17
+; GFX10-NEXT: s_addc_u32 s9, s9, s16
+; GFX10-NEXT: s_mul_i32 s16, s8, s10
+; GFX10-NEXT: s_mul_hi_u32 s17, s8, s10
+; GFX10-NEXT: s_mul_i32 s8, s8, s9
+; GFX10-NEXT: s_mul_i32 s11, s11, s10
+; GFX10-NEXT: s_add_i32 s8, s17, s8
+; GFX10-NEXT: s_mul_hi_u32 s18, s10, s16
+; GFX10-NEXT: s_add_i32 s8, s8, s11
+; GFX10-NEXT: s_mul_i32 s17, s9, s16
+; GFX10-NEXT: s_mul_hi_u32 s11, s9, s16
+; GFX10-NEXT: s_mul_i32 s16, s10, s8
+; GFX10-NEXT: s_mul_hi_u32 s19, s10, s8
; GFX10-NEXT: s_add_u32 s16, s18, s16
; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_add_u32 s16, s16, s17
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_mul_i32 s17, s8, s10
-; GFX10-NEXT: s_add_i32 s18, s18, s16
+; GFX10-NEXT: s_addc_u32 s16, s18, 0
+; GFX10-NEXT: s_mul_i32 s20, s9, s8
; GFX10-NEXT: s_add_u32 s11, s11, s19
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s11, s11, s17
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s10, s8, s10
-; GFX10-NEXT: s_add_i32 s16, s16, s17
-; GFX10-NEXT: s_add_u32 s11, s11, s18
; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s20
+; GFX10-NEXT: s_addc_u32 s17, s17, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s16
+; GFX10-NEXT: s_mul_hi_u32 s8, s9, s8
+; GFX10-NEXT: s_addc_u32 s16, s17, 0
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX10-NEXT: s_add_i32 s16, s16, s17
-; GFX10-NEXT: s_add_i32 s10, s10, s16
-; GFX10-NEXT: s_add_u32 s9, s9, s11
-; GFX10-NEXT: s_addc_u32 s10, s8, s10
-; GFX10-NEXT: s_mul_hi_u32 s8, s4, s9
-; GFX10-NEXT: s_mul_i32 s16, s4, s10
-; GFX10-NEXT: s_mul_i32 s11, s5, s9
-; GFX10-NEXT: s_add_u32 s8, s8, s16
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s8, s8, s11
-; GFX10-NEXT: s_cselect_b32 s8, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s9, s5, s9
-; GFX10-NEXT: s_mul_hi_u32 s17, s4, s10
-; GFX10-NEXT: s_add_i32 s16, s16, s8
+; GFX10-NEXT: s_add_i32 s8, s8, s16
+; GFX10-NEXT: s_add_u32 s10, s10, s11
+; GFX10-NEXT: s_addc_u32 s8, s9, s8
+; GFX10-NEXT: s_mul_hi_u32 s9, s4, s10
+; GFX10-NEXT: s_mul_i32 s16, s4, s8
; GFX10-NEXT: s_mul_i32 s11, s5, s10
-; GFX10-NEXT: s_add_u32 s8, s9, s17
+; GFX10-NEXT: s_mul_hi_u32 s17, s4, s8
+; GFX10-NEXT: s_mul_i32 s18, s5, s8
+; GFX10-NEXT: s_mul_hi_u32 s19, s5, s8
+; GFX10-NEXT: s_add_u32 s8, s9, s16
; GFX10-NEXT: s_cselect_b32 s9, 1, 0
-; GFX10-NEXT: s_add_u32 s8, s8, s11
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s10, s5, s10
-; GFX10-NEXT: s_add_i32 s9, s9, s11
-; GFX10-NEXT: s_add_u32 s8, s8, s16
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_mul_i32 s16, s13, s8
-; GFX10-NEXT: s_add_i32 s9, s9, s11
-; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX10-NEXT: s_add_i32 s9, s10, s9
+; GFX10-NEXT: s_add_u32 s8, s8, s11
+; GFX10-NEXT: s_addc_u32 s8, s9, 0
+; GFX10-NEXT: s_add_u32 s9, s10, s17
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_add_u32 s9, s9, s18
+; GFX10-NEXT: s_addc_u32 s10, s10, 0
+; GFX10-NEXT: s_add_u32 s8, s9, s8
+; GFX10-NEXT: s_addc_u32 s9, s10, 0
; GFX10-NEXT: s_mul_hi_u32 s10, s12, s8
+; GFX10-NEXT: s_add_i32 s9, s19, s9
+; GFX10-NEXT: s_mul_i32 s16, s13, s8
; GFX10-NEXT: s_mul_i32 s11, s12, s9
+; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; GFX10-NEXT: s_add_i32 s10, s10, s11
; GFX10-NEXT: s_mul_i32 s11, s12, s8
; GFX10-NEXT: s_add_i32 s16, s10, s16
@@ -2110,21 +2038,18 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_mul_i32 s20, s12, s13
; GFX10-NEXT: s_add_u32 s17, s17, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_add_u32 s17, s17, s20
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s13, s12, s13
; GFX10-NEXT: s_mul_hi_u32 s21, s18, s16
-; GFX10-NEXT: s_add_i32 s19, s19, s17
-; GFX10-NEXT: s_mul_i32 s20, s12, s16
+; GFX10-NEXT: s_add_u32 s17, s17, s20
+; GFX10-NEXT: s_addc_u32 s17, s19, 0
+; GFX10-NEXT: s_mul_i32 s22, s12, s16
; GFX10-NEXT: s_add_u32 s13, s13, s21
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
-; GFX10-NEXT: s_add_u32 s13, s13, s20
-; GFX10-NEXT: s_cselect_b32 s20, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s16, s12, s16
-; GFX10-NEXT: s_add_i32 s17, s17, s20
-; GFX10-NEXT: s_add_u32 s13, s13, s19
; GFX10-NEXT: s_cselect_b32 s19, 1, 0
-; GFX10-NEXT: s_add_i32 s17, s17, s19
+; GFX10-NEXT: s_add_u32 s13, s13, s22
+; GFX10-NEXT: s_addc_u32 s19, s19, 0
+; GFX10-NEXT: s_add_u32 s13, s13, s17
+; GFX10-NEXT: s_mul_hi_u32 s16, s12, s16
+; GFX10-NEXT: s_addc_u32 s17, s19, 0
; GFX10-NEXT: s_add_i32 s16, s16, s17
; GFX10-NEXT: s_add_u32 s13, s18, s13
; GFX10-NEXT: s_addc_u32 s12, s12, s16
@@ -2142,44 +2067,38 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: s_add_u32 s16, s18, s16
; GFX10-NEXT: s_cselect_b32 s18, 1, 0
; GFX10-NEXT: s_add_u32 s16, s16, s17
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_mul_i32 s17, s12, s10
-; GFX10-NEXT: s_add_i32 s18, s18, s16
+; GFX10-NEXT: s_addc_u32 s16, s18, 0
+; GFX10-NEXT: s_mul_i32 s20, s12, s10
; GFX10-NEXT: s_add_u32 s11, s11, s19
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s11, s11, s17
; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s20
+; GFX10-NEXT: s_addc_u32 s17, s17, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s16
; GFX10-NEXT: s_mul_hi_u32 s10, s12, s10
-; GFX10-NEXT: s_add_i32 s16, s16, s17
-; GFX10-NEXT: s_add_u32 s11, s11, s18
-; GFX10-NEXT: s_cselect_b32 s17, 1, 0
-; GFX10-NEXT: s_add_i32 s16, s16, s17
+; GFX10-NEXT: s_addc_u32 s16, s17, 0
; GFX10-NEXT: s_add_i32 s10, s10, s16
; GFX10-NEXT: s_add_u32 s11, s13, s11
-; GFX10-NEXT: s_addc_u32 s12, s12, s10
-; GFX10-NEXT: s_mul_hi_u32 s10, s6, s11
-; GFX10-NEXT: s_mul_i32 s16, s6, s12
+; GFX10-NEXT: s_addc_u32 s10, s12, s10
+; GFX10-NEXT: s_mul_hi_u32 s12, s6, s11
+; GFX10-NEXT: s_mul_i32 s16, s6, s10
; GFX10-NEXT: s_mul_i32 s13, s7, s11
-; GFX10-NEXT: s_add_u32 s10, s10, s16
-; GFX10-NEXT: s_cselect_b32 s16, 1, 0
-; GFX10-NEXT: s_add_u32 s10, s10, s13
-; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_mul_hi_u32 s17, s6, s10
+; GFX10-NEXT: s_mul_i32 s18, s7, s10
+; GFX10-NEXT: s_mul_hi_u32 s19, s7, s10
+; GFX10-NEXT: s_add_u32 s10, s12, s16
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
; GFX10-NEXT: s_mul_hi_u32 s11, s7, s11
-; GFX10-NEXT: s_mul_hi_u32 s17, s6, s12
-; GFX10-NEXT: s_add_i32 s16, s16, s10
-; GFX10-NEXT: s_mul_i32 s13, s7, s12
-; GFX10-NEXT: s_add_u32 s10, s11, s17
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_add_u32 s10, s10, s13
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_mul_hi_u32 s12, s7, s12
-; GFX10-NEXT: s_add_i32 s11, s11, s13
-; GFX10-NEXT: s_add_u32 s10, s10, s16
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_mul_i32 s16, s15, s10
-; GFX10-NEXT: s_add_i32 s11, s11, s13
-; GFX10-NEXT: s_add_i32 s11, s12, s11
+; GFX10-NEXT: s_addc_u32 s10, s12, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s17
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_add_u32 s11, s11, s18
+; GFX10-NEXT: s_addc_u32 s12, s12, 0
+; GFX10-NEXT: s_add_u32 s10, s11, s10
+; GFX10-NEXT: s_addc_u32 s11, s12, 0
; GFX10-NEXT: s_mul_hi_u32 s12, s14, s10
+; GFX10-NEXT: s_add_i32 s11, s19, s11
+; GFX10-NEXT: s_mul_i32 s16, s15, s10
; GFX10-NEXT: s_mul_i32 s13, s14, s11
; GFX10-NEXT: s_add_i32 s12, s12, s13
; GFX10-NEXT: s_mul_i32 s13, s14, s10
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 69dca2f91764c8d..724ca3be5ef65cd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -46,22 +46,19 @@ define i64 @v_urem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v13, v0, v8
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
@@ -75,22 +72,19 @@ define i64 @v_urem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
; CHECK-NEXT: v_mul_lo_u32 v10, v1, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v11, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
@@ -98,22 +92,19 @@ define i64 @v_urem_i64(i64 %num, i64 %den) {
; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
; CHECK-NEXT: v_mul_lo_u32 v8, v5, v0
; CHECK-NEXT: v_mul_hi_u32 v0, v5, v0
+; CHECK-NEXT: v_mul_hi_u32 v9, v4, v1
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v8, v4, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_lo_u32 v7, v5, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; CHECK-NEXT: v_mul_hi_u32 v1, v5, v1
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v2, v0
; CHECK-NEXT: v_mul_lo_u32 v1, v2, v1
@@ -204,43 +195,40 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
; CHECK-NEXT: v_readfirstlane_b32 s7, v1
-; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: v_readfirstlane_b32 s6, v2
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
; CHECK-NEXT: s_mul_i32 s8, s4, s6
-; CHECK-NEXT: v_readfirstlane_b32 s12, v3
+; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: s_mul_i32 s11, s6, s9
+; CHECK-NEXT: v_mul_hi_u32 v5, v2, s9
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s10, s5, s7
-; CHECK-NEXT: s_add_i32 s8, s12, s8
+; CHECK-NEXT: s_add_i32 s8, s9, s8
; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: v_readfirstlane_b32 s10, v4
-; CHECK-NEXT: s_mul_i32 s12, s7, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v2, s9
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
-; CHECK-NEXT: s_mul_i32 s11, s6, s9
-; CHECK-NEXT: s_add_u32 s10, s10, s12
-; CHECK-NEXT: s_cselect_b32 s12, 1, 0
-; CHECK-NEXT: s_add_u32 s10, s10, s11
-; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_i32 s12, s12, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v3
-; CHECK-NEXT: v_readfirstlane_b32 s10, v1
-; CHECK-NEXT: s_mul_i32 s11, s6, s8
+; CHECK-NEXT: v_readfirstlane_b32 s9, v4
+; CHECK-NEXT: s_mul_i32 s10, s7, s8
; CHECK-NEXT: s_add_u32 s9, s9, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_u32 s9, s9, s11
-; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_addc_u32 s9, s10, 0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v5
+; CHECK-NEXT: v_readfirstlane_b32 s11, v1
+; CHECK-NEXT: s_mul_i32 s12, s6, s8
+; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: v_mul_hi_u32 v1, v2, s8
-; CHECK-NEXT: s_add_i32 s10, s10, s11
-; CHECK-NEXT: s_add_u32 s8, s9, s12
-; CHECK-NEXT: s_cselect_b32 s9, 1, 0
-; CHECK-NEXT: s_add_i32 s10, s10, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v1
-; CHECK-NEXT: s_add_i32 s9, s9, s10
-; CHECK-NEXT: s_add_u32 s7, s7, s8
+; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_add_u32 s10, s10, s12
+; CHECK-NEXT: s_addc_u32 s8, s11, 0
+; CHECK-NEXT: s_add_u32 s9, s10, s9
+; CHECK-NEXT: s_addc_u32 s8, s8, 0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v1
+; CHECK-NEXT: s_add_i32 s10, s10, s8
+; CHECK-NEXT: s_add_u32 s7, s7, s9
; CHECK-NEXT: v_mov_b32_e32 v1, s7
; CHECK-NEXT: v_mul_hi_u32 v1, s4, v1
; CHECK-NEXT: s_mul_i32 s8, s4, s7
-; CHECK-NEXT: s_addc_u32 s6, s6, s9
+; CHECK-NEXT: s_addc_u32 s6, s6, s10
; CHECK-NEXT: s_mul_i32 s4, s4, s6
; CHECK-NEXT: v_readfirstlane_b32 s9, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s8
@@ -251,27 +239,24 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s6
; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
; CHECK-NEXT: v_mul_hi_u32 v3, s7, v2
+; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: s_mul_i32 s10, s6, s8
; CHECK-NEXT: s_add_u32 s5, s5, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s10
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s9, s9, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_addc_u32 s5, s9, 0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: s_mul_i32 s4, s6, s4
-; CHECK-NEXT: s_add_u32 s5, s5, s8
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_u32 s4, s5, s4
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: v_mul_hi_u32 v1, s6, v2
-; CHECK-NEXT: s_add_i32 s5, s8, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s9
-; CHECK-NEXT: s_cselect_b32 s8, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s8
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s4, s8, s4
+; CHECK-NEXT: s_addc_u32 s8, s9, 0
+; CHECK-NEXT: s_add_u32 s4, s4, s5
+; CHECK-NEXT: s_addc_u32 s5, s8, 0
; CHECK-NEXT: v_readfirstlane_b32 s8, v1
; CHECK-NEXT: s_add_i32 s8, s8, s5
; CHECK-NEXT: s_add_u32 s4, s7, s4
@@ -279,31 +264,28 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mul_hi_u32 v1, s0, v1
; CHECK-NEXT: s_addc_u32 s5, s6, s8
; CHECK-NEXT: v_mov_b32_e32 v2, s5
-; CHECK-NEXT: s_mul_i32 s7, s0, s5
+; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
; CHECK-NEXT: v_readfirstlane_b32 s6, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s1
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
+; CHECK-NEXT: s_mul_i32 s7, s0, s5
; CHECK-NEXT: s_mul_i32 s8, s1, s4
; CHECK-NEXT: s_add_u32 s6, s6, s7
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: s_add_u32 s6, s6, s8
-; CHECK-NEXT: s_cselect_b32 s4, 1, 0
-; CHECK-NEXT: s_add_i32 s7, s7, s4
-; CHECK-NEXT: v_readfirstlane_b32 s4, v1
-; CHECK-NEXT: v_readfirstlane_b32 s6, v3
+; CHECK-NEXT: s_add_u32 s4, s6, s8
+; CHECK-NEXT: s_addc_u32 s4, s7, 0
+; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
; CHECK-NEXT: s_mul_i32 s5, s1, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_u32 s4, s4, s5
-; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s6, s5
-; CHECK-NEXT: s_add_u32 s4, s4, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s7
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s6, s5
+; CHECK-NEXT: s_addc_u32 s6, s7, 0
+; CHECK-NEXT: s_add_u32 s4, s5, s4
; CHECK-NEXT: v_mul_hi_u32 v1, s1, v2
; CHECK-NEXT: v_mov_b32_e32 v2, s4
; CHECK-NEXT: v_mul_hi_u32 v2, s2, v2
-; CHECK-NEXT: s_cselect_b32 s6, 1, 0
-; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_addc_u32 s5, s6, 0
; CHECK-NEXT: v_readfirstlane_b32 s6, v1
; CHECK-NEXT: s_add_i32 s6, s6, s5
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
@@ -421,22 +403,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v15, v8, v12
; GISEL-NEXT: v_mul_lo_u32 v16, v9, v14
; GISEL-NEXT: v_mul_hi_u32 v14, v9, v14
+; GISEL-NEXT: v_mul_hi_u32 v17, v8, v12
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v16, v8, v12
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
; GISEL-NEXT: v_mul_lo_u32 v15, v9, v12
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v17
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v12, vcc
@@ -444,29 +423,25 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v13, v10, v9
; GISEL-NEXT: v_mul_lo_u32 v11, v11, v8
; GISEL-NEXT: v_mul_lo_u32 v10, v10, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v7
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; GISEL-NEXT: v_mul_hi_u32 v12, v8, v10
; GISEL-NEXT: v_mul_lo_u32 v13, v8, v11
; GISEL-NEXT: v_mul_lo_u32 v14, v9, v10
; GISEL-NEXT: v_mul_hi_u32 v10, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v15, v8, v11
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v14, v8, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_mul_lo_u32 v13, v9, v11
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v15
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v11, vcc
@@ -474,29 +449,25 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v11, v0, v9
; GISEL-NEXT: v_mul_lo_u32 v12, v1, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v1, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v14, v6
+; GISEL-NEXT: v_mul_hi_u32 v13, v0, v9
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v0, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_lo_u32 v11, v1, v9
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; GISEL-NEXT: v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
; GISEL-NEXT: v_mul_hi_u32 v10, v4, v8
; GISEL-NEXT: v_mul_lo_u32 v9, v4, v9
; GISEL-NEXT: v_mul_lo_u32 v11, v4, v8
; GISEL-NEXT: v_mul_lo_u32 v8, v5, v8
-; GISEL-NEXT: v_mac_f32_e32 v14, 0x4f800000, v15
+; GISEL-NEXT: v_cvt_f32_u32_e32 v14, v6
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v11
@@ -508,12 +479,14 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v5
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v7
; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v10, s[4:5]
; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v0, v4
; GISEL-NEXT: v_subbrev_u32_e64 v11, s[4:5], 0, v1, vcc
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v5
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v4
+; GISEL-NEXT: v_mac_f32_e32 v14, 0x4f800000, v15
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v5
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
@@ -541,23 +514,20 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_hi_u32 v15, v5, v16
; GISEL-NEXT: v_mul_lo_u32 v17, v5, v14
; GISEL-NEXT: v_mul_lo_u32 v11, v10, v16
+; GISEL-NEXT: v_mul_hi_u32 v16, v10, v16
+; GISEL-NEXT: v_mul_hi_u32 v18, v5, v14
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v10, v16
-; GISEL-NEXT: v_mul_hi_u32 v16, v5, v14
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v17, v11
-; GISEL-NEXT: v_mul_lo_u32 v17, v10, v14
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_mul_lo_u32 v15, v10, v14
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v14, v10, v14
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v14, vcc
@@ -573,22 +543,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GISEL-NEXT: v_mul_lo_u32 v4, v10, v12
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT: v_mul_hi_u32 v14, v5, v11
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v8, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v8, v10, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v5, v11
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v13, v4
-; GISEL-NEXT: v_mul_lo_u32 v13, v10, v11
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v8, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v8, v13
+; GISEL-NEXT: v_mul_lo_u32 v8, v10, v11
+; GISEL-NEXT: v_addc_u32_e64 v4, s[4:5], 0, v13, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v12, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v12, v8
; GISEL-NEXT: v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v13, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v8, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v12, v8
+; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], 0, v12, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v11, v8
; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v5, v4
; GISEL-NEXT: v_addc_u32_e64 v5, s[4:5], v10, v8, s[4:5]
@@ -598,21 +565,18 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v9, v3, v4
; GISEL-NEXT: v_mul_hi_u32 v4, v3, v4
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v5
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v2, v5
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v5
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GISEL-NEXT: v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v6, v4
; GISEL-NEXT: v_mul_lo_u32 v5, v6, v5
@@ -686,22 +650,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v15, v0, v12
; CGP-NEXT: v_mul_lo_u32 v16, v1, v14
; CGP-NEXT: v_mul_hi_u32 v14, v1, v14
+; CGP-NEXT: v_mul_hi_u32 v17, v0, v12
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
; CGP-NEXT: v_mul_lo_u32 v15, v1, v12
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v17
; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v15
-; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
@@ -715,22 +676,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v13, v0, v3
; CGP-NEXT: v_mul_lo_u32 v14, v1, v2
; CGP-NEXT: v_mul_hi_u32 v2, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v15, v0, v3
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v0, v3
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v1, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
@@ -738,22 +696,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v3, v10, v1
; CGP-NEXT: v_mul_lo_u32 v12, v11, v0
; CGP-NEXT: v_mul_hi_u32 v0, v11, v0
+; CGP-NEXT: v_mul_hi_u32 v13, v10, v1
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v10, v1
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; CGP-NEXT: v_mul_lo_u32 v3, v11, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
; CGP-NEXT: v_mul_hi_u32 v1, v11, v1
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CGP-NEXT: v_mul_hi_u32 v2, v4, v0
; CGP-NEXT: v_mul_lo_u32 v1, v4, v1
@@ -850,22 +805,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v13, v2, v10
; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v15, v2, v10
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v2, v10
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v3, v10
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v10, vcc
@@ -879,22 +831,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v11, v2, v5
; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v5
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v2, v5
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_mul_lo_u32 v11, v3, v5
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CGP-NEXT: v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc
@@ -902,22 +851,19 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v5, v8, v3
; CGP-NEXT: v_mul_lo_u32 v10, v9, v2
; CGP-NEXT: v_mul_hi_u32 v2, v9, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v8, v3
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v10, v8, v3
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; CGP-NEXT: v_mul_lo_u32 v5, v9, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
; CGP-NEXT: v_mul_hi_u32 v3, v9, v3
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v6, v2
; CGP-NEXT: v_mul_lo_u32 v3, v6, v3
@@ -1015,22 +961,19 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: v_mul_lo_u32 v5, v0, v4
; CHECK-NEXT: v_mul_lo_u32 v6, v1, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT: v_mul_hi_u32 v7, v0, v4
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v6, v0, v4
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CHECK-NEXT: v_mul_lo_u32 v5, v1, v4
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; CHECK-NEXT: v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
; CHECK-NEXT: v_lshr_b64 v[2:3], v[2:3], 20
; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
@@ -1055,44 +998,38 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_mul_hi_u32 v4, v0, v6
; GISEL-NEXT: v_mul_lo_u32 v5, v0, v7
; GISEL-NEXT: v_mul_lo_u32 v8, v1, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, v0, v7
-; GISEL-NEXT: v_mul_lo_u32 v10, v2, v7
+; GISEL-NEXT: v_mul_hi_u32 v9, v1, v6
+; GISEL-NEXT: v_mul_hi_u32 v10, v0, v7
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, v6
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; GISEL-NEXT: v_mul_lo_u32 v5, v1, v7
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v10
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
; GISEL-NEXT: v_mul_hi_u32 v8, v1, v7
; GISEL-NEXT: v_mul_hi_u32 v9, v2, v6
+; GISEL-NEXT: v_mul_lo_u32 v10, v2, v7
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v7
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
; GISEL-NEXT: v_mul_lo_u32 v8, v3, v6
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v6, v3, v6
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v6, v3, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, v2, v7
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v7
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
; GISEL-NEXT: v_mul_hi_u32 v7, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; GISEL-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
; GISEL-NEXT: v_lshr_b64 v[6:7], v[6:7], 20
@@ -1119,51 +1056,45 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: v_mul_hi_u32 v4, v0, v6
; CGP-NEXT: v_mul_lo_u32 v5, v0, v7
; CGP-NEXT: v_mul_lo_u32 v8, v1, v6
-; CGP-NEXT: v_mul_hi_u32 v9, v0, v7
-; CGP-NEXT: v_mul_lo_u32 v11, v2, v7
+; CGP-NEXT: v_mul_hi_u32 v9, v1, v6
+; CGP-NEXT: v_mul_hi_u32 v10, v0, v7
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v1, v6
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
; CGP-NEXT: v_mul_lo_u32 v5, v1, v7
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v10
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CGP-NEXT: v_mul_hi_u32 v9, v1, v7
; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
; CGP-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
; CGP-NEXT: v_mov_b32_e32 v8, 0xffed2705
; CGP-NEXT: v_mul_hi_u32 v9, v4, v8
; CGP-NEXT: v_mul_lo_u32 v5, v5, v8
; CGP-NEXT: v_mul_lo_u32 v10, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v11, v2, v7
; CGP-NEXT: v_sub_i32_e32 v4, vcc, v9, v4
; CGP-NEXT: v_mul_hi_u32 v9, v2, v6
; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v5
; CGP-NEXT: v_mul_lo_u32 v4, v3, v6
; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v11
+; CGP-NEXT: v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT: v_mul_hi_u32 v11, v2, v7
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v5, v3, v6
-; CGP-NEXT: v_mul_hi_u32 v6, v2, v7
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v9, v4
-; CGP-NEXT: v_mul_lo_u32 v9, v3, v7
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_mul_lo_u32 v5, v3, v7
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v9, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; CGP-NEXT: v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; CGP-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v10
@@ -1225,22 +1156,19 @@ define i64 @v_urem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v13, v0, v8
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v13
; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
@@ -1254,22 +1182,19 @@ define i64 @v_urem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
; CHECK-NEXT: v_mul_lo_u32 v10, v1, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT: v_mul_hi_u32 v11, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
@@ -1277,22 +1202,19 @@ define i64 @v_urem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-NEXT: v_mul_lo_u32 v7, v3, v1
; CHECK-NEXT: v_mul_lo_u32 v8, v4, v0
; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mul_hi_u32 v9, v3, v1
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v8, v3, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v7, vcc
; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; CHECK-NEXT: v_mul_hi_u32 v1, v4, v1
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v7, vcc
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v5, v0
; CHECK-NEXT: v_mul_lo_u32 v1, v5, v1
@@ -1386,22 +1308,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v16, v4, v13
; GISEL-NEXT: v_mul_lo_u32 v17, v5, v15
; GISEL-NEXT: v_mul_hi_u32 v15, v5, v15
+; GISEL-NEXT: v_mul_hi_u32 v18, v4, v13
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v16
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v17
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v17, v4, v13
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v16, vcc
; GISEL-NEXT: v_mul_lo_u32 v16, v5, v13
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v17, v16
; GISEL-NEXT: v_mul_hi_u32 v13, v5, v13
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v14
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
@@ -1415,22 +1334,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v14, v4, v12
; GISEL-NEXT: v_mul_lo_u32 v15, v5, v11
; GISEL-NEXT: v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v16, v4, v12
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v4, v12
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_mul_lo_u32 v14, v5, v12
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v15
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v16
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v15, v14
; GISEL-NEXT: v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v4, v11
; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v5, v12, vcc
@@ -1438,24 +1354,20 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v14, v0, v12
; GISEL-NEXT: v_lshl_b64 v[4:5], v[9:10], v6
; GISEL-NEXT: v_mul_lo_u32 v6, v1, v11
-; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v5
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, v11
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v13, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v1, v11
-; GISEL-NEXT: v_mul_hi_u32 v11, v0, v12
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v6
-; GISEL-NEXT: v_mul_lo_u32 v10, v1, v12
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, v12
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v13
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v11, vcc
; GISEL-NEXT: v_mul_hi_u32 v11, v1, v12
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
; GISEL-NEXT: v_mul_hi_u32 v10, v7, v6
; GISEL-NEXT: v_mul_lo_u32 v9, v7, v9
@@ -1473,6 +1385,7 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v8
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc
+; GISEL-NEXT: v_cvt_f32_u32_e32 v15, v5
; GISEL-NEXT: v_cndmask_b32_e64 v6, v6, v10, s[4:5]
; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v0, v7
; GISEL-NEXT: v_subbrev_u32_e64 v11, s[4:5], 0, v1, vcc
@@ -1507,23 +1420,20 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_hi_u32 v15, v8, v16
; GISEL-NEXT: v_mul_lo_u32 v17, v8, v14
; GISEL-NEXT: v_mul_lo_u32 v11, v10, v16
+; GISEL-NEXT: v_mul_hi_u32 v16, v10, v16
+; GISEL-NEXT: v_mul_hi_u32 v18, v8, v14
; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v15, v10, v16
-; GISEL-NEXT: v_mul_hi_u32 v16, v8, v14
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v17, v11
-; GISEL-NEXT: v_mul_lo_u32 v17, v10, v14
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT: v_mul_lo_u32 v15, v10, v14
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v18
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
; GISEL-NEXT: v_mul_hi_u32 v14, v10, v14
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v15
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11
; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v14, vcc
@@ -1539,22 +1449,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v6, v10, v12
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT: v_mul_hi_u32 v14, v8, v11
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v8, v11
-; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v13, v6
-; GISEL-NEXT: v_mul_lo_u32 v13, v10, v11
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_mul_lo_u32 v7, v10, v11
+; GISEL-NEXT: v_addc_u32_e64 v6, s[4:5], 0, v13, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v12, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v12, v7
; GISEL-NEXT: v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v13, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], 0, v12, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v11, v7
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v8, v6
; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v10, v7, s[4:5]
@@ -1564,21 +1471,18 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-NEXT: v_mul_lo_u32 v9, v3, v6
; GISEL-NEXT: v_mul_hi_u32 v6, v3, v6
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v7
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v2, v7
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_mul_lo_u32 v10, v3, v7
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
; GISEL-NEXT: v_mul_hi_u32 v7, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; GISEL-NEXT: v_mul_hi_u32 v8, v4, v6
; GISEL-NEXT: v_mul_lo_u32 v7, v4, v7
@@ -1655,22 +1559,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v16, v0, v13
; CGP-NEXT: v_mul_lo_u32 v17, v1, v15
; CGP-NEXT: v_mul_hi_u32 v15, v1, v15
+; CGP-NEXT: v_mul_hi_u32 v18, v0, v13
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v17
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v17, v0, v13
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v16, v14
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v16, vcc
; CGP-NEXT: v_mul_lo_u32 v16, v1, v13
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v18
; CGP-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v16
-; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v17, v16
; CGP-NEXT: v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT: v_addc_u32_e32 v16, vcc, 0, v17, vcc
; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
-; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v14
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
@@ -1684,22 +1585,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v14, v0, v12
; CGP-NEXT: v_mul_lo_u32 v15, v1, v4
; CGP-NEXT: v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v14
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v15, v0, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_mul_lo_u32 v14, v1, v12
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v15
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v16
; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v14
-; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v15, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
@@ -1707,22 +1605,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v12, v8, v1
; CGP-NEXT: v_mul_lo_u32 v13, v9, v0
; CGP-NEXT: v_mul_hi_u32 v0, v9, v0
+; CGP-NEXT: v_mul_hi_u32 v14, v8, v1
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v13, v8, v1
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v12, vcc
; CGP-NEXT: v_mul_lo_u32 v12, v9, v1
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v14
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; CGP-NEXT: v_mul_hi_u32 v1, v9, v1
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
; CGP-NEXT: v_mul_hi_u32 v4, v2, v0
; CGP-NEXT: v_mul_lo_u32 v1, v2, v1
@@ -1821,22 +1716,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v13, v2, v8
; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v15, v2, v8
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v14, v2, v8
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
; CGP-NEXT: v_mul_lo_u32 v13, v3, v8
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; CGP-NEXT: v_mul_hi_u32 v8, v3, v8
+; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
@@ -1850,22 +1742,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v11, v2, v6
; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v6
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v12, v2, v6
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
; CGP-NEXT: v_mul_lo_u32 v11, v3, v6
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v13
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
; CGP-NEXT: v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc
@@ -1873,22 +1762,19 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v6, v5, v3
; CGP-NEXT: v_mul_lo_u32 v8, v7, v2
; CGP-NEXT: v_mul_hi_u32 v2, v7, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v5, v3
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v6
; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v8, v5, v3
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
; CGP-NEXT: v_mul_lo_u32 v6, v7, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CGP-NEXT: v_mul_hi_u32 v3, v7, v3
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_mul_hi_u32 v4, v9, v2
; CGP-NEXT: v_mul_lo_u32 v3, v9, v3
@@ -2013,11 +1899,11 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-LABEL: v_urem_v2i64_24bit:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v4
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v1, 0
-; GISEL-NEXT: v_sub_i32_e32 v7, vcc, 0, v4
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; GISEL-NEXT: v_sub_i32_e32 v7, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0
@@ -2038,22 +1924,19 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v12, v3, v9
; GISEL-NEXT: v_mul_lo_u32 v13, v5, v11
; GISEL-NEXT: v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v14, v3, v9
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v3, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v12, vcc
; GISEL-NEXT: v_mul_lo_u32 v12, v5, v9
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; GISEL-NEXT: v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v13, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v3, v10
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
@@ -2069,64 +1952,60 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v6, v5, v7
; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v10, v8
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v9, v10, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v11, v6
-; GISEL-NEXT: v_mul_lo_u32 v11, v5, v8
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; GISEL-NEXT: v_mul_lo_u32 v9, v5, v8
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v12
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; GISEL-NEXT: v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v8, v7
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v6
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
; GISEL-NEXT: v_mul_hi_u32 v7, v0, v6
-; GISEL-NEXT: v_mul_lo_u32 v8, v0, v5
; GISEL-NEXT: v_mul_hi_u32 v6, 0, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, v0, v5
-; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v3
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_lo_u32 v8, 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT: v_mul_lo_u32 v10, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v9, v0, v5
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v4, v6
-; GISEL-NEXT: v_mul_lo_u32 v5, v4, v5
-; GISEL-NEXT: v_mul_lo_u32 v8, v4, v6
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT: v_mul_hi_u32 v7, v1, v6
+; GISEL-NEXT: v_mul_lo_u32 v5, v1, v5
+; GISEL-NEXT: v_mul_lo_u32 v8, v1, v6
; GISEL-NEXT: v_mul_lo_u32 v6, 0, v6
-; GISEL-NEXT: v_mac_f32_e32 v10, 0x4f800000, v1
+; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v3
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
; GISEL-NEXT: v_subb_u32_e64 v6, s[4:5], 0, v5, vcc
; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], 0, v5
; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v0, v4
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v0, v1
; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v4
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v10
+; GISEL-NEXT: v_mac_f32_e32 v10, 0x4f800000, v4
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v1
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v10
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v1
; GISEL-NEXT: v_cndmask_b32_e32 v9, -1, v9, vcc
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v8, v1
; GISEL-NEXT: v_subbrev_u32_e32 v11, vcc, 0, v5, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v1
+; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v4
; GISEL-NEXT: v_trunc_f32_e32 v8, v8
-; GISEL-NEXT: v_mac_f32_e32 v1, 0xcf800000, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
; GISEL-NEXT: v_cvt_u32_f32_e32 v8, v8
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
@@ -2134,36 +2013,33 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_sub_i32_e32 v9, vcc, 0, v3
; GISEL-NEXT: v_cndmask_b32_e64 v7, -1, v7, s[4:5]
; GISEL-NEXT: v_subb_u32_e64 v10, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mul_hi_u32 v11, v9, v1
+; GISEL-NEXT: v_mul_hi_u32 v11, v9, v4
; GISEL-NEXT: v_mul_lo_u32 v12, v9, v8
-; GISEL-NEXT: v_mul_lo_u32 v14, v10, v1
-; GISEL-NEXT: v_mul_lo_u32 v13, v9, v1
+; GISEL-NEXT: v_mul_lo_u32 v14, v10, v4
+; GISEL-NEXT: v_mul_lo_u32 v13, v9, v4
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; GISEL-NEXT: v_mul_hi_u32 v12, v1, v13
-; GISEL-NEXT: v_mul_lo_u32 v14, v1, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v4, v13
+; GISEL-NEXT: v_mul_lo_u32 v14, v4, v11
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GISEL-NEXT: v_mul_lo_u32 v4, v8, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v1, v8, v13
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v12, v14
+; GISEL-NEXT: v_mul_hi_u32 v13, v8, v13
+; GISEL-NEXT: v_mul_hi_u32 v14, v4, v11
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v7, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v7, v8, v13
-; GISEL-NEXT: v_mul_hi_u32 v13, v1, v11
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v12, v4
-; GISEL-NEXT: v_mul_lo_u32 v12, v8, v11
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v7, v1
+; GISEL-NEXT: v_mul_lo_u32 v7, v8, v11
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], 0, v12, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v13, v14
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v13, v12
-; GISEL-NEXT: v_mul_hi_u32 v11, v8, v11
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v7, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT: v_mul_hi_u32 v11, v8, v11
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v13, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v7, v1
+; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], 0, v12, s[4:5]
; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v11, v7
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v1, v4
+; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v1
; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v8, v7, s[4:5]
; GISEL-NEXT: v_mul_hi_u32 v1, v9, v4
; GISEL-NEXT: v_mul_lo_u32 v8, v9, v7
@@ -2175,37 +2051,33 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v11, v4, v8
; GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v5, vcc
; GISEL-NEXT: v_mul_lo_u32 v5, v7, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, v9
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v4, v8
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v6, v7, v9
-; GISEL-NEXT: v_mul_hi_u32 v9, v4, v8
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; GISEL-NEXT: v_mul_lo_u32 v10, v7, v8
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT: v_mul_lo_u32 v6, v7, v8
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
; GISEL-NEXT: v_mul_hi_u32 v8, v7, v8
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc
; GISEL-NEXT: v_mul_hi_u32 v6, v2, v4
-; GISEL-NEXT: v_mul_lo_u32 v7, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
-; GISEL-NEXT: v_mul_hi_u32 v8, v2, v5
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v7, v2, v5
+; GISEL-NEXT: v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT: v_mul_lo_u32 v8, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
; GISEL-NEXT: v_mul_hi_u32 v6, v3, v4
; GISEL-NEXT: v_mul_lo_u32 v5, v3, v5
; GISEL-NEXT: v_mul_lo_u32 v7, v3, v4
diff --git a/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
index 771374648fe9748..06f992c9fc4973b 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
@@ -5,41 +5,22 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
define amdgpu_ps void @s_uaddo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: s_uaddo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s1
-; GFX9-SDAG-NEXT: s_addc_u32 s0, s0, 0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: s_uaddo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s1
-; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GFX9-GISEL-NEXT: s_add_i32 s0, s0, s1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: s_uaddo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_add_co_u32 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT: s_add_co_ci_u32 s0, s0, 0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: s_uaddo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, s1
-; GFX12-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: s_uaddo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_add_u32 s0, s0, s1
+; GFX9-NEXT: s_addc_u32 s0, s0, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_uaddo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_add_co_u32 s0, s0, s1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_add_co_ci_u32 s0, s0, 0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%sum = extractvalue { i32, i1 } %o, 0
%c = extractvalue { i32, i1 } %o, 1
@@ -50,37 +31,20 @@ define amdgpu_ps void @s_uaddo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1
}
define amdgpu_ps void @v_uaddo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: v_uaddo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
-; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: v_uaddo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: v_uaddo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: v_uaddo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: v_uaddo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_uaddo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%sum = extractvalue { i32, i1 } %o, 0
%c = extractvalue { i32, i1 } %o, 1
@@ -91,41 +55,22 @@ define amdgpu_ps void @v_uaddo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
}
define amdgpu_ps void @s_usubo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: s_usubo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_sub_u32 s0, s0, s1
-; GFX9-SDAG-NEXT: s_subb_u32 s0, s0, 0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: s_usubo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_sub_u32 s0, s0, s1
-; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, s1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: s_usubo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_sub_co_u32 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT: s_sub_co_ci_u32 s0, s0, 0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: s_usubo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_sub_co_u32 s0, s0, s1
-; GFX12-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: s_usubo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_sub_u32 s0, s0, s1
+; GFX9-NEXT: s_subb_u32 s0, s0, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_usubo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_sub_co_u32 s0, s0, s1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_sub_co_ci_u32 s0, s0, 0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
%diff = extractvalue { i32, i1 } %o, 0
%c = extractvalue { i32, i1 } %o, 1
@@ -136,37 +81,20 @@ define amdgpu_ps void @s_usubo_carry(i32 inreg %a, i32 inreg %b, ptr addrspace(1
}
define amdgpu_ps void @v_usubo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: v_usubo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
-; GFX9-SDAG-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
-; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: v_usubo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-GISEL-NEXT: v_sub_u32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: v_usubo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: v_usubo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_sub_co_u32 v0, s0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX12-GISEL-NEXT: v_sub_nc_u32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: v_usubo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_usubo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
%diff = extractvalue { i32, i1 } %o, 0
%c = extractvalue { i32, i1 } %o, 1
@@ -177,37 +105,20 @@ define amdgpu_ps void @v_usubo_carry(i32 %a, i32 %b, ptr addrspace(1) %out) {
}
define amdgpu_ps void @v_uaddo_carry_commuted(i32 %a, i32 %b, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: v_uaddo_carry_commuted:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
-; GFX9-SDAG-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: v_uaddo_carry_commuted:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v1, v0
-; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: v_uaddo_carry_commuted:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: v_uaddo_carry_commuted:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v1, v0
-; GFX12-GISEL-NEXT: global_store_b32 v[2:3], v0, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: v_uaddo_carry_commuted:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v0, vcc
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_uaddo_carry_commuted:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v0, vcc_lo
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%sum = extractvalue { i32, i1 } %o, 0
%c = extractvalue { i32, i1 } %o, 1
@@ -218,37 +129,20 @@ define amdgpu_ps void @v_uaddo_carry_commuted(i32 %a, i32 %b, ptr addrspace(1) %
}
define amdgpu_ps void @v_add_of_usubo_carry(i32 %a, i32 %b, i32 %x, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: v_add_of_usubo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
-; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
-; GFX9-SDAG-NEXT: global_store_dword v[3:4], v0, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: v_add_of_usubo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-GISEL-NEXT: v_add_u32_e32 v0, v2, v0
-; GFX9-GISEL-NEXT: global_store_dword v[3:4], v0, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: v_add_of_usubo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[3:4], v0, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: v_add_of_usubo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_sub_co_u32 v0, s0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0
-; GFX12-GISEL-NEXT: global_store_b32 v[3:4], v0, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: v_add_of_usubo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_add_of_usubo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
+; GFX12-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
%c = extractvalue { i32, i1 } %o, 1
%ce = zext i1 %c to i32
@@ -258,37 +152,20 @@ define amdgpu_ps void @v_add_of_usubo_carry(i32 %a, i32 %b, i32 %x, ptr addrspac
}
define amdgpu_ps void @v_sub_of_uaddo_carry(i32 %a, i32 %b, i32 %x, ptr addrspace(1) %out) {
-; GFX9-SDAG-LABEL: v_sub_of_uaddo_carry:
-; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-SDAG-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v2, vcc
-; GFX9-SDAG-NEXT: global_store_dword v[3:4], v0, off
-; GFX9-SDAG-NEXT: s_endpgm
-;
-; GFX9-GISEL-LABEL: v_sub_of_uaddo_carry:
-; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-GISEL-NEXT: v_sub_u32_e32 v0, v2, v0
-; GFX9-GISEL-NEXT: global_store_dword v[3:4], v0, off
-; GFX9-GISEL-NEXT: s_endpgm
-;
-; GFX12-SDAG-LABEL: v_sub_of_uaddo_carry:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[3:4], v0, off
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: v_sub_of_uaddo_carry:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, s0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX12-GISEL-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX12-GISEL-NEXT: global_store_b32 v[3:4], v0, off
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX9-LABEL: v_sub_of_uaddo_carry:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_subbrev_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_sub_of_uaddo_carry:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_subrev_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
+; GFX12-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-NEXT: s_endpgm
%o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
%c = extractvalue { i32, i1 } %o, 1
%ce = zext i1 %c to i32
@@ -347,6 +224,3 @@ define amdgpu_ps void @v_uaddo_carry_multi_use(i32 %a, i32 %b, ptr addrspace(1)
store i32 %r2, ptr addrspace(1) %out
ret void
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12: {{.*}}
-; GFX9: {{.*}}
>From edf5dc3342a9a0409196afb2a779d92afc07b0ae Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Thu, 24 Sep 2026 18:19:25 +0200
Subject: [PATCH 3/4] update RUN line
---
.../postlegalizer-combiner-carry-into-uadde-usube.mir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
index 4b30c9f5cc3c7a1..44f6d58b5510137 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-carry-into-uadde-usube.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs -o - %s | FileCheck %s
---
name: add_zext_uaddo_carry
>From 69ed4d83d7602d5fb1b1426964987753e983e036 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 25 Sep 2026 18:15:20 +0200
Subject: [PATCH 4/4] update
---
.../AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 4 +--
.../AMDGPU/combine-carry-into-uadde-usube.ll | 30 +++++++++++++++++++
2 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index b18ef441bf02583..8304aaf901aab75 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -423,8 +423,8 @@ bool AMDGPUPostLegalizerCombinerImpl::shouldFoldCarryIntoAdd(
// When both operands are a carry only the right hand one is folded, so that
// the two carries never have to be live at the same time.
Register Carry;
- return !(mi_match(Src, MRI, m_GZExt(m_Reg(Carry))) &&
- MRI.hasOneNonDBGUse(Src) && isCarryOut(Carry));
+ return !mi_match(Src, MRI, m_GZExt(m_Reg(Carry))) ||
+ !MRI.hasOneNonDBGUse(Src) || !isCarryOut(Carry);
}
// Pass boilerplate
diff --git a/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
index 06f992c9fc4973b..d9c82226f10ab66 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-carry-into-uadde-usube.ll
@@ -224,3 +224,33 @@ define amdgpu_ps void @v_uaddo_carry_multi_use(i32 %a, i32 %b, ptr addrspace(1)
store i32 %r2, ptr addrspace(1) %out
ret void
}
+
+define amdgpu_ps void @v_uaddo_carry_commuted_rhs_zext_icmp(i32 %a, i32 %b, i32 %x, ptr addrspace(1) %out) {
+; GFX9-LABEL: v_uaddo_carry_commuted_rhs_zext_icmp:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT: global_store_dword v[3:4], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: v_uaddo_carry_commuted_rhs_zext_icmp:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v0, null, 0, v2, vcc_lo
+; GFX12-NEXT: global_store_b32 v[3:4], v0, off
+; GFX12-NEXT: s_endpgm
+ %o = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+ %c = extractvalue { i32, i1 } %o, 1
+ %ce = zext i1 %c to i32
+ %cmp = icmp eq i32 %x, %a
+ %cmpe = zext i1 %cmp to i32
+ %r = add i32 %ce, %cmpe
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
More information about the llvm-commits
mailing list