[llvm] [AMDGPU] Implement int div/rem by fp recip more efficiently (PR #204950)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 01:45:58 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/204950
>From 30b879e161958286f6a42db6425fb54c4ca8324c Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Thu, 11 Jun 2026 15:15:16 -0500
Subject: [PATCH 1/3] WIP
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 124 ++++++++++++------
1 file changed, 86 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 3d59e6af41db3..f551f23aa1d3b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1093,21 +1093,44 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
Num = Builder.CreateTrunc(Num, I32Ty);
Den = Builder.CreateTrunc(Den, I32Ty);
+ // The calculation:
+ // fq = fa*recip(fb)
+ // may be too small due to the 1ulp accuracy in the recip
+ // operation and rounding issues. Since fq is truncated to produce
+ // an integer value it may be too small by one. This
+ // can be dealt with in one of two ways.
+ //
+ // 1. If fabs(fa) <= 0x200000 increment recip(fb) by 1ulp.
+ // fq = fa*(recip(fb)+1ulp)
+ // This method is preferred since it only requires one
+ // integer +1 operation on the floating-point reciprocal encoding.
+ // The maximum possible error is:
+ // fa * (2.5ulp)/0x800000
+ // => 0x200000 * (2.5ulp)/0x800000
+ // => 0x500000/0x800000
+ // so the calculated fq can exceed
+ //
+ // 2. A. Calculate fr = fa - fq*fb
+ // B. If fabs(fr) >= fabs(fb) increment fq by 1.0.
+
+
+ // Since fb represents an integer within a 24-bit
+ // range it is always safe to calculate fb' by decrementing the
+ // binary representation of fb by one. Since fabs(fb')<fabs(fb)
+ // fq' will never be too small and trunc(fq') will produce the desired value.
+ // Since the magnitude of the ratio between fb' and fb is at most 1/0x80000
+ // (note: 0x800000 is the smallest fp32 mantissa),
+ // fq' is bigger than fq by at most num/0x800000. Thus, fq' will
+ // be off from the desired value by at most num/0x800000 + num/0x800000
+ // Thus is is safe to use a smaller
+ // denominator if the maximum possible fabs(N) is 0x200000.
+ bool IncrementFA = (DivBits < (IsSigned ? 23 : 22));
+
+ dbgs() << "XXXXXXXXXXXXXXXXXXXXXX DivBits=" << DivBits << " IsSigned=" << IsSigned << " IncrementFA=" << IncrementFA << "\n";
+
Type *F32Ty = Builder.getFloatTy();
ConstantInt *One = Builder.getInt32(1);
- Value *JQ = One;
-
- if (IsSigned) {
- // char|short jq = ia ^ ib;
- JQ = Builder.CreateXor(Num, Den);
-
- // jq = jq >> (bitsize - 2)
- JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
-
- // jq = jq | 0x1
- JQ = Builder.CreateOr(JQ, One);
- }
-
+
// int ia = (int)LHS;
Value *IA = Num;
@@ -1120,46 +1143,71 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// float fb = (float)ib;
Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
- : Builder.CreateUIToFP(IB,F32Ty);
-
+ : Builder.CreateUIToFP(IB,F32Ty);
+
Value *RCP = Builder.CreateIntrinsic(Intrinsic::amdgcn_rcp,
Builder.getFloatTy(), {FB});
+
+ if (IncrementFA) {
+ Value* FABits = Builder.CreateBitCast(FA, I32Ty);
+ Value* FABitsInc = Builder.CreateAdd(FABits, One);
+ FA = Builder.CreateBitCast(FABitsInc, F32Ty);
+ }
+
Value *FQM = Builder.CreateFMul(FA, RCP);
// fq = trunc(fqm);
Value *FQ = Builder.CreateUnaryIntrinsic(Intrinsic::trunc, FQM);
- auto *FQI = dyn_cast<Instruction>(FQ);
- if (FQI)
- FQI->copyFastMathFlags(Builder.getFastMathFlags());
-
- // float fqneg = -fq;
- Value *FQNeg = Builder.CreateFNeg(FQ);
-
- // float fr = mad(fqneg, fb, fa);
- auto FMAD = !ST.hasMadMacF32Insts()
- ? Intrinsic::fma
- : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
- Value *FR =
- Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
// int iq = (int)fq;
Value *IQ = IsSigned ? Builder.CreateFPToSI(FQ, I32Ty)
: Builder.CreateFPToUI(FQ, I32Ty);
- // fr = fabs(fr);
- FR = Builder.CreateFAbs(FR, FQI);
+ Value *Div;
+ if (IncrementFA) {
+ Div = IQ;
+ } else {
+ Value *JQ = One;
+ auto *FQI = dyn_cast<Instruction>(FQ);
+ if (FQI)
+ FQI->copyFastMathFlags(Builder.getFastMathFlags());
+
+ if (IsSigned) {
+ // char|short jq = ia ^ ib;
+ JQ = Builder.CreateXor(Num, Den);
- // fb = fabs(fb);
- FB = Builder.CreateFAbs(FB, FQI);
+ // jq = jq >> (bitsize - 2)
+ JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
- // int cv = fr >= fb;
- Value *CV = Builder.CreateFCmpOGE(FR, FB);
+ // jq = jq | 0x1
+ JQ = Builder.CreateOr(JQ, One);
+ }
- // jq = (cv ? jq : 0);
- JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
+ // float fqneg = -fq;
+ Value *FQNeg = Builder.CreateFNeg(FQ);
- // dst = iq + jq;
- Value *Div = Builder.CreateAdd(IQ, JQ);
+ // float fr = mad(fqneg, fb, fa);
+ auto FMAD = !ST.hasMadMacF32Insts()
+ ? Intrinsic::fma
+ : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
+ Value *FR =
+ Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
+
+ // fr = fabs(fr);
+ FR = Builder.CreateFAbs(FR, FQI);
+
+ // fb = fabs(fb);
+ FB = Builder.CreateFAbs(FB, FQI);
+
+ // int cv = fr >= fb;
+ Value *CV = Builder.CreateFCmpOGE(FR, FB);
+
+ // jq = (cv ? jq : 0);
+ JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
+
+ // dst = iq + jq;
+ Div = Builder.CreateAdd(IQ, JQ);
+ }
Value *Res = Div;
if (!IsDiv) {
>From 6e07497feed7ce4d33e3edd89764dc7403c41ebb Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Sat, 20 Jun 2026 15:47:36 -0500
Subject: [PATCH 2/3] Use fp recip to do int div/rem more efficiently
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 88 +-
.../AMDGPU/amdgpu-codegenprepare-idiv.ll | 3338 +++++++----------
llvm/test/CodeGen/AMDGPU/divrem24-assume.ll | 16 +-
llvm/test/CodeGen/AMDGPU/idiv-licm.ll | 355 +-
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 680 ++--
llvm/test/CodeGen/AMDGPU/sdiv.ll | 154 +-
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 80 +-
llvm/test/CodeGen/AMDGPU/srem64.ll | 8 +-
llvm/test/CodeGen/AMDGPU/udiv.ll | 194 +-
llvm/test/CodeGen/AMDGPU/udivrem24.ll | 170 +-
10 files changed, 1993 insertions(+), 3090 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index f551f23aa1d3b..329b9924e667f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1093,44 +1093,9 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
Num = Builder.CreateTrunc(Num, I32Ty);
Den = Builder.CreateTrunc(Den, I32Ty);
- // The calculation:
- // fq = fa*recip(fb)
- // may be too small due to the 1ulp accuracy in the recip
- // operation and rounding issues. Since fq is truncated to produce
- // an integer value it may be too small by one. This
- // can be dealt with in one of two ways.
- //
- // 1. If fabs(fa) <= 0x200000 increment recip(fb) by 1ulp.
- // fq = fa*(recip(fb)+1ulp)
- // This method is preferred since it only requires one
- // integer +1 operation on the floating-point reciprocal encoding.
- // The maximum possible error is:
- // fa * (2.5ulp)/0x800000
- // => 0x200000 * (2.5ulp)/0x800000
- // => 0x500000/0x800000
- // so the calculated fq can exceed
- //
- // 2. A. Calculate fr = fa - fq*fb
- // B. If fabs(fr) >= fabs(fb) increment fq by 1.0.
-
-
- // Since fb represents an integer within a 24-bit
- // range it is always safe to calculate fb' by decrementing the
- // binary representation of fb by one. Since fabs(fb')<fabs(fb)
- // fq' will never be too small and trunc(fq') will produce the desired value.
- // Since the magnitude of the ratio between fb' and fb is at most 1/0x80000
- // (note: 0x800000 is the smallest fp32 mantissa),
- // fq' is bigger than fq by at most num/0x800000. Thus, fq' will
- // be off from the desired value by at most num/0x800000 + num/0x800000
- // Thus is is safe to use a smaller
- // denominator if the maximum possible fabs(N) is 0x200000.
- bool IncrementFA = (DivBits < (IsSigned ? 23 : 22));
-
- dbgs() << "XXXXXXXXXXXXXXXXXXXXXX DivBits=" << DivBits << " IsSigned=" << IsSigned << " IncrementFA=" << IncrementFA << "\n";
-
Type *F32Ty = Builder.getFloatTy();
ConstantInt *One = Builder.getInt32(1);
-
+
// int ia = (int)LHS;
Value *IA = Num;
@@ -1142,18 +1107,41 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
: Builder.CreateUIToFP(IA, F32Ty);
// float fb = (float)ib;
- Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
- : Builder.CreateUIToFP(IB,F32Ty);
-
+ Value *FB = IsSigned ? Builder.CreateSIToFP(IB, F32Ty)
+ : Builder.CreateUIToFP(IB, F32Ty);
+
Value *RCP = Builder.CreateIntrinsic(Intrinsic::amdgcn_rcp,
Builder.getFloatTy(), {FB});
+ // The calculation:
+ // fq = fa*recip(fb)
+ // may be too small due to the 1ulp accuracy in the recip
+ // operation and rounding issues. Since fq is truncated to produce
+ // an integer value it may be too small by one. This
+ // can be dealt with in one of two ways.
+ //
+ // 1. If fabs(fa) <= 0x200000 increment fa by 1ulp:
+ // fq = (fa+1ulp)*recip(fb)
+ // This method is preferred since it only requires one
+ // integer +1 operation on the floating-point encoding of fa.
+ // This will increase fa's magnitude by at most 0.25
+ // (i.e. when fabs(fa)==0x200000 the LSB of the mantissa represents 0.25).
+ // Thus, this method is safe since fa must be incremented by at least 1.0
+ // for quotient to increase by one. Note that setting the boundary to
+ // fabs(fa) <= 0x400000 and incrementing fa's magnitude by at most 0.5 is
+ // unsafe because recip(fb) can have a 1 ulp error.
+ //
+ // 2. A. Calculate fr = fa - fq*fb
+ // B. If fabs(fr) >= fabs(fb) increment the quotient by 1.
+
+ bool IncrementFA = (DivBits < (IsSigned ? 23 : 22));
+
if (IncrementFA) {
- Value* FABits = Builder.CreateBitCast(FA, I32Ty);
- Value* FABitsInc = Builder.CreateAdd(FABits, One);
+ Value *FABits = Builder.CreateBitCast(FA, I32Ty);
+ Value *FABitsInc = Builder.CreateAdd(FABits, One);
FA = Builder.CreateBitCast(FABitsInc, F32Ty);
}
-
+
Value *FQM = Builder.CreateFMul(FA, RCP);
// fq = trunc(fqm);
@@ -1165,13 +1153,13 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
Value *Div;
if (IncrementFA) {
- Div = IQ;
+ Div = IQ;
} else {
Value *JQ = One;
auto *FQI = dyn_cast<Instruction>(FQ);
if (FQI)
FQI->copyFastMathFlags(Builder.getFastMathFlags());
-
+
if (IsSigned) {
// char|short jq = ia ^ ib;
JQ = Builder.CreateXor(Num, Den);
@@ -1188,11 +1176,11 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// float fr = mad(fqneg, fb, fa);
auto FMAD = !ST.hasMadMacF32Insts()
- ? Intrinsic::fma
- : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
+ ? Intrinsic::fma
+ : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
Value *FR =
- Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
-
+ Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
+
// fr = fabs(fr);
FR = Builder.CreateFAbs(FR, FQI);
@@ -1201,10 +1189,10 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// int cv = fr >= fb;
Value *CV = Builder.CreateFCmpOGE(FR, FB);
-
+
// jq = (cv ? jq : 0);
JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
-
+
// dst = iq + jq;
Div = Builder.CreateAdd(IQ, JQ);
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index d83bb728e47f9..82b0459a6fb53 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -432,17 +432,13 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i16
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i16
; CHECK-NEXT: store i16 [[TMP17]], ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
@@ -457,35 +453,31 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: udiv_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s1, s0, 16
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1
; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v0, s[0:1]
+; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i16 %x, %y
store i16 %r, ptr addrspace(1) %out
@@ -500,17 +492,13 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
; CHECK-NEXT: store i16 [[TMP19]], ptr addrspace(1) [[OUT]], align 2
@@ -526,13 +514,11 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-NEXT: s_and_b32 s0, s6, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
@@ -548,14 +534,12 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -571,73 +555,54 @@ define amdgpu_kernel void @sdiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i16
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i16
; CHECK-NEXT: store i16 [[TMP21]], ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s4, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_ashr_i32 s1, s0, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_sext_i32_i16 s0, s0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: sdiv_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_ashr_i32 s1, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s3, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_sext_i32_i16 s2, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i16 %x, %y
@@ -650,23 +615,16 @@ define amdgpu_kernel void @srem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
; CHECK-NEXT: store i16 [[TMP23]], ptr addrspace(1) [[OUT]], align 2
@@ -675,54 +633,42 @@ define amdgpu_kernel void @srem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-LABEL: srem_i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s7, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX6-NEXT: s_sext_i32_i16 s2, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX6-NEXT: s_xor_b32 s2, s2, s7
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_or_b32 s4, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s4, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_ashr_i32 s2, s6, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX6-NEXT: s_sext_i32_i16 s0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: srem_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s7, s6, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX9-NEXT: s_sext_i32_i16 s2, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s7
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_ashr_i32 s3, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_sext_i32_i16 s0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i16 %x, %y
@@ -738,54 +684,46 @@ define amdgpu_kernel void @udiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i8
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i8
; CHECK-NEXT: store i8 [[TMP17]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: udiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0xb
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: udiv_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, s2
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
-; GFX9-NEXT: global_store_byte v2, v0, s[0:1]
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i8 %x, %y
store i8 %r, ptr addrspace(1) %out
@@ -800,17 +738,13 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i8
; CHECK-NEXT: store i8 [[TMP19]], ptr addrspace(1) [[OUT]], align 1
@@ -823,15 +757,13 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b32 s2, s6, 8
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
@@ -843,16 +775,14 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_lshr_b32 s3, s2, 8
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
@@ -867,73 +797,54 @@ define amdgpu_kernel void @sdiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i8 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i8
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i8
; CHECK-NEXT: store i8 [[TMP21]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i8 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_bfe_i32 s1, s0, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_sext_i32_i8 s0, s0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: sdiv_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bfe_i32 s1, s0, 0x80008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s3, s2, 0x80008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_sext_i32_i8 s2, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i8 %x, %y
@@ -946,23 +857,16 @@ define amdgpu_kernel void @srem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i8 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
-; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
-; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i8
; CHECK-NEXT: store i8 [[TMP23]], ptr addrspace(1) [[OUT]], align 1
@@ -971,56 +875,44 @@ define amdgpu_kernel void @srem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; GFX6-LABEL: srem_i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_sext_i32_i8 s3, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_sext_i32_i8 s0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: srem_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_sext_i32_i8 s3, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX9-NEXT: s_xor_b32 s2, s3, s2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_lshr_b32 s4, s6, 8
-; GFX9-NEXT: s_or_b32 s5, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s4
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x80008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: s_sext_i32_i8 s0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_lshr_b32 s3, s2, 8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i8 %x, %y
@@ -2454,17 +2346,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i16
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i16> poison, i16 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i16> [[Y]], i64 1
@@ -2473,17 +2361,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i16
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i16
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i16> [[TMP20]], i16 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i16> [[Y]], i64 2
@@ -2492,17 +2376,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i16
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i16
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <4 x i16> [[TMP40]], i16 [[TMP59]], i64 2
; CHECK-NEXT: [[TMP61:%.*]] = extractelement <4 x i16> [[X]], i64 3
; CHECK-NEXT: [[TMP62:%.*]] = extractelement <4 x i16> [[Y]], i64 3
@@ -2511,17 +2391,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP65:%.*]] = uitofp fast i32 [[TMP63]] to float
; CHECK-NEXT: [[TMP66:%.*]] = uitofp fast i32 [[TMP64]] to float
; CHECK-NEXT: [[TMP67:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP68:%.*]] = fmul fast float [[TMP65]], [[TMP67]]
+; CHECK-NEXT: [[TMP53:%.*]] = bitcast float [[TMP65]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP68:%.*]] = fmul fast float [[TMP55]], [[TMP67]]
; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.trunc.f32(float [[TMP68]])
-; CHECK-NEXT: [[TMP70:%.*]] = fneg fast float [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP70]], float [[TMP66]], float [[TMP65]])
; CHECK-NEXT: [[TMP72:%.*]] = fptoui float [[TMP69]] to i32
-; CHECK-NEXT: [[TMP73:%.*]] = call fast float @llvm.fabs.f32(float [[TMP71]])
-; CHECK-NEXT: [[TMP74:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP75:%.*]] = fcmp fast oge float [[TMP73]], [[TMP74]]
-; CHECK-NEXT: [[TMP76:%.*]] = select i1 [[TMP75]], i32 1, i32 0
-; CHECK-NEXT: [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP76]]
-; CHECK-NEXT: [[TMP79:%.*]] = trunc i32 [[TMP77]] to i16
+; CHECK-NEXT: [[TMP79:%.*]] = trunc i32 [[TMP72]] to i16
; CHECK-NEXT: [[TMP80:%.*]] = insertelement <4 x i16> [[TMP60]], i16 [[TMP79]], i64 3
; CHECK-NEXT: store <4 x i16> [[TMP80]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -2535,112 +2411,95 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_lshr_b32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v2, v3
; GFX6-NEXT: s_lshr_b32 s4, s11, 16
-; GFX6-NEXT: v_mad_f32 v3, -v1, v4, v5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
; GFX6-NEXT: s_lshr_b32 s4, s9, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4
-; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_rcp_f32_e32 v7, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mul_f32_e32 v3, v6, v7
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: v_rcp_f32_e32 v4, v4
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX6-NEXT: v_mad_f32 v3, -v3, v5, v6
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: udiv_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_lshr_b32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v5, v7
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_mad_f32 v3, -v1, v4, v5
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s0
-; GFX9-NEXT: v_rcp_f32_e32 v8, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
+; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2657,17 +2516,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i16
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> poison, i16 [[TMP21]], i64 0
@@ -2678,17 +2533,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i16
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[TMP43]], i64 1
@@ -2699,17 +2550,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <4 x i16> [[TMP44]], i16 [[TMP65]], i64 2
@@ -2720,17 +2567,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP71:%.*]] = uitofp fast i32 [[TMP69]] to float
; CHECK-NEXT: [[TMP72:%.*]] = uitofp fast i32 [[TMP70]] to float
; CHECK-NEXT: [[TMP73:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP72]])
-; CHECK-NEXT: [[TMP74:%.*]] = fmul fast float [[TMP71]], [[TMP73]]
+; CHECK-NEXT: [[TMP59:%.*]] = bitcast float [[TMP71]] to i32
+; CHECK-NEXT: [[TMP60:%.*]] = add i32 [[TMP59]], 1
+; CHECK-NEXT: [[TMP61:%.*]] = bitcast i32 [[TMP60]] to float
+; CHECK-NEXT: [[TMP74:%.*]] = fmul fast float [[TMP61]], [[TMP73]]
; CHECK-NEXT: [[TMP75:%.*]] = call fast float @llvm.trunc.f32(float [[TMP74]])
-; CHECK-NEXT: [[TMP76:%.*]] = fneg fast float [[TMP75]]
-; CHECK-NEXT: [[TMP77:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP76]], float [[TMP72]], float [[TMP71]])
; CHECK-NEXT: [[TMP78:%.*]] = fptoui float [[TMP75]] to i32
-; CHECK-NEXT: [[TMP79:%.*]] = call fast float @llvm.fabs.f32(float [[TMP77]])
-; CHECK-NEXT: [[TMP80:%.*]] = call fast float @llvm.fabs.f32(float [[TMP72]])
-; CHECK-NEXT: [[TMP81:%.*]] = fcmp fast oge float [[TMP79]], [[TMP80]]
-; CHECK-NEXT: [[TMP82:%.*]] = select i1 [[TMP81]], i32 1, i32 0
-; CHECK-NEXT: [[TMP83:%.*]] = add i32 [[TMP78]], [[TMP82]]
-; CHECK-NEXT: [[TMP84:%.*]] = mul i32 [[TMP83]], [[TMP70]]
+; CHECK-NEXT: [[TMP84:%.*]] = mul i32 [[TMP78]], [[TMP70]]
; CHECK-NEXT: [[TMP85:%.*]] = sub i32 [[TMP69]], [[TMP84]]
; CHECK-NEXT: [[TMP87:%.*]] = trunc i32 [[TMP85]] to i16
; CHECK-NEXT: [[TMP88:%.*]] = insertelement <4 x i16> [[TMP66]], i16 [[TMP87]], i64 3
@@ -2744,63 +2587,55 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: s_and_b32 s4, s10, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mad_f32 v1, -v1, v2, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_and_b32 s6, s9, 0xffff
+; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s6, s9, 16
; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: s_and_b32 s5, s9, 0xffff
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: s_lshr_b32 s5, s11, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v1
-; GFX6-NEXT: s_lshr_b32 s4, s11, 16
-; GFX6-NEXT: v_mul_f32_e32 v1, v3, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX6-NEXT: v_trunc_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s4, v1
+; GFX6-NEXT: v_mul_lo_u32 v1, v3, s5
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: s_lshr_b32 s5, s9, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s5
-; GFX6-NEXT: v_rcp_f32_e32 v7, v4
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, v6, v7
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v6
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
@@ -2808,65 +2643,57 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
-; GFX9-NEXT: s_and_b32 s5, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s5
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_lshr_b32 s2, s3, 16
-; GFX9-NEXT: v_mad_f32 v3, -v2, v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v4
+; GFX9-NEXT: s_and_b32 s9, s3, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: s_and_b32 s8, s1, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s8
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_rcp_f32_e32 v8, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s1
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s4
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
-; GFX9-NEXT: v_sub_u32_e32 v4, s0, v1
-; GFX9-NEXT: v_sub_u32_e32 v1, s5, v2
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v3
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
+; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = urem <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2880,89 +2707,61 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i16
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> poison, i16 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i16 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i16 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i16
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i16
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <4 x i16> [[TMP24]], i16 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <4 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i16 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i16 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i16
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <4 x i16> [[TMP48]], i16 [[TMP71]], i64 2
; CHECK-NEXT: [[TMP73:%.*]] = extractelement <4 x i16> [[X]], i64 3
; CHECK-NEXT: [[TMP74:%.*]] = extractelement <4 x i16> [[Y]], i64 3
; CHECK-NEXT: [[TMP75:%.*]] = sext i16 [[TMP73]] to i32
; CHECK-NEXT: [[TMP76:%.*]] = sext i16 [[TMP74]] to i32
-; CHECK-NEXT: [[TMP77:%.*]] = xor i32 [[TMP75]], [[TMP76]]
-; CHECK-NEXT: [[TMP78:%.*]] = ashr i32 [[TMP77]], 30
-; CHECK-NEXT: [[TMP79:%.*]] = or i32 [[TMP78]], 1
; CHECK-NEXT: [[TMP80:%.*]] = sitofp fast i32 [[TMP75]] to float
; CHECK-NEXT: [[TMP81:%.*]] = sitofp fast i32 [[TMP76]] to float
; CHECK-NEXT: [[TMP82:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP81]])
-; CHECK-NEXT: [[TMP83:%.*]] = fmul fast float [[TMP80]], [[TMP82]]
+; CHECK-NEXT: [[TMP53:%.*]] = bitcast float [[TMP80]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP83:%.*]] = fmul fast float [[TMP55]], [[TMP82]]
; CHECK-NEXT: [[TMP84:%.*]] = call fast float @llvm.trunc.f32(float [[TMP83]])
-; CHECK-NEXT: [[TMP85:%.*]] = fneg fast float [[TMP84]]
-; CHECK-NEXT: [[TMP86:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP85]], float [[TMP81]], float [[TMP80]])
; CHECK-NEXT: [[TMP87:%.*]] = fptosi float [[TMP84]] to i32
-; CHECK-NEXT: [[TMP88:%.*]] = call fast float @llvm.fabs.f32(float [[TMP86]])
-; CHECK-NEXT: [[TMP89:%.*]] = call fast float @llvm.fabs.f32(float [[TMP81]])
-; CHECK-NEXT: [[TMP90:%.*]] = fcmp fast oge float [[TMP88]], [[TMP89]]
-; CHECK-NEXT: [[TMP91:%.*]] = select i1 [[TMP90]], i32 [[TMP79]], i32 0
-; CHECK-NEXT: [[TMP92:%.*]] = add i32 [[TMP87]], [[TMP91]]
-; CHECK-NEXT: [[TMP95:%.*]] = trunc i32 [[TMP92]] to i16
+; CHECK-NEXT: [[TMP95:%.*]] = trunc i32 [[TMP87]] to i16
; CHECK-NEXT: [[TMP96:%.*]] = insertelement <4 x i16> [[TMP72]], i16 [[TMP95]], i64 3
; CHECK-NEXT: store <4 x i16> [[TMP96]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -2974,76 +2773,48 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s5, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
; GFX6-NEXT: s_sext_i32_i16 s4, s9
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s11, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s9, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v5, v5
-; GFX6-NEXT: v_mad_f32 v4, -v5, v0, v4
-; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v2, v3
+; GFX6-NEXT: s_ashr_i32 s4, s11, 16
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
+; GFX6-NEXT: s_ashr_i32 s4, s9, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_rcp_f32_e32 v4, v4
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX6-NEXT: v_trunc_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: sdiv_v4i16:
@@ -3052,74 +2823,46 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: s_xor_b32 s4, s5, s4
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s8, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s8, 0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_add_u32_e32 v3, s4, v3
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v4
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v1, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v1, -v5, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: s_ashr_i32 s2, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: v_add_u32_e32 v1, s0, v5
-; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v6, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
-; GFX9-NEXT: v_trunc_f32_e32 v6, v6
-; GFX9-NEXT: v_mad_f32 v5, -v6, v0, v5
-; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v6
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: s_ashr_i32 s0, s3, 16
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
+; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <4 x i16> %x, %y
@@ -3134,23 +2877,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i16
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> poison, i16 [[TMP25]], i64 0
@@ -3158,23 +2894,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i16 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i16 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i16
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i16> [[TMP26]], i16 [[TMP51]], i64 1
@@ -3182,23 +2911,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i16 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i16 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i16
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <4 x i16> [[TMP52]], i16 [[TMP77]], i64 2
@@ -3206,23 +2928,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP80:%.*]] = extractelement <4 x i16> [[Y]], i64 3
; CHECK-NEXT: [[TMP81:%.*]] = sext i16 [[TMP79]] to i32
; CHECK-NEXT: [[TMP82:%.*]] = sext i16 [[TMP80]] to i32
-; CHECK-NEXT: [[TMP83:%.*]] = xor i32 [[TMP81]], [[TMP82]]
-; CHECK-NEXT: [[TMP84:%.*]] = ashr i32 [[TMP83]], 30
-; CHECK-NEXT: [[TMP85:%.*]] = or i32 [[TMP84]], 1
; CHECK-NEXT: [[TMP86:%.*]] = sitofp fast i32 [[TMP81]] to float
; CHECK-NEXT: [[TMP87:%.*]] = sitofp fast i32 [[TMP82]] to float
; CHECK-NEXT: [[TMP88:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP87]])
-; CHECK-NEXT: [[TMP89:%.*]] = fmul fast float [[TMP86]], [[TMP88]]
+; CHECK-NEXT: [[TMP59:%.*]] = bitcast float [[TMP86]] to i32
+; CHECK-NEXT: [[TMP65:%.*]] = add i32 [[TMP59]], 1
+; CHECK-NEXT: [[TMP66:%.*]] = bitcast i32 [[TMP65]] to float
+; CHECK-NEXT: [[TMP89:%.*]] = fmul fast float [[TMP66]], [[TMP88]]
; CHECK-NEXT: [[TMP90:%.*]] = call fast float @llvm.trunc.f32(float [[TMP89]])
-; CHECK-NEXT: [[TMP91:%.*]] = fneg fast float [[TMP90]]
-; CHECK-NEXT: [[TMP92:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP91]], float [[TMP87]], float [[TMP86]])
; CHECK-NEXT: [[TMP93:%.*]] = fptosi float [[TMP90]] to i32
-; CHECK-NEXT: [[TMP94:%.*]] = call fast float @llvm.fabs.f32(float [[TMP92]])
-; CHECK-NEXT: [[TMP95:%.*]] = call fast float @llvm.fabs.f32(float [[TMP87]])
-; CHECK-NEXT: [[TMP96:%.*]] = fcmp fast oge float [[TMP94]], [[TMP95]]
-; CHECK-NEXT: [[TMP97:%.*]] = select i1 [[TMP96]], i32 [[TMP85]], i32 0
-; CHECK-NEXT: [[TMP98:%.*]] = add i32 [[TMP93]], [[TMP97]]
-; CHECK-NEXT: [[TMP99:%.*]] = mul i32 [[TMP98]], [[TMP82]]
+; CHECK-NEXT: [[TMP99:%.*]] = mul i32 [[TMP93]], [[TMP82]]
; CHECK-NEXT: [[TMP100:%.*]] = sub i32 [[TMP81]], [[TMP99]]
; CHECK-NEXT: [[TMP103:%.*]] = trunc i32 [[TMP100]] to i16
; CHECK-NEXT: [[TMP104:%.*]] = insertelement <4 x i16> [[TMP78]], i16 [[TMP103]], i64 3
@@ -3238,85 +2953,57 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_sext_i32_i16 s4, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b32 s4, s10, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s6
+; GFX6-NEXT: s_ashr_i32 s6, s8, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_sext_i32_i16 s6, s11
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_sext_i32_i16 s6, s9
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s6
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: s_ashr_i32 s4, s11, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: s_ashr_i32 s4, s9, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s11
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s6, v1
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v2, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s11, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_ashr_i32 s5, s9, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s9, 16
-; GFX6-NEXT: s_lshr_b32 s7, s11, 16
-; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v5, v5
-; GFX6-NEXT: v_mad_f32 v4, -v5, v2, v4
-; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v5
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s7
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: s_lshr_b32 s4, s11, 16
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s5, v1
+; GFX6-NEXT: v_mul_lo_u32 v1, v2, s11
+; GFX6-NEXT: v_mul_lo_u32 v2, v3, s4
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: s_lshr_b32 s4, s9, 16
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s6, v2
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
@@ -3326,82 +3013,54 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s8, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
-; GFX9-NEXT: s_xor_b32 s4, s9, s8
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s10, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v4
+; GFX9-NEXT: s_sext_i32_i16 s9, s3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
+; GFX9-NEXT: s_sext_i32_i16 s8, s1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
+; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s1
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v1, s4, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s10
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT: s_sext_i32_i16 s8, s1
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v4
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: s_xor_b32 s0, s8, s2
-; GFX9-NEXT: v_rcp_f32_e32 v5, v3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v4, -v5, v3, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v3|
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s3
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v5
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: s_ashr_i32 s2, s1, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s2
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
-; GFX9-NEXT: s_xor_b32 s0, s2, s3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
-; GFX9-NEXT: v_trunc_f32_e32 v6, v6
-; GFX9-NEXT: v_mad_f32 v5, -v6, v4, v5
-; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v4|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v6
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT: v_sub_u32_e32 v5, s9, v1
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_sub_u32_e32 v3, s2, v4
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = srem <4 x i16> %x, %y
@@ -3417,38 +3076,32 @@ define amdgpu_kernel void @udiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i3
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i3
; CHECK-NEXT: store i3 [[TMP17]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: udiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0xb
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
+; GFX6-NEXT: s_and_b32 s3, s2, 7
+; GFX6-NEXT: s_bfe_u32 s2, s2, 0x30008
; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s4, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
@@ -3456,21 +3109,19 @@ define amdgpu_kernel void @udiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s3, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_and_b32 s2, s2, 7
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, s2
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX9-NEXT: global_store_byte v2, v0, s[0:1]
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i3 %x, %y
store i3 %r, ptr addrspace(1) %out
@@ -3485,17 +3136,13 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i3
; CHECK-NEXT: store i3 [[TMP19]], ptr addrspace(1) [[OUT]], align 1
@@ -3504,25 +3151,24 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX6-LABEL: urem_i3:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s3, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX6-NEXT: s_bfe_u32 s1, s6, 0x30008
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s1
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s0, s6, 7
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b32 s2, s6, 8
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: urem_i3:
@@ -3531,17 +3177,15 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: s_and_b32 s1, s2, 7
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s1
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_and_b32 s0, s2, 7
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: s_lshr_b32 s0, s2, 8
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
@@ -3559,75 +3203,56 @@ define amdgpu_kernel void @sdiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i3 [[X:%.*]], i3 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i3 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i3 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i3
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i3
; CHECK-NEXT: store i3 [[TMP21]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_bfe_i32 s5, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_bfe_i32 s1, s0, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: sdiv_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bfe_i32 s1, s0, 0x30008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_bfe_i32 s0, s0, 0x30000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s3, s2, 0x30008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_bfe_i32 s2, s2, 0x30000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i3 %x, %y
@@ -3640,23 +3265,16 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i3 [[X:%.*]], i3 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i3 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i3 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i3
; CHECK-NEXT: store i3 [[TMP23]], ptr addrspace(1) [[OUT]], align 1
@@ -3665,30 +3283,24 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX6-LABEL: srem_i3:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_bfe_i32 s3, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: srem_i3:
@@ -3697,24 +3309,17 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s0, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_bfe_i32 s1, s2, 0x30000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GFX9-NEXT: s_xor_b32 s0, s1, s0
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_lshr_b32 s3, s2, 8
-; GFX9-NEXT: s_or_b32 s6, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s6, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x30000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s2, 8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -3735,17 +3340,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i16
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <3 x i16> poison, i16 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x i16> [[Y]], i64 1
@@ -3754,17 +3355,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i16
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i16
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <3 x i16> [[TMP20]], i16 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x i16> [[Y]], i64 2
@@ -3773,17 +3370,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i16
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i16
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <3 x i16> [[TMP40]], i16 [[TMP59]], i64 2
; CHECK-NEXT: store <3 x i16> [[TMP60]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -3797,38 +3390,32 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_lshr_b32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
@@ -3837,47 +3424,40 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-LABEL: udiv_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX9-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v6, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v6, v0, s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -3894,17 +3474,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i16
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x i16> poison, i16 [[TMP21]], i64 0
@@ -3915,17 +3491,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i16
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <3 x i16> [[TMP22]], i16 [[TMP43]], i64 1
@@ -3936,17 +3508,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <3 x i16> [[TMP44]], i16 [[TMP65]], i64 2
@@ -3960,46 +3528,40 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: s_and_b32 s4, s10, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
; GFX6-NEXT: s_and_b32 s6, s9, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s6
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
@@ -4008,52 +3570,47 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-LABEL: urem_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_lshr_b32 s8, s0, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mad_f32 v2, -v5, v1, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s3
-; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1
-; GFX9-NEXT: v_rcp_f32_e32 v6, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v6
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: s_and_b32 s9, s1, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX9-NEXT: v_mad_f32 v2, -v2, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v1, s0, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v2
+; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4067,67 +3624,46 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i16
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x i16> poison, i16 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i16 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i16 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i16
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i16
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <3 x i16> [[TMP24]], i16 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <3 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i16 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i16 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i16
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <3 x i16> [[TMP48]], i16 [[TMP71]], i64 2
; CHECK-NEXT: store <3 x i16> [[TMP72]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4139,118 +3675,76 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s5, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v3
; GFX6-NEXT: s_sext_i32_i16 s4, s9
; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v0, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
-; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
; GFX9-LABEL: sdiv_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX9-NEXT: s_xor_b32 s4, s5, s4
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s8, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s8, 0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: s_sext_i32_i16 s0, s3
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v4, -v5, v0, v4
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v5
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: global_store_short v1, v0, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v1, v2, s[6:7]
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4264,23 +3758,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i16
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x i16> poison, i16 [[TMP25]], i64 0
@@ -4288,23 +3775,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i16 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i16 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i16
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <3 x i16> [[TMP26]], i16 [[TMP51]], i64 1
@@ -4312,23 +3792,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i16 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i16 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i16
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <3 x i16> [[TMP52]], i16 [[TMP77]], i64 2
@@ -4344,61 +3817,40 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_sext_i32_i16 s4, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
-; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT: s_or_b32 s7, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s7, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v4
+; GFX6-NEXT: s_sext_i32_i16 s4, s9
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s4, s10, 16
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s4
; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_lshr_b32 s4, s8, 16
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
@@ -4407,67 +3859,47 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-LABEL: srem_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s8, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
-; GFX9-NEXT: s_xor_b32 s4, s9, s8
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s10, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX9-NEXT: s_sext_i32_i16 s7, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s7
+; GFX9-NEXT: s_sext_i32_i16 s6, s0
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s6
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_ashr_i32 s8, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s8
+; GFX9-NEXT: s_sext_i32_i16 s3, s3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: s_sext_i32_i16 s9, s1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v1, s4, v2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s10
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_sext_i32_i16 s3, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: s_xor_b32 s0, s3, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s0, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT: v_sub_u32_e32 v2, s3, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4484,17 +3916,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i15
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i15
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <3 x i15> poison, i15 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x i15> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x i15> [[Y]], i64 1
@@ -4503,17 +3931,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i15
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i15
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <3 x i15> [[TMP20]], i15 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x i15> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x i15> [[Y]], i64 2
@@ -4522,17 +3946,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i15
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i15
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <3 x i15> [[TMP40]], i15 [[TMP59]], i64 2
; CHECK-NEXT: store <3 x i15> [[TMP60]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4545,45 +3965,39 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_mov_b32 s0, s8
-; GFX6-NEXT: s_and_b32 s8, s6, 0x7fff
+; GFX6-NEXT: s_and_b32 s1, s6, 0x7fff
; GFX6-NEXT: s_and_b32 s6, s4, 0x7fff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_and_b32 s6, s10, 0x7fff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT: s_mov_b32 s0, s8
+; GFX6-NEXT: s_and_b32 s8, s10, 0x7fff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], 30
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
; GFX6-NEXT: s_bfe_u32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: s_bfe_u32 s9, s10, 0xf000f
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s9
-; GFX6-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_and_b32 s5, s6, 0x7fff
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, v4, v5
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: s_bfe_u32 s10, s10, 0xf000f
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s10
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_mad_f32 v4, -v0, v3, v4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX6-NEXT: v_rcp_f32_e32 v6, v1
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc
-; GFX6-NEXT: v_mul_f32_e32 v0, v5, v6
-; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0
-; GFX6-NEXT: v_mad_f32 v0, -v0, v1, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
@@ -4596,52 +4010,46 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_and_b32 s3, s6, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
; GFX9-NEXT: s_and_b32 s5, s2, 0x7fff
; GFX9-NEXT: s_bfe_u32 s8, s2, 0xf000f
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[6:7], 30
+; GFX9-NEXT: s_and_b32 s2, s6, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[6:7], 30
; GFX9-NEXT: s_bfe_u32 s3, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_mul_f32_e32 v0, v5, v6
; GFX9-NEXT: s_and_b32 s4, s4, 0x7fff
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_mad_f32 v5, -v0, v4, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s4
-; GFX9-NEXT: v_rcp_f32_e32 v7, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v0, vcc
-; GFX9-NEXT: v_mul_f32_e32 v0, v6, v7
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v0, v3, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v0
-; GFX9-NEXT: v_mad_f32 v0, -v0, v1, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v5, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 15, v3
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: global_store_dword v4, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v4, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = udiv <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -4658,17 +4066,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i15
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x i15> poison, i15 [[TMP21]], i64 0
@@ -4679,17 +4083,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i15
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <3 x i15> [[TMP22]], i15 [[TMP43]], i64 1
@@ -4700,17 +4100,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i15
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <3 x i15> [[TMP44]], i15 [[TMP65]], i64 2
@@ -4729,51 +4125,45 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_and_b32 s8, s6, 0x7fff
; GFX6-NEXT: s_and_b32 s6, s4, 0x7fff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_and_b32 s6, s10, 0x7fff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT: s_and_b32 s11, s10, 0x7fff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11
; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], 30
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: s_and_b32 s5, s6, 0x7fff
; GFX6-NEXT: s_bfe_u32 s6, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: s_bfe_u32 s11, s10, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v3
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX6-NEXT: s_lshr_b32 s4, s4, 15
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s10, v0
-; GFX6-NEXT: v_mul_f32_e32 v0, v1, v2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT: s_bfe_u32 s12, s10, 0xf000f
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s12
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_mad_f32 v1, -v0, v3, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_rcp_f32_e32 v6, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v3
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
-; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v2, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s6, s4, 15
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s6
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v2, s5
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: s_lshr_b32 s9, s10, 15
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v0
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v1
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 15, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -4787,52 +4177,46 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_and_b32 s5, s6, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s9, s4, 0x7fff
+; GFX9-NEXT: s_and_b32 s4, s6, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
-; GFX9-NEXT: s_and_b32 s8, s4, 0x7fff
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], 30
-; GFX9-NEXT: s_bfe_u32 s5, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s5
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: s_bfe_u32 s9, s2, 0xf000f
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX9-NEXT: s_and_b32 s3, s4, 0x7fff
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s9
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s3
-; GFX9-NEXT: v_mul_f32_e32 v1, v5, v6
-; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s6, 0xf000f
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX9-NEXT: s_bfe_u32 s8, s2, 0xf000f
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s3
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_rcp_f32_e32 v7, v3
-; GFX9-NEXT: v_mad_f32 v5, -v1, v4, v5
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v6, v7
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_f32 v4, -v4, v3, v6
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: s_lshr_b32 s4, s6, 15
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s4
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v5, vcc
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s3
; GFX9-NEXT: s_lshr_b32 s3, s2, 15
+; GFX9-NEXT: v_sub_u32_e32 v1, s3, v1
; GFX9-NEXT: v_sub_u32_e32 v4, s2, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s3, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v0, s9, v3
+; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v4
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v1
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
@@ -4851,67 +4235,46 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i15> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i15 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i15 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i15
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i15
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x i15> poison, i15 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x i15> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x i15> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i15 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i15 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i15
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i15
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <3 x i15> [[TMP24]], i15 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <3 x i15> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x i15> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i15 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i15 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i15
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i15
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <3 x i15> [[TMP48]], i15 [[TMP71]], i64 2
; CHECK-NEXT: store <3 x i15> [[TMP72]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4924,62 +4287,41 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_bfe_i32 s7, s4, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s7
+; GFX6-NEXT: s_bfe_i32 s1, s4, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s1, s10, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_bfe_i32 s1, s4, 0xf000f
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b64 s[12:13], s[4:5], 30
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_bfe_i32 s1, s10, 0xf000f
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s1, s12, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s1
+; GFX6-NEXT: s_bfe_i32 s1, s6, 0xf0000
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v1
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: s_mov_b32 s0, s8
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: s_lshr_b64 s[8:9], s[4:5], 30
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_xor_b32 s5, s5, s7
-; GFX6-NEXT: s_ashr_i32 s5, s5, 30
-; GFX6-NEXT: s_or_b32 s5, s5, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[12:13], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[12:13], s[12:13], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s5, s5, 0
-; GFX6-NEXT: s_bfe_i32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s5, v2
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s7, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_cselect_b32 s4, s7, 0
-; GFX6-NEXT: s_bfe_i32 s5, s8, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, s4, v3
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v4
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -4990,67 +4332,46 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_bfe_i32 s3, s6, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_bfe_i32 s5, s2, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: s_xor_b32 s3, s5, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 30
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
-; GFX9-NEXT: s_or_b32 s3, s3, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[10:11], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, 0
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
+; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf0000
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_bfe_i32 s2, s2, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s5
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v3, s3, v3
-; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT: s_or_b32 s5, s2, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: s_cselect_b32 s2, s5, 0
-; GFX9-NEXT: s_bfe_i32 s3, s8, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: v_add_u32_e32 v4, s2, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_i32 s2, s8, 0xf0000
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_bfe_i32 s2, s4, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v1, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v1, -v5, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v5
+; GFX9-NEXT: v_rcp_f32_e32 v1, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v0, v2, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v3, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = sdiv <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -5064,23 +4385,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i15> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i15 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i15 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i15
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x i15> poison, i15 [[TMP25]], i64 0
@@ -5088,23 +4402,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x i15> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i15 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i15 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i15
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <3 x i15> [[TMP26]], i15 [[TMP51]], i64 1
@@ -5112,23 +4419,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x i15> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i15 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i15 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i15
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <3 x i15> [[TMP52]], i15 [[TMP77]], i64 2
@@ -5147,68 +4447,47 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_lshr_b64 s[8:9], s[4:5], 30
; GFX6-NEXT: s_bfe_i32 s5, s4, 0xf0000
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_bfe_i32 s12, s10, 0xf0000
+; GFX6-NEXT: s_bfe_i32 s9, s10, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s9
+; GFX6-NEXT: s_bfe_i32 s12, s4, 0xf000f
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b32 s9, s4, 15
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s12
-; GFX6-NEXT: s_xor_b32 s5, s12, s5
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: s_bfe_i32 s12, s10, 0xf000f
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s12
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_ashr_i32 s5, s5, 30
-; GFX6-NEXT: s_and_b32 s7, s6, 0x7fff
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_lshr_b32 s11, s10, 15
-; GFX6-NEXT: s_and_b32 s9, s8, 0x7fff
-; GFX6-NEXT: s_lshr_b32 s14, s4, 15
-; GFX6-NEXT: s_or_b32 s5, s5, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[12:13], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[12:13], s[12:13], exec
-; GFX6-NEXT: s_cselect_b32 s5, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s5, v2
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: s_bfe_i32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf000f
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s10, v0
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s10, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v0, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v0, -v3, v1, v0
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s10, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v3
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
; GFX6-NEXT: s_bfe_i32 s4, s8, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_bfe_i32 s5, s6, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v1
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s14
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v1, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
+; GFX6-NEXT: s_bfe_i32 s4, s6, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: s_and_b32 s5, s8, 0x7fff
; GFX6-NEXT: v_mul_lo_u32 v1, v1, s9
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s11, v0
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v2, s5
+; GFX6-NEXT: s_lshr_b32 s11, s10, 15
+; GFX6-NEXT: s_and_b32 s7, s6, 0x7fff
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s11, v1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s7, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 15, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s7, v1
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -5219,77 +4498,56 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_bfe_i32 s3, s6, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
-; GFX9-NEXT: s_bfe_i32 s7, s2, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s7
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_xor_b32 s5, s7, s5
-; GFX9-NEXT: s_ashr_i32 s5, s5, 30
-; GFX9-NEXT: s_lshr_b32 s3, s2, 15
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: s_and_b32 s9, s4, 0x7fff
-; GFX9-NEXT: s_and_b32 s12, s8, 0x7fff
-; GFX9-NEXT: s_lshr_b32 s13, s6, 15
-; GFX9-NEXT: s_or_b32 s5, s5, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[10:11], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
-; GFX9-NEXT: s_cselect_b32 s5, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s5, v2
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
-; GFX9-NEXT: s_bfe_i32 s6, s2, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v3, v1
-; GFX9-NEXT: s_xor_b32 s5, s6, s5
-; GFX9-NEXT: s_ashr_i32 s5, s5, 30
-; GFX9-NEXT: s_or_b32 s5, s5, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v1, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v2|, |v1|
-; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX9-NEXT: s_cselect_b32 s5, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v1, s5, v3
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_i32 s9, s6, 0xf000f
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_bfe_i32 s5, s2, 0xf000f
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
; GFX9-NEXT: s_bfe_i32 s5, s8, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s5
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s3, s4, 0x7fff
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_bfe_i32 s4, s4, 0xf0000
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s5
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s13
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v4
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s12
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX9-NEXT: s_lshr_b32 s4, s6, 15
+; GFX9-NEXT: s_and_b32 s7, s8, 0x7fff
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s4
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s7
+; GFX9-NEXT: s_lshr_b32 s4, s2, 15
+; GFX9-NEXT: v_sub_u32_e32 v5, s4, v1
; GFX9-NEXT: v_sub_u32_e32 v4, s2, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s3, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s9, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v4
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v5
+; GFX9-NEXT: v_sub_u32_e32 v0, s3, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v5
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 15, v3
+; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v3, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = srem <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -10016,14 +9274,11 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x41200000
-; GFX6-NEXT: v_mul_f32_e32 v1, 0x3dcccccd, v0
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX6-NEXT: v_mad_f32 v0, -v1, s4, v0
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4
; GFX6-NEXT: v_mov_b32_e32 v1, 0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x3dcccccd, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: udiv_i64_9divbits:
@@ -10032,14 +9287,11 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_add_u32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x41200000
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x3dcccccd, v0
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX9-NEXT: v_mad_f32 v0, -v1, s4, v0
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x3dcccccd, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
; CHECK-LABEL: define i64 @udiv_i64_9divbits(
; CHECK-SAME: i8 [[SIZE:%.*]]) #[[ATTR0]] {
@@ -10050,17 +9302,13 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.amdgcn.rcp.f32(float 1.000000e+01)
-; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP10]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP9]], [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.trunc.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fneg float [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.fmad.ftz.f32(float [[TMP9]], float 1.000000e+01, float [[TMP5]])
; CHECK-NEXT: [[TMP11:%.*]] = fptoui float [[TMP8]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = call float @llvm.fabs.f32(float [[TMP10]])
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.fabs.f32(float 1.000000e+01)
-; CHECK-NEXT: [[TMP14:%.*]] = fcmp oge float [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i32 1, i32 0
-; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP11]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP16]] to i64
+; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: ret i64 [[TMP17]]
;
%zextend = zext i8 %size to i64
@@ -10107,13 +9355,13 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX6-NEXT: s_cselect_b32 s4, s6, 0
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v3
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7
@@ -10135,12 +9383,12 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; GFX9-NEXT: s_or_b32 s4, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v3
; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
+; GFX9-NEXT: v_add_u32_e32 v0, s2, v4
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2
@@ -10149,17 +9397,17 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i23 [[X:%.*]], i23 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i23 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i23 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = xor i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP21:%.*]] = ashr i32 [[TMP20]], 30
+; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP21]], 1
; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
-; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
@@ -10360,9 +9608,9 @@ define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) {
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
-; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
diff --git a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
index c65a3bf1de30c..ccd1b49e63011 100644
--- a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
+++ b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
@@ -10,18 +10,14 @@ define amdgpu_kernel void @divrem24_assume(ptr addrspace(1) %arg, i32 %arg1) {
; CHECK-NEXT: [[TMP0:%.*]] = uitofp fast i32 [[TMP]] to float
; CHECK-NEXT: [[TMP1:%.*]] = uitofp fast i32 [[ARG1]] to float
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast float [[TMP0]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP6]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP8]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP5]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call fast float @llvm.trunc.f32(float [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = fneg fast float [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP5]], float [[TMP1]], float [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = fptoui float [[TMP4]] to i32
-; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.fabs.f32(float [[TMP1]])
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp fast oge float [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP7]], [[TMP11]]
-; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 [[TMP8]]
; CHECK-NEXT: store i32 0, ptr addrspace(1) [[TMP5]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index 1f751ecbe6645..1e11462a94b2e 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -605,24 +605,22 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB4_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX9-NEXT: s_add_i32 s2, s2, 1
; GFX9-NEXT: s_lshl_b32 s3, s3, 1
; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v5, vcc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -636,23 +634,21 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX10-NEXT: s_mov_b32 s2, 0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB4_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_and_b32 s3, 0xffff, s2
; GFX10-NEXT: s_add_i32 s2, s2, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX10-NEXT: s_lshl_b32 s3, s3, 1
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
+; GFX10-NEXT: v_mov_b32_e32 v2, s3
; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: s_cmpk_eq_i32 s3, 0x400
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_store_short v4, v2, s[0:1]
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB4_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -667,28 +663,24 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .LBB4_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_and_b32 s3, 0xffff, s2
; GFX11-NEXT: s_add_i32 s2, s2, 1
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX11-NEXT: s_lshl_b32 s3, s3, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mov_b32_e32 v2, s3
; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x400
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v3, vcc_lo
-; GFX11-NEXT: global_store_b16 v4, v2, s[0:1]
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB4_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -718,26 +710,24 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB5_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_and_b32 s4, 0xffff, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_add_i32 s3, s3, 1
; GFX9-NEXT: s_lshl_b32 s5, s4, 1
; GFX9-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NEXT: s_cmpk_eq_i32 s6, 0x400
-; GFX9-NEXT: v_sub_u32_e32 v2, s4, v2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB5_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -751,25 +741,23 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB5_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_and_b32 s4, 0xffff, s3
; GFX10-NEXT: s_add_i32 s3, s3, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX10-NEXT: s_lshl_b32 s5, s4, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB5_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -784,31 +772,28 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB5_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_and_b32 s4, 0xffff, s3
; GFX11-NEXT: s_add_i32 s3, s3, 1
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX11-NEXT: s_lshl_b32 s5, s4, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_add_nc_u32 v1, 1, v1
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX11-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v3, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v3, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX11-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB5_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -833,35 +818,28 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-LABEL: sdiv16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s2, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB6_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_sext_i32_i16 s4, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
-; GFX9-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX9-NEXT: s_add_i32 s3, s3, 1
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT: s_lshl_b32 s4, s5, 1
-; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
+; GFX9-NEXT: s_sext_i32_i16 s3, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX9-NEXT: s_add_i32 s2, s2, 1
+; GFX9-NEXT: s_lshl_b32 s3, s3, 1
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB6_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -871,33 +849,26 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: s_mov_b32 s2, 0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB6_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_sext_i32_i16 s4, s3
-; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX10-NEXT: s_xor_b32 s4, s4, s2
-; GFX10-NEXT: s_ashr_i32 s4, s4, 30
-; GFX10-NEXT: s_or_b32 s4, s4, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 s5, |v2|, |v0|
-; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_cselect_b32 s4, s4, 0
-; GFX10-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX10-NEXT: s_add_i32 s3, s3, 1
-; GFX10-NEXT: s_lshl_b32 s5, s5, 1
-; GFX10-NEXT: v_add_nc_u32_e32 v2, s4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: s_sext_i32_i16 s3, s2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX10-NEXT: s_add_i32 s2, s2, 1
+; GFX10-NEXT: s_lshl_b32 s3, s3, 1
+; GFX10-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX10-NEXT: s_cmpk_eq_i32 s3, 0x400
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB6_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -907,39 +878,32 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_sext_i32_i16 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: s_mov_b32 s2, 0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB6_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_sext_i32_i16 s4, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX11-NEXT: s_xor_b32 s4, s4, s2
-; GFX11-NEXT: s_ashr_i32 s4, s4, 30
+; GFX11-NEXT: s_sext_i32_i16 s3, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX11-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX11-NEXT: s_add_i32 s2, s2, 1
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_add_nc_u32 v1, 1, v1
+; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s4, s4, 1
+; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x400
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cmp_ge_f32_e64 s5, |v2|, |v0|
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, s4, 0
-; GFX11-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX11-NEXT: s_add_i32 s3, s3, 1
-; GFX11-NEXT: s_lshl_b32 s5, s5, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_add_nc_u32 v2, s4, v2
-; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -969,32 +933,25 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_sext_i32_i16 s2, s0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB7_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_sext_i32_i16 s6, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s6
-; GFX9-NEXT: s_xor_b32 s4, s6, s2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s7, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s7, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_and_b32 s5, 0xffff, s3
; GFX9-NEXT: s_add_i32 s3, s3, 1
-; GFX9-NEXT: s_lshl_b32 s4, s5, 1
-; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
-; GFX9-NEXT: v_sub_u32_e32 v2, s6, v2
+; GFX9-NEXT: s_lshl_b32 s5, s5, 1
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -1008,31 +965,24 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB7_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_sext_i32_i16 s4, s3
-; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX10-NEXT: s_xor_b32 s5, s4, s2
-; GFX10-NEXT: s_ashr_i32 s5, s5, 30
-; GFX10-NEXT: s_or_b32 s5, s5, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 s6, |v2|, |v0|
-; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s5, s5, 0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, s5, v2
; GFX10-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX10-NEXT: s_add_i32 s3, s3, 1
+; GFX10-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX10-NEXT: s_lshl_b32 s5, s5, 1
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX10-NEXT: s_add_i32 s3, s3, 1
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB7_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -1047,39 +997,28 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_sext_i32_i16 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_sext_i32_i16 s4, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX11-NEXT: s_xor_b32 s5, s4, s2
-; GFX11-NEXT: s_ashr_i32 s5, s5, 30
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s5, s5, 1
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cmp_ge_f32_e64 s6, |v2|, |v0|
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, s5, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e32 v2, s5, v2
; GFX11-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX11-NEXT: s_add_i32 s3, s3, 1
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX11-NEXT: s_lshl_b32 s5, s5, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX11-NEXT: v_mov_b32_e32 v3, s5
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX11-NEXT: s_add_i32 s3, s3, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_add_nc_u32 v1, 1, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX11-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index b54d80a03abd0..d4baf984a5ae3 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -1497,61 +1497,36 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v14, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v15, v1
-; GFX10-NEXT: v_rcp_f32_e32 v16, v10
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v19, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v17, v12
-; GFX10-NEXT: v_xor_b32_sdwa v0, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX10-NEXT: v_rcp_f32_e32 v18, v14
-; GFX10-NEXT: v_xor_b32_sdwa v3, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX10-NEXT: v_xor_b32_sdwa v11, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX10-NEXT: v_xor_b32_sdwa v13, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_3
-; GFX10-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GFX10-NEXT: v_mul_f32_e32 v15, v2, v15
-; GFX10-NEXT: v_mul_f32_e32 v16, v19, v16
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 30, v3
-; GFX10-NEXT: v_mul_f32_e32 v17, v2, v17
-; GFX10-NEXT: v_or_b32_e32 v0, 1, v0
-; GFX10-NEXT: v_trunc_f32_e32 v15, v15
-; GFX10-NEXT: v_trunc_f32_e32 v16, v16
-; GFX10-NEXT: v_mul_f32_e32 v18, v1, v18
-; GFX10-NEXT: v_trunc_f32_e32 v17, v17
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 30, v11
-; GFX10-NEXT: v_mad_f32 v20, -v15, v1, v2
-; GFX10-NEXT: v_mad_f32 v19, -v16, v10, v19
-; GFX10-NEXT: v_or_b32_e32 v3, 1, v3
-; GFX10-NEXT: v_trunc_f32_e32 v18, v18
-; GFX10-NEXT: v_mad_f32 v2, -v17, v12, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v20|, |v1|
-; GFX10-NEXT: v_ashrrev_i32_e32 v13, 30, v13
-; GFX10-NEXT: v_or_b32_e32 v11, 1, v11
-; GFX10-NEXT: v_mad_f32 v21, -v18, v14, v1
-; GFX10-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v19|, |v10|
-; GFX10-NEXT: v_or_b32_e32 v13, 1, v13
-; GFX10-NEXT: v_cvt_i32_f32_e32 v16, v16
-; GFX10-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX10-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, |v12|
-; GFX10-NEXT: v_add_nc_u32_e32 v0, v15, v0
-; GFX10-NEXT: v_add_nc_u32_sdwa v1, v16, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v21|, |v14|
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_add_nc_u32_e32 v2, v17, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v13, vcc_lo
-; GFX10-NEXT: v_add_nc_u32_sdwa v3, v18, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_rcp_f32_e32 v12, v0
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_mul_f32_e32 v12, v1, v12
+; GFX10-NEXT: v_mul_f32_e32 v2, v11, v2
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v10
+; GFX10-NEXT: v_trunc_f32_e32 v3, v12
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x60706
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1569,63 +1544,38 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x60706
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v9, v4, s4
-; GFX9-NEXT: v_xor_b32_sdwa v1, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_xor_b32_sdwa v10, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_xor_b32_sdwa v9, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_xor_b32_sdwa v14, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_3
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_rcp_f32_e32 v15, v2
-; GFX9-NEXT: v_rcp_f32_e32 v16, v12
-; GFX9-NEXT: v_rcp_f32_e32 v17, v13
-; GFX9-NEXT: v_rcp_f32_e32 v18, v4
-; GFX9-NEXT: v_mul_f32_e32 v15, v3, v15
-; GFX9-NEXT: v_mul_f32_e32 v16, v11, v16
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 30, v1
-; GFX9-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX9-NEXT: v_mul_f32_e32 v18, v2, v18
-; GFX9-NEXT: v_trunc_f32_e32 v16, v16
-; GFX9-NEXT: v_mad_f32 v19, -v15, v2, v3
-; GFX9-NEXT: v_ashrrev_i32_e32 v10, 30, v10
-; GFX9-NEXT: v_or_b32_e32 v1, 1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_trunc_f32_e32 v18, v18
-; GFX9-NEXT: v_mad_f32 v11, -v16, v12, v11
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, |v2|
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 30, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 1, v10
-; GFX9-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX9-NEXT: v_cvt_i32_f32_e32 v16, v16
-; GFX9-NEXT: v_mad_f32 v3, -v17, v13, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX9-NEXT: v_mad_f32 v2, -v18, v4, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v11|, |v12|
-; GFX9-NEXT: v_ashrrev_i32_e32 v14, 30, v14
-; GFX9-NEXT: v_or_b32_e32 v9, 1, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v13|
-; GFX9-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v4|
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v14, vcc
-; GFX9-NEXT: v_add_u32_e32 v1, v15, v1
-; GFX9-NEXT: v_add_u32_sdwa v4, v16, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_add_u32_e32 v3, v17, v3
-; GFX9-NEXT: v_add_u32_sdwa v2, v18, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: global_store_dword v[5:6], v1, off
-; GFX9-NEXT: global_store_dword v[7:8], v0, off
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_rcp_f32_e32 v12, v0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v10, v10
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_mul_f32_e32 v12, v1, v12
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v10
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v11
+; GFX9-NEXT: v_trunc_f32_e32 v3, v12
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_dword v[5:6], v0, off
+; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1850,69 +1800,45 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v12, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v15, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v17, v2
-; GFX10-NEXT: v_rcp_f32_e32 v18, v13
-; GFX10-NEXT: v_rcp_f32_e32 v19, v3
-; GFX10-NEXT: v_xor_b32_sdwa v1, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX10-NEXT: v_rcp_f32_e32 v20, v15
-; GFX10-NEXT: v_xor_b32_sdwa v11, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v21, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_xor_b32_sdwa v14, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_2
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 30, v1
-; GFX10-NEXT: v_xor_b32_sdwa v16, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_3
-; GFX10-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX10-NEXT: v_mul_f32_e32 v18, v12, v18
-; GFX10-NEXT: v_mul_f32_e32 v19, v15, v19
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 30, v11
-; GFX10-NEXT: v_or_b32_e32 v1, 1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v17, v17
-; GFX10-NEXT: v_trunc_f32_e32 v18, v18
-; GFX10-NEXT: v_mul_f32_e32 v20, v21, v20
-; GFX10-NEXT: v_trunc_f32_e32 v19, v19
-; GFX10-NEXT: v_ashrrev_i32_e32 v14, 30, v14
-; GFX10-NEXT: v_mad_f32 v22, -v17, v2, v3
-; GFX10-NEXT: v_mad_f32 v12, -v18, v13, v12
-; GFX10-NEXT: v_or_b32_e32 v11, 1, v11
-; GFX10-NEXT: v_trunc_f32_e32 v20, v20
-; GFX10-NEXT: v_mad_f32 v23, -v19, v3, v15
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v22|, |v2|
-; GFX10-NEXT: v_ashrrev_i32_e32 v16, 30, v16
-; GFX10-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX10-NEXT: v_mad_f32 v21, -v20, v15, v21
-; GFX10-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v12|, |v13|
-; GFX10-NEXT: v_or_b32_e32 v16, 1, v16
-; GFX10-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX10-NEXT: v_cvt_i32_f32_e32 v19, v19
-; GFX10-NEXT: v_cvt_i32_f32_e32 v20, v20
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v23|, |v3|
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v13, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v2
+; GFX10-NEXT: v_mul_f32_e32 v2, v10, v12
+; GFX10-NEXT: v_mul_f32_e32 v3, v11, v13
; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v1, v17, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v14, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v21|, |v15|
-; GFX10-NEXT: v_add_nc_u32_e32 v2, v18, v2
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v3, v19, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v11, 0, v16, vcc_lo
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, v10
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX10-NEXT: v_add_nc_u32_e32 v11, v20, v11
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, v0, v1
-; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_mul_lo_u32 v10, v11, v12
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v12, v3
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
+; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v10
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v14
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v11
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v14, v0
+; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_e32 v2, v11, v2
+; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2070306
@@ -1933,68 +1859,44 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x2070306
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v14, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_rcp_f32_e32 v17, v3
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v16, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_xor_b32_sdwa v15, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_2
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
-; GFX9-NEXT: v_mul_f32_e32 v17, v10, v17
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_mad_f32 v19, -v17, v3, v10
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, |v3|
-; GFX9-NEXT: v_rcp_f32_e32 v3, v14
-; GFX9-NEXT: v_rcp_f32_e32 v19, v10
-; GFX9-NEXT: v_xor_b32_sdwa v2, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX9-NEXT: v_xor_b32_sdwa v12, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX9-NEXT: v_mul_f32_e32 v3, v13, v3
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v12, v12
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v17, 1, v13
+; GFX9-NEXT: v_rcp_f32_e32 v13, v13
+; GFX9-NEXT: v_add_u32_e32 v11, 1, v11
+; GFX9-NEXT: v_add_u32_e32 v15, 1, v15
+; GFX9-NEXT: v_mul_f32_e32 v2, v16, v2
+; GFX9-NEXT: v_mul_f32_e32 v11, v11, v12
+; GFX9-NEXT: v_mul_f32_e32 v3, v17, v3
+; GFX9-NEXT: v_mul_f32_e32 v12, v15, v13
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v11, v11
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v13, -v3, v14, v13
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v13|, |v14|
-; GFX9-NEXT: v_ashrrev_i32_e32 v14, 30, v15
-; GFX9-NEXT: v_mul_f32_e32 v15, v16, v19
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_mad_f32 v19, -v15, v10, v16
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v19|, |v10|
-; GFX9-NEXT: v_rcp_f32_e32 v10, v16
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GFX9-NEXT: v_xor_b32_sdwa v19, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX9-NEXT: v_mul_f32_e32 v10, v13, v10
-; GFX9-NEXT: v_trunc_f32_e32 v10, v10
+; GFX9-NEXT: v_trunc_f32_e32 v12, v12
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX9-NEXT: v_mad_f32 v13, -v10, v16, v13
-; GFX9-NEXT: v_cvt_i32_f32_e32 v10, v10
-; GFX9-NEXT: v_or_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_ashrrev_i32_e32 v12, 30, v12
-; GFX9-NEXT: v_ashrrev_i32_e32 v19, 30, v19
-; GFX9-NEXT: v_or_b32_e32 v12, 1, v12
-; GFX9-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX9-NEXT: v_or_b32_e32 v19, 1, v19
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v13|, |v16|
-; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, v14, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v13, 0, v19, vcc
+; GFX9-NEXT: v_cvt_i32_f32_e32 v12, v12
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 8, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v18, 24, v4
-; GFX9-NEXT: v_add_u32_e32 v2, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v3, v3, v12
-; GFX9-NEXT: v_add_u32_e32 v12, v15, v14
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v4
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v11
-; GFX9-NEXT: v_mul_lo_u32 v4, v12, v0
-; GFX9-NEXT: v_mul_lo_u32 v10, v10, v18
+; GFX9-NEXT: v_mul_lo_u32 v4, v11, v10
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX9-NEXT: v_mul_lo_u32 v10, v12, v14
; GFX9-NEXT: v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_sub_u32_e32 v3, v18, v4
+; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_e32 v3, v14, v3
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -2208,56 +2110,46 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v9, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v2
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v14, v0
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
-; GFX10-NEXT: v_rcp_f32_e32 v10, v1
-; GFX10-NEXT: v_rcp_f32_e32 v11, v3
-; GFX10-NEXT: v_rcp_f32_e32 v13, v9
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v15, v0
-; GFX10-NEXT: v_rcp_f32_e32 v12, v4
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x40207
-; GFX10-NEXT: v_mul_f32_e32 v10, v14, v10
-; GFX10-NEXT: v_mul_f32_e32 v11, v4, v11
-; GFX10-NEXT: v_mul_f32_e32 v13, v1, v13
-; GFX10-NEXT: v_mul_f32_e32 v12, v15, v12
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
-; GFX10-NEXT: v_trunc_f32_e32 v11, v11
-; GFX10-NEXT: v_trunc_f32_e32 v13, v13
-; GFX10-NEXT: v_trunc_f32_e32 v12, v12
-; GFX10-NEXT: v_mad_f32 v14, -v10, v1, v14
-; GFX10-NEXT: v_cvt_u32_f32_e32 v10, v10
-; GFX10-NEXT: v_mad_f32 v16, -v11, v3, v4
-; GFX10-NEXT: v_mad_f32 v17, -v13, v9, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v14|, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX10-NEXT: v_mad_f32 v15, -v12, v4, v15
-; GFX10-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v16|, v3
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v17|, v9
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v15|, v4
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
-; GFX10-NEXT: v_or_b32_sdwa v3, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-NEXT: global_store_dword v[5:6], v1, off
-; GFX10-NEXT: global_store_dword v[7:8], v0, off
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v0, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v9
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v2, v9
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v3, v9
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v10, v9
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v11, v4
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v12
+; GFX10-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX10-NEXT: v_mul_f32_e32 v3, v11, v13
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v10
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x40207
+; GFX10-NEXT: global_store_dword v[5:6], v0, off
+; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: udiv:
@@ -2269,53 +2161,43 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: global_load_dword v4, v[2:3], off
-; GFX9-NEXT: global_load_dword v9, v[0:1], off
+; GFX9-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-NEXT: global_load_dword v9, v[2:3], off
; GFX9-NEXT: s_mov_b32 s4, 0x40207
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v11, v2
-; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
-; GFX9-NEXT: v_rcp_f32_e32 v12, v3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v1, v9
-; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v10, v4
-; GFX9-NEXT: v_rcp_f32_e32 v13, v10
-; GFX9-NEXT: v_mul_f32_e32 v11, v1, v11
-; GFX9-NEXT: v_perm_b32 v0, v9, v4, s4
-; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v11, v11
-; GFX9-NEXT: v_rcp_f32_e32 v14, v4
-; GFX9-NEXT: v_mul_f32_e32 v12, v10, v12
-; GFX9-NEXT: v_mad_f32 v1, -v11, v2, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v9, v9
-; GFX9-NEXT: v_trunc_f32_e32 v12, v12
-; GFX9-NEXT: v_mul_f32_e32 v13, v9, v13
-; GFX9-NEXT: v_mad_f32 v15, -v12, v3, v10
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX9-NEXT: v_trunc_f32_e32 v13, v13
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
-; GFX9-NEXT: v_mul_f32_e32 v14, v2, v14
-; GFX9-NEXT: v_mad_f32 v9, -v13, v10, v9
-; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v14, v14
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v15|, v3
-; GFX9-NEXT: v_mad_f32 v16, -v14, v4, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v14, v14
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v12, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v9|, v10
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v13, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v16|, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v14, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: global_store_dword v[5:6], v1, off
-; GFX9-NEXT: global_store_dword v[7:8], v0, off
+; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v0, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v9
+; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v2, v9
+; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v3, v9
+; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v11, v9
+; GFX9-NEXT: v_rcp_f32_e32 v12, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v13, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v10, v4
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_add_u32_e32 v10, 1, v10
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v12
+; GFX9-NEXT: v_mul_f32_e32 v2, v13, v2
+; GFX9-NEXT: v_mul_f32_e32 v3, v10, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v11
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT: global_store_dword v[5:6], v0, off
+; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2400,60 +2282,50 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v2
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v15, v0
-; GFX10-NEXT: v_rcp_f32_e32 v10, v1
-; GFX10-NEXT: v_rcp_f32_e32 v11, v3
-; GFX10-NEXT: v_rcp_f32_e32 v12, v4
-; GFX10-NEXT: v_rcp_f32_e32 v13, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v16, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v17, 24, v2
-; GFX10-NEXT: v_mul_f32_e32 v10, v3, v10
-; GFX10-NEXT: v_mul_f32_e32 v11, v3, v11
-; GFX10-NEXT: v_mul_f32_e32 v12, v3, v12
-; GFX10-NEXT: v_mul_f32_e32 v13, v15, v13
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
-; GFX10-NEXT: v_trunc_f32_e32 v11, v11
-; GFX10-NEXT: v_trunc_f32_e32 v12, v12
-; GFX10-NEXT: v_trunc_f32_e32 v13, v13
-; GFX10-NEXT: v_mad_f32 v18, -v10, v1, v3
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v4
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v10, v9
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-NEXT: v_rcp_f32_e32 v11, v1
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX10-NEXT: v_mul_f32_e32 v11, v1, v11
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX10-NEXT: v_mul_f32_e32 v2, v10, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v4
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v10, v11
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v10, v10
-; GFX10-NEXT: v_mad_f32 v19, -v11, v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX10-NEXT: v_mad_f32 v20, -v12, v4, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v18|, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX10-NEXT: v_mad_f32 v15, -v13, v9, v15
-; GFX10-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v19|, v3
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v20|, v4
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v16
-; GFX10-NEXT: v_sub_nc_u32_e32 v1, v16, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v15|, v9
-; GFX10-NEXT: v_mul_lo_u32 v4, v4, v14
-; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v16, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_mul_lo_u32 v9, v9, v17
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, v16, v4
-; GFX10-NEXT: v_sub_nc_u32_sdwa v9, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x2050505
-; GFX10-NEXT: v_or_b32_sdwa v3, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-NEXT: global_store_dword v[5:6], v1, off
-; GFX10-NEXT: global_store_dword v[7:8], v0, off
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX10-NEXT: v_mul_lo_u32 v10, v10, v3
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v12
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v11
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v3, v0
+; GFX10-NEXT: v_sub_nc_u32_sdwa v10, v3, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, v3, v1
+; GFX10-NEXT: v_sub_nc_u32_sdwa v2, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2050505
+; GFX10-NEXT: global_store_dword v[5:6], v0, off
+; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: urem_store_div:
@@ -2470,53 +2342,43 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: s_mov_b32 s4, 0x2050505
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v15, v2
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
-; GFX9-NEXT: v_rcp_f32_e32 v16, v3
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v11, v4
-; GFX9-NEXT: v_rcp_f32_e32 v17, v11
-; GFX9-NEXT: v_mul_f32_e32 v15, v3, v15
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v14, v4
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_rcp_f32_e32 v18, v14
-; GFX9-NEXT: v_mul_f32_e32 v16, v3, v16
-; GFX9-NEXT: v_mad_f32 v19, -v15, v2, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v15, v15
-; GFX9-NEXT: v_trunc_f32_e32 v16, v16
-; GFX9-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, v2
-; GFX9-NEXT: v_mad_f32 v2, -v16, v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v16, v16
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v15, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_rcp_f32_e32 v14, v14
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v13, v9
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_mul_f32_e32 v18, v13, v18
-; GFX9-NEXT: v_mad_f32 v19, -v17, v11, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v17, v17
-; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v18, v18
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT: v_mad_f32 v13, -v18, v14, v13
-; GFX9-NEXT: v_cvt_u32_f32_e32 v18, v18
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v16, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, v11
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v17, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v13|, v14
+; GFX9-NEXT: v_add_u32_e32 v13, 1, v13
+; GFX9-NEXT: v_mul_f32_e32 v2, v15, v2
+; GFX9-NEXT: v_mul_f32_e32 v3, v15, v3
+; GFX9-NEXT: v_mul_f32_e32 v11, v15, v11
+; GFX9-NEXT: v_mul_f32_e32 v13, v13, v14
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v11, v11
+; GFX9-NEXT: v_trunc_f32_e32 v13, v13
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
+; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, 0, v18, vcc
; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
-; GFX9-NEXT: v_mul_lo_u32 v4, v15, v4
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, v10
-; GFX9-NEXT: v_mul_lo_u32 v0, v3, v0
-; GFX9-NEXT: v_mul_lo_u32 v3, v11, v12
-; GFX9-NEXT: v_sub_u32_e32 v4, v10, v4
-; GFX9-NEXT: v_sub_u32_sdwa v2, v10, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX9-NEXT: v_mul_lo_u32 v0, v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v4, v13, v12
+; GFX9-NEXT: v_sub_u32_e32 v2, v10, v2
+; GFX9-NEXT: v_sub_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v0, v10, v0
-; GFX9-NEXT: v_sub_u32_sdwa v3, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index c1de1bd96bd5b..c63111ab5e488 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -201,25 +201,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_ashr_i32 s5, s5, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GCN-NEXT: s_ashr_i32 s4, s4, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GCN-NEXT: s_xor_b32 s4, s4, s5
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_ashr_i32 s4, s4, 30
-; GCN-NEXT: s_or_b32 s6, s4, 1
+; GCN-NEXT: s_ashr_i32 s3, s3, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GCN-NEXT: s_ashr_i32 s2, s2, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2
; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, s6, 0
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
@@ -227,25 +219,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; TONGA: ; %bb.0:
; TONGA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: s_mov_b64 s[4:5], s[2:3]
-; TONGA-NEXT: s_ashr_i32 s5, s5, 10
-; TONGA-NEXT: v_cvt_f32_i32_e32 v0, s5
-; TONGA-NEXT: s_ashr_i32 s4, s4, 10
-; TONGA-NEXT: v_cvt_f32_i32_e32 v1, s4
-; TONGA-NEXT: s_xor_b32 s4, s4, s5
-; TONGA-NEXT: v_rcp_f32_e32 v2, v0
-; TONGA-NEXT: s_ashr_i32 s4, s4, 30
-; TONGA-NEXT: s_or_b32 s6, s4, 1
+; TONGA-NEXT: s_ashr_i32 s3, s3, 10
+; TONGA-NEXT: v_cvt_f32_i32_e32 v0, s3
+; TONGA-NEXT: s_ashr_i32 s2, s2, 10
+; TONGA-NEXT: v_cvt_f32_i32_e32 v1, s2
; TONGA-NEXT: s_mov_b32 s3, 0xf000
-; TONGA-NEXT: v_mul_f32_e32 v2, v1, v2
-; TONGA-NEXT: v_trunc_f32_e32 v2, v2
-; TONGA-NEXT: v_mad_f32 v1, -v2, v0, v1
-; TONGA-NEXT: v_cvt_i32_f32_e32 v2, v2
-; TONGA-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; TONGA-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; TONGA-NEXT: s_cselect_b32 s4, s6, 0
+; TONGA-NEXT: v_rcp_f32_e32 v0, v0
; TONGA-NEXT: s_mov_b32 s2, -1
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; TONGA-NEXT: v_mul_f32_e32 v0, v1, v0
+; TONGA-NEXT: v_trunc_f32_e32 v0, v0
+; TONGA-NEXT: v_cvt_i32_f32_e32 v0, v0
; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0
; TONGA-NEXT: s_endpgm
;
@@ -253,25 +237,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_ashr_i32 s5, s5, 10
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX9-NEXT: s_ashr_i32 s4, s4, 10
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s5
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
+; GFX9-NEXT: s_ashr_i32 s3, s3, 10
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_ashr_i32 s2, s2, 10
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_add_u32_e32 v0, s4, v2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -1579,20 +1555,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GCN-NEXT: s_mov_b32 s4, s0
; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_i32_e32 v3, v1
-; GCN-NEXT: v_xor_b32_e32 v0, v1, v0
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GCN-NEXT: v_rcp_f32_e32 v4, v2
-; GCN-NEXT: v_or_b32_e32 v0, 1, v0
-; GCN-NEXT: v_mul_f32_e32 v1, v3, v4
-; GCN-NEXT: v_trunc_f32_e32 v1, v1
-; GCN-NEXT: v_mad_f32 v3, -v1, v2, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 8
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
@@ -1612,20 +1582,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; TONGA-NEXT: s_mov_b32 s4, s0
; TONGA-NEXT: s_mov_b32 s5, s1
; TONGA-NEXT: s_waitcnt vmcnt(1)
-; TONGA-NEXT: v_cvt_f32_i32_e32 v2, v0
+; TONGA-NEXT: v_cvt_f32_i32_e32 v0, v0
; TONGA-NEXT: s_waitcnt vmcnt(0)
-; TONGA-NEXT: v_cvt_f32_i32_e32 v3, v1
-; TONGA-NEXT: v_xor_b32_e32 v0, v1, v0
-; TONGA-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; TONGA-NEXT: v_rcp_f32_e32 v4, v2
-; TONGA-NEXT: v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT: v_mul_f32_e32 v1, v3, v4
-; TONGA-NEXT: v_trunc_f32_e32 v1, v1
-; TONGA-NEXT: v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT: v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT: v_cvt_f32_i32_e32 v1, v1
+; TONGA-NEXT: v_rcp_f32_e32 v0, v0
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; TONGA-NEXT: v_mul_f32_e32 v0, v1, v0
+; TONGA-NEXT: v_trunc_f32_e32 v0, v0
+; TONGA-NEXT: v_cvt_i32_f32_e32 v0, v0
; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 8
; TONGA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; TONGA-NEXT: s_endpgm
@@ -1645,20 +1609,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GFX9-NEXT: s_mov_b32 s4, s0
; GFX9-NEXT: s_mov_b32 s5, s1
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v1
-; GFX9-NEXT: v_xor_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v2, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2|
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
@@ -1743,11 +1701,11 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_or_b32_e32 v0, 1, v0
; GCN-NEXT: v_mul_f32_e32 v1, v3, v4
; GCN-NEXT: v_trunc_f32_e32 v1, v1
-; GCN-NEXT: v_mad_f32 v3, -v1, v2, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GCN-NEXT: v_mad_f32 v1, -v1, v2, v3
+; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2|
; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 23
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
@@ -1786,11 +1744,11 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: v_or_b32_e32 v0, 1, v0
; TONGA-NEXT: v_mul_f32_e32 v1, v3, v4
; TONGA-NEXT: v_trunc_f32_e32 v1, v1
-; TONGA-NEXT: v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT: v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; TONGA-NEXT: v_cvt_i32_f32_e32 v4, v1
+; TONGA-NEXT: v_mad_f32 v1, -v1, v2, v3
+; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2|
; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 23
; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0
; TONGA-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index c447d6aa270d4..4a54f2fa8322f 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -480,62 +480,42 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
define amdgpu_kernel void @s_test_sdiv22_64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-LABEL: s_test_sdiv22_64:
; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_ashr_i32 s0, s2, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: s_ashr_i32 s1, s3, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_xor_b32 s0, s1, s0
-; GCN-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-NEXT: s_or_b32 s2, s0, 1
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT: s_ashr_i32 s2, s6, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT: s_ashr_i32 s2, s3, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GCN-IR-LABEL: s_test_sdiv22_64:
; GCN-IR: ; %bb.0:
+; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-IR-NEXT: s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT: s_mov_b32 s6, -1
-; GCN-IR-NEXT: s_mov_b32 s4, s0
-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_ashr_i32 s0, s2, 10
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-IR-NEXT: s_mov_b32 s5, s1
-; GCN-IR-NEXT: s_ashr_i32 s1, s3, 10
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-IR-NEXT: s_xor_b32 s0, s1, s0
-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-IR-NEXT: s_or_b32 s2, s0, 1
-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT: s_ashr_i32 s2, s6, 10
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-IR-NEXT: s_ashr_i32 s2, s3, 10
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-IR-NEXT: s_mov_b32 s2, -1
+; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-IR-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-IR-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 42
%2 = ashr i64 %y, 42
@@ -1069,12 +1049,12 @@ define amdgpu_kernel void @s_test_sdiv23_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-NEXT: s_or_b32 s2, s0, 1
; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
; GCN-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-NEXT: v_cvt_i32_f32_e32 v3, v2
; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GCN-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v3
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
@@ -1099,12 +1079,12 @@ define amdgpu_kernel void @s_test_sdiv23_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_or_b32 s2, s0, 1
; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v2
; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v3
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-IR-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index 661e2b391af3c..d2bba69a060c8 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -482,12 +482,12 @@ define amdgpu_kernel void @s_test_srem23_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
; GCN-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-NEXT: v_cvt_i32_f32_e32 v3, v2
; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
; GCN-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GCN-NEXT: s_cselect_b32 s2, s5, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v2
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v3
; GCN-NEXT: v_mul_lo_u32 v0, v0, s4
; GCN-NEXT: s_mov_b32 s4, s0
; GCN-NEXT: s_mov_b32 s5, s1
@@ -513,12 +513,12 @@ define amdgpu_kernel void @s_test_srem23_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_mov_b32 s6, -1
; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v2
; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
; GCN-IR-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GCN-IR-NEXT: s_cselect_b32 s2, s5, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s2, v2
+; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s2, v3
; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4
; GCN-IR-NEXT: s_mov_b32 s4, s0
; GCN-IR-NEXT: s_mov_b32 s5, s1
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 17a579c59eeb3..d36d41ca5f04a 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -1404,15 +1404,13 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -1433,15 +1431,13 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: v_and_b32_e32 v0, 0xff, v0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -1455,19 +1451,17 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NEXT: flat_load_ushort v2, v[0:1]
+; GCN-NEXT: flat_load_ushort v0, v[0:1]
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT: v_rcp_f32_e32 v1, v1
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT: v_add_u32_e32 v0, vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GCN-NEXT: v_rcp_f32_e32 v4, v3
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
-; GCN-NEXT: v_mul_f32_e32 v4, v2, v4
-; GCN-NEXT: v_trunc_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GCN-NEXT: v_mad_f32 v2, -v4, v3, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
; GCN-NEXT: v_and_b32_e32 v2, 0xff, v2
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -1481,13 +1475,11 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: v_cvt_f32_ubyte1_e32 v2, v1
; GFX1030-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; GFX1030-NEXT: v_rcp_f32_e32 v3, v2
-; GFX1030-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3
-; GFX1030-NEXT: v_fma_f32 v1, -v3, v2, v1
-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v1|, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX1030-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v1, v1
+; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX1030-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT: s_endpgm
@@ -1549,13 +1541,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v0
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_rcp_f32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -1578,13 +1568,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; VI-NEXT: v_rcp_f32_e32 v2, v0
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -1599,18 +1587,16 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3
; GCN-NEXT: flat_load_dword v0, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GCN-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GCN-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GCN-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GCN-NEXT: v_rcp_f32_e32 v1, v1
+; GCN-NEXT: v_add_u32_e32 v0, vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_rcp_f32_e32 v4, v2
-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4
-; GCN-NEXT: v_trunc_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -1624,13 +1610,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX1030-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX1030-NEXT: v_rcp_f32_e32 v3, v2
-; GFX1030-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3
-; GFX1030-NEXT: v_fma_f32 v1, -v3, v2, v1
-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v1|, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX1030-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v1, v1
+; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1030-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT: s_endpgm
@@ -2458,20 +2442,14 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_i32_e32 v2, v0
+; SI-NEXT: v_cvt_f32_i32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; SI-NEXT: v_xor_b32_e32 v0, v1, v0
-; SI-NEXT: v_rcp_f32_e32 v4, v2
-; SI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; SI-NEXT: v_or_b32_e32 v0, 1, v0
-; SI-NEXT: v_mul_f32_e32 v1, v3, v4
-; SI-NEXT: v_trunc_f32_e32 v1, v1
-; SI-NEXT: v_mad_f32 v3, -v1, v2, v3
-; SI-NEXT: v_cvt_i32_f32_e32 v1, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; SI-NEXT: v_cvt_f32_i32_e32 v1, v1
+; SI-NEXT: v_rcp_f32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_i32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -2487,20 +2465,14 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; VI-NEXT: s_mov_b32 s7, s3
; VI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_i32_e32 v2, v0
+; VI-NEXT: v_cvt_f32_i32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; VI-NEXT: v_xor_b32_e32 v0, v1, v0
-; VI-NEXT: v_rcp_f32_e32 v4, v2
-; VI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; VI-NEXT: v_or_b32_e32 v0, 1, v0
-; VI-NEXT: v_mul_f32_e32 v1, v3, v4
-; VI-NEXT: v_trunc_f32_e32 v1, v1
-; VI-NEXT: v_mad_f32 v3, -v1, v2, v3
-; VI-NEXT: v_cvt_i32_f32_e32 v1, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; VI-NEXT: v_cvt_f32_i32_e32 v1, v1
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_i32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -2518,47 +2490,35 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: flat_load_sbyte v3, v[0:1]
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_cvt_f32_i32_e32 v4, v2
+; GCN-NEXT: v_cvt_f32_i32_e32 v2, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_i32_e32 v5, v3
-; GCN-NEXT: v_rcp_f32_e32 v6, v4
-; GCN-NEXT: v_xor_b32_e32 v2, v3, v2
-; GCN-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GCN-NEXT: v_or_b32_e32 v2, 1, v2
-; GCN-NEXT: v_mul_f32_e32 v3, v5, v6
-; GCN-NEXT: v_trunc_f32_e32 v3, v3
-; GCN-NEXT: v_mad_f32 v5, -v3, v4, v5
-; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, |v4|
-; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v3
+; GCN-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GCN-NEXT: v_rcp_f32_e32 v2, v2
+; GCN-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; GCN-NEXT: v_mul_f32_e32 v2, v3, v2
+; GCN-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
;
; GFX1030-LABEL: fdiv_test_denormals:
; GFX1030: ; %bb.0: ; %bb
; GFX1030-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX1030-NEXT: v_mov_b32_e32 v2, 0
; GFX1030-NEXT: v_mov_b32_e32 v0, 0
; GFX1030-NEXT: v_mov_b32_e32 v1, 0
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: global_load_sbyte v2, v0, s[0:1]
-; GFX1030-NEXT: v_mov_b32_e32 v0, 0
+; GFX1030-NEXT: global_load_sbyte v2, v2, s[0:1]
; GFX1030-NEXT: global_load_sbyte v3, v[0:1], off
; GFX1030-NEXT: s_waitcnt vmcnt(1)
-; GFX1030-NEXT: v_cvt_f32_i32_e32 v4, v2
-; GFX1030-NEXT: v_rcp_f32_e32 v5, v4
+; GFX1030-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX1030-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-NEXT: v_cvt_f32_i32_e32 v6, v3
-; GFX1030-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX1030-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GFX1030-NEXT: v_mul_f32_e32 v5, v6, v5
-; GFX1030-NEXT: v_or_b32_e32 v2, 1, v2
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v5
-; GFX1030-NEXT: v_fma_f32 v5, -v3, v4, v6
-; GFX1030-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v5|, |v4|
-; GFX1030-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc_lo
-; GFX1030-NEXT: v_add_nc_u32_e32 v2, v3, v2
+; GFX1030-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX1030-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v2, v2
+; GFX1030-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX1030-NEXT: global_store_byte v[0:1], v2, off
; GFX1030-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
index d119f8400acf2..dafb575233bbf 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem24.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
@@ -22,13 +22,11 @@ define amdgpu_kernel void @udiv24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v1
-; SI-NEXT: v_mul_f32_e32 v2, v0, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; SI-NEXT: v_rcp_f32_e32 v1, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -48,15 +46,13 @@ define amdgpu_kernel void @udiv24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -125,13 +121,11 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in_out(ptr addrspace(1) %out,
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v1
-; SI-NEXT: v_mul_f32_e32 v2, v0, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; SI-NEXT: v_rcp_f32_e32 v1, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -151,15 +145,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in_out(ptr addrspace(1) %out,
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -228,13 +220,11 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in(ptr addrspace(1) %out, ptr
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v1
-; SI-NEXT: v_mul_f32_e32 v2, v0, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; SI-NEXT: v_rcp_f32_e32 v1, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -254,15 +244,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in(ptr addrspace(1) %out, ptr
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -331,13 +319,11 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_out(ptr addrspace(1) %out, ptr
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v1
-; SI-NEXT: v_mul_f32_e32 v2, v0, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; SI-NEXT: v_rcp_f32_e32 v1, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -357,15 +343,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_out(ptr addrspace(1) %out, ptr
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -434,13 +418,11 @@ define amdgpu_kernel void @udiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v1
-; SI-NEXT: v_mul_f32_e32 v2, v0, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; SI-NEXT: v_rcp_f32_e32 v1, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -462,13 +444,11 @@ define amdgpu_kernel void @udiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; VI-NEXT: v_rcp_f32_e32 v2, v0
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -1351,13 +1331,11 @@ define amdgpu_kernel void @urem24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
-; SI-NEXT: v_rcp_f32_e32 v4, v3
-; SI-NEXT: v_mul_f32_e32 v4, v2, v4
-; SI-NEXT: v_trunc_f32_e32 v4, v4
-; SI-NEXT: v_fma_f32 v2, -v4, v3, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v4, v4
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc
+; SI-NEXT: v_rcp_f32_e32 v3, v3
+; SI-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; SI-NEXT: v_mul_f32_e32 v2, v2, v3
+; SI-NEXT: v_trunc_f32_e32 v2, v2
+; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
; SI-NEXT: v_mul_lo_u32 v1, v2, v1
; SI-NEXT: v_subrev_i32_e32 v0, vcc, v1, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
@@ -1379,15 +1357,13 @@ define amdgpu_kernel void @urem24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
-; VI-NEXT: v_rcp_f32_e32 v3, v2
+; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v4, v1
-; VI-NEXT: v_mul_f32_e32 v3, v4, v3
-; VI-NEXT: v_trunc_f32_e32 v3, v3
-; VI-NEXT: v_cvt_u32_f32_e32 v5, v3
-; VI-NEXT: v_mad_f32 v3, -v3, v2, v4
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; VI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
+; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; VI-NEXT: v_mul_f32_e32 v2, v3, v2
+; VI-NEXT: v_trunc_f32_e32 v2, v2
+; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
; VI-NEXT: v_mul_lo_u32 v0, v2, v0
; VI-NEXT: v_subrev_u32_e32 v0, vcc, v0, v1
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
@@ -1460,13 +1436,11 @@ define amdgpu_kernel void @urem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v2, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v3, v1
-; SI-NEXT: v_rcp_f32_e32 v4, v3
-; SI-NEXT: v_mul_f32_e32 v4, v2, v4
-; SI-NEXT: v_trunc_f32_e32 v4, v4
-; SI-NEXT: v_fma_f32 v2, -v4, v3, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v4, v4
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc
+; SI-NEXT: v_rcp_f32_e32 v3, v3
+; SI-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; SI-NEXT: v_mul_f32_e32 v2, v2, v3
+; SI-NEXT: v_trunc_f32_e32 v2, v2
+; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
; SI-NEXT: v_mul_lo_u32 v1, v2, v1
; SI-NEXT: v_subrev_i32_e32 v0, vcc, v1, v0
; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
@@ -1490,13 +1464,11 @@ define amdgpu_kernel void @urem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v2, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v3, v1
-; VI-NEXT: v_rcp_f32_e32 v4, v2
-; VI-NEXT: v_mul_f32_e32 v4, v3, v4
-; VI-NEXT: v_trunc_f32_e32 v4, v4
-; VI-NEXT: v_cvt_u32_f32_e32 v5, v4
-; VI-NEXT: v_mad_f32 v3, -v4, v2, v3
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; VI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; VI-NEXT: v_rcp_f32_e32 v2, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; VI-NEXT: v_mul_f32_e32 v2, v3, v2
+; VI-NEXT: v_trunc_f32_e32 v2, v2
+; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
; VI-NEXT: v_mul_lo_u32 v0, v2, v0
; VI-NEXT: v_subrev_u32_e32 v0, vcc, v0, v1
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
>From eb3afd33c4c6c1ea5eb146b1c48fe394e2090f9a Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 13 Jul 2026 03:45:42 -0500
Subject: [PATCH 3/3] Allow larger range of numerators
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 91 ++++---------------
1 file changed, 16 insertions(+), 75 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 329b9924e667f..cdac34b291bef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1083,10 +1083,10 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
//
// Thus, we conservatively restrict expandDivRemToFloatImpl to
- // [-0x40000,0x3FFFFF] for IsSigned
- // [0x000000,0x3FFFFF] for !IsSigned.
+ // [-0x400000,0x3FFFFF] for IsSigned
+ // [ 0x000000,0x3FFFFF] for !IsSigned.
assert(0 < DivBits && DivBits <= (IsSigned ? 23 : 22) &&
- "abs(Num) must be <= than 0x40000 for expandDivRemToFloatImpl to work "
+ "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
"correctly");
Type *I32Ty = Builder.getInt32Ty();
@@ -1117,30 +1117,17 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// fq = fa*recip(fb)
// may be too small due to the 1ulp accuracy in the recip
// operation and rounding issues. Since fq is truncated to produce
- // an integer value it may be too small by one. This
- // can be dealt with in one of two ways.
- //
- // 1. If fabs(fa) <= 0x200000 increment fa by 1ulp:
- // fq = (fa+1ulp)*recip(fb)
- // This method is preferred since it only requires one
- // integer +1 operation on the floating-point encoding of fa.
- // This will increase fa's magnitude by at most 0.25
- // (i.e. when fabs(fa)==0x200000 the LSB of the mantissa represents 0.25).
- // Thus, this method is safe since fa must be incremented by at least 1.0
- // for quotient to increase by one. Note that setting the boundary to
- // fabs(fa) <= 0x400000 and incrementing fa's magnitude by at most 0.5 is
- // unsafe because recip(fb) can have a 1 ulp error.
- //
- // 2. A. Calculate fr = fa - fq*fb
- // B. If fabs(fr) >= fabs(fb) increment the quotient by 1.
-
- bool IncrementFA = (DivBits < (IsSigned ? 23 : 22));
-
- if (IncrementFA) {
- Value *FABits = Builder.CreateBitCast(FA, I32Ty);
- Value *FABitsInc = Builder.CreateAdd(FABits, One);
- FA = Builder.CreateBitCast(FABitsInc, F32Ty);
- }
+ // an integer value it may be too small by one. This is
+ // dealt with by incrementing fa by 1ulp:
+ // fq = (fa+1ulp)*recip(fb)
+ // This will increase fa's magnitude by at most 0.5
+ // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
+ // Thus, this method is safe since fa must be incremented by at least 1.0
+ // for the quotient to increase by one.
+
+ Value *FABits = Builder.CreateBitCast(FA, I32Ty);
+ Value *FABitsInc = Builder.CreateAdd(FABits, One);
+ FA = Builder.CreateBitCast(FABitsInc, F32Ty);
Value *FQM = Builder.CreateFMul(FA, RCP);
@@ -1151,56 +1138,10 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
Value *IQ = IsSigned ? Builder.CreateFPToSI(FQ, I32Ty)
: Builder.CreateFPToUI(FQ, I32Ty);
- Value *Div;
- if (IncrementFA) {
- Div = IQ;
- } else {
- Value *JQ = One;
- auto *FQI = dyn_cast<Instruction>(FQ);
- if (FQI)
- FQI->copyFastMathFlags(Builder.getFastMathFlags());
-
- if (IsSigned) {
- // char|short jq = ia ^ ib;
- JQ = Builder.CreateXor(Num, Den);
-
- // jq = jq >> (bitsize - 2)
- JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
-
- // jq = jq | 0x1
- JQ = Builder.CreateOr(JQ, One);
- }
-
- // float fqneg = -fq;
- Value *FQNeg = Builder.CreateFNeg(FQ);
-
- // float fr = mad(fqneg, fb, fa);
- auto FMAD = !ST.hasMadMacF32Insts()
- ? Intrinsic::fma
- : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
- Value *FR =
- Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
-
- // fr = fabs(fr);
- FR = Builder.CreateFAbs(FR, FQI);
-
- // fb = fabs(fb);
- FB = Builder.CreateFAbs(FB, FQI);
-
- // int cv = fr >= fb;
- Value *CV = Builder.CreateFCmpOGE(FR, FB);
-
- // jq = (cv ? jq : 0);
- JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
-
- // dst = iq + jq;
- Div = Builder.CreateAdd(IQ, JQ);
- }
-
- Value *Res = Div;
+ Value *Res = IQ;
if (!IsDiv) {
// Rem needs compensation, it's easier to recompute it
- Value *Rem = Builder.CreateMul(Div, Den);
+ Value *Rem = Builder.CreateMul(IQ, Den);
Res = Builder.CreateSub(Num, Rem);
}
More information about the llvm-commits
mailing list