[llvm] ba61e4d - [AMDGPU] Implement int div/rem by fp recip more efficiently (#204950)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 08:53:26 PDT 2026
Author: LU-JOHN
Date: 2026-07-13T10:53:20-05:00
New Revision: ba61e4d1c110199ef07d7ef4c34c23bcc64b4b54
URL: https://github.com/llvm/llvm-project/commit/ba61e4d1c110199ef07d7ef4c34c23bcc64b4b54
DIFF: https://github.com/llvm/llvm-project/commit/ba61e4d1c110199ef07d7ef4c34c23bcc64b4b54.diff
LOG: [AMDGPU] Implement int div/rem by fp recip more efficiently (#204950)
Integer division q = a/b can be implemented by fp reciprocal with:
`fq = fa * recip(fb)`
fq is truncated to produce q. Due to fp rounding and reciprocal accuracy
issues fq can be too small and truncation can produce a value too small
by one.
If abs(a)<=0x400000, this underestimate can be guarded more efficiently
by calculating:
`fq=fa+1ulp/b`
If abs(a)<=0x400000, adding 1 ulp will increase a by at most 0.5, so the
calculated q will be the same. Adding 1ulp can be done with one integer
add.
Incrementing `fa` by 1 ulp was exhaustively tested for all dividend and
divisor combinations with values between [-0x400000,0x3FFFFF] with no
errors.
---------
Signed-off-by: John Lu <John.Lu at amd.com>
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
llvm/test/CodeGen/AMDGPU/idiv-licm.ll
llvm/test/CodeGen/AMDGPU/permute_i8.ll
llvm/test/CodeGen/AMDGPU/sdiv.ll
llvm/test/CodeGen/AMDGPU/sdiv64.ll
llvm/test/CodeGen/AMDGPU/srem64.ll
llvm/test/CodeGen/AMDGPU/udiv.ll
llvm/test/CodeGen/AMDGPU/udivrem24.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 0bde97afd1655..f6679fce1c4fa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1082,10 +1082,10 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
// (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
//
// Thus, we conservatively restrict expandDivRemToFloatImpl to
- // [-0x40000,0x3FFFFF] for IsSigned
- // [0x000000,0x3FFFFF] for !IsSigned.
+ // [-0x400000,0x3FFFFF] for IsSigned
+ // [ 0x000000,0x3FFFFF] for !IsSigned.
assert(0 < DivBits && DivBits <= (IsSigned ? 23 : 22) &&
- "abs(Num) must be <= than 0x40000 for expandDivRemToFloatImpl to work "
+ "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
"correctly");
Type *I32Ty = Builder.getInt32Ty();
@@ -1094,18 +1094,6 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
Type *F32Ty = Builder.getFloatTy();
ConstantInt *One = Builder.getInt32(1);
- Value *JQ = One;
-
- if (IsSigned) {
- // char|short jq = ia ^ ib;
- JQ = Builder.CreateXor(Num, Den);
-
- // jq = jq >> (bitsize - 2)
- JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
-
- // jq = jq | 0x1
- JQ = Builder.CreateOr(JQ, One);
- }
// int ia = (int)LHS;
Value *IA = Num;
@@ -1118,52 +1106,41 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
: Builder.CreateUIToFP(IA, F32Ty);
// float fb = (float)ib;
- Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
- : Builder.CreateUIToFP(IB,F32Ty);
+ Value *FB = IsSigned ? Builder.CreateSIToFP(IB, F32Ty)
+ : Builder.CreateUIToFP(IB, F32Ty);
Value *RCP = Builder.CreateIntrinsic(Intrinsic::amdgcn_rcp,
Builder.getFloatTy(), {FB});
+
+ // The calculation:
+ // fq = fa*recip(fb)
+ // may be too small due to the 1ulp accuracy in the recip
+ // operation and rounding issues. Since fq is truncated to produce
+ // an integer value it may be too small by one. This is
+ // dealt with by incrementing fa by 1ulp:
+ // fq = (fa+1ulp)*recip(fb)
+ // This will increase fa's magnitude by at most 0.5
+ // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
+ // Thus, this method is safe since fa must be incremented by at least 1.0
+ // for the quotient to increase by one.
+
+ Value *FABits = Builder.CreateBitCast(FA, I32Ty);
+ Value *FABitsInc = Builder.CreateAdd(FABits, One);
+ FA = Builder.CreateBitCast(FABitsInc, F32Ty);
+
Value *FQM = Builder.CreateFMul(FA, RCP);
// fq = trunc(fqm);
Value *FQ = Builder.CreateUnaryIntrinsic(Intrinsic::trunc, FQM);
- auto *FQI = dyn_cast<Instruction>(FQ);
- if (FQI)
- FQI->copyFastMathFlags(Builder.getFastMathFlags());
-
- // float fqneg = -fq;
- Value *FQNeg = Builder.CreateFNeg(FQ);
-
- // float fr = mad(fqneg, fb, fa);
- auto FMAD = !ST.hasMadMacF32Insts()
- ? Intrinsic::fma
- : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
- Value *FR =
- Builder.CreateIntrinsic(FMAD, {FQNeg->getType()}, {FQNeg, FB, FA}, FQI);
// int iq = (int)fq;
Value *IQ = IsSigned ? Builder.CreateFPToSI(FQ, I32Ty)
: Builder.CreateFPToUI(FQ, I32Ty);
- // fr = fabs(fr);
- FR = Builder.CreateFAbs(FR, FQI);
-
- // fb = fabs(fb);
- FB = Builder.CreateFAbs(FB, FQI);
-
- // int cv = fr >= fb;
- Value *CV = Builder.CreateFCmpOGE(FR, FB);
-
- // jq = (cv ? jq : 0);
- JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
-
- // dst = iq + jq;
- Value *Div = Builder.CreateAdd(IQ, JQ);
-
- Value *Res = Div;
+ Value *Res = IQ;
if (!IsDiv) {
// Rem needs compensation, it's easier to recompute it
- Value *Rem = Builder.CreateMul(Div, Den);
+ Value *Rem = Builder.CreateMul(IQ, Den);
Res = Builder.CreateSub(Num, Rem);
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 4a33008661b8e..fb6aef4cec282 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -69,6 +69,7 @@ define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -163,6 +164,7 @@ define amdgpu_kernel void @urem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -275,6 +277,7 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -388,6 +391,7 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -432,17 +436,13 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i16
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i16
; CHECK-NEXT: store i16 [[TMP17]], ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
@@ -457,35 +457,32 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s1, s0, 16
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1
; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v0, s[0:1]
+; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i16 %x, %y
store i16 %r, ptr addrspace(1) %out
@@ -500,17 +497,13 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
; CHECK-NEXT: store i16 [[TMP19]], ptr addrspace(1) [[OUT]], align 2
@@ -526,19 +519,18 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-NEXT: s_and_b32 s0, s6, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -548,14 +540,12 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -571,73 +561,55 @@ define amdgpu_kernel void @sdiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i16
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i16
; CHECK-NEXT: store i16 [[TMP21]], ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s4, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_ashr_i32 s1, s0, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_sext_i32_i16 s0, s0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_ashr_i32 s1, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s3, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_sext_i32_i16 s2, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i16 %x, %y
@@ -650,23 +622,16 @@ define amdgpu_kernel void @srem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
; CHECK-NEXT: store i16 [[TMP23]], ptr addrspace(1) [[OUT]], align 2
@@ -675,54 +640,43 @@ define amdgpu_kernel void @srem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX6-LABEL: srem_i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s7, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX6-NEXT: s_sext_i32_i16 s2, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX6-NEXT: s_xor_b32 s2, s2, s7
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_or_b32 s4, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s4, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_ashr_i32 s2, s6, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX6-NEXT: s_sext_i32_i16 s0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s7, s6, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX9-NEXT: s_sext_i32_i16 s2, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s7
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_ashr_i32 s3, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_sext_i32_i16 s0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i16 %x, %y
@@ -738,54 +692,47 @@ define amdgpu_kernel void @udiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i8
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i8
; CHECK-NEXT: store i8 [[TMP17]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: udiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0xb
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, s2
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
-; GFX9-NEXT: global_store_byte v2, v0, s[0:1]
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i8 %x, %y
store i8 %r, ptr addrspace(1) %out
@@ -800,17 +747,13 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i8
; CHECK-NEXT: store i8 [[TMP19]], ptr addrspace(1) [[OUT]], align 1
@@ -823,36 +766,33 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b32 s2, s6, 8
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_lshr_b32 s3, s2, 8
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
@@ -867,73 +807,55 @@ define amdgpu_kernel void @sdiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i8 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i8
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i8
; CHECK-NEXT: store i8 [[TMP21]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i8 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_bfe_i32 s1, s0, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_sext_i32_i8 s0, s0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bfe_i32 s1, s0, 0x80008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s3, s2, 0x80008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_sext_i32_i8 s2, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i8 %x, %y
@@ -946,23 +868,16 @@ define amdgpu_kernel void @srem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i8 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i8
; CHECK-NEXT: store i8 [[TMP23]], ptr addrspace(1) [[OUT]], align 1
@@ -971,56 +886,45 @@ define amdgpu_kernel void @srem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
; GFX6-LABEL: srem_i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_sext_i32_i8 s3, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_sext_i32_i8 s0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_sext_i32_i8 s3, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX9-NEXT: s_xor_b32 s2, s3, s2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_lshr_b32 s4, s6, 8
-; GFX9-NEXT: s_or_b32 s5, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s4
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x80008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: s_sext_i32_i8 s0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_lshr_b32 s3, s2, 8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i8 %x, %y
@@ -1259,6 +1163,7 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -1568,6 +1473,7 @@ define amdgpu_kernel void @urem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -1948,6 +1854,7 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s0, v3
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -2334,6 +2241,7 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -2454,17 +2362,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i16
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i16> poison, i16 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i16> [[Y]], i64 1
@@ -2473,17 +2377,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i16
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i16
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i16> [[TMP20]], i16 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i16> [[Y]], i64 2
@@ -2492,17 +2392,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i16
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i16
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <4 x i16> [[TMP40]], i16 [[TMP59]], i64 2
; CHECK-NEXT: [[TMP61:%.*]] = extractelement <4 x i16> [[X]], i64 3
; CHECK-NEXT: [[TMP62:%.*]] = extractelement <4 x i16> [[Y]], i64 3
@@ -2511,17 +2407,13 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP65:%.*]] = uitofp fast i32 [[TMP63]] to float
; CHECK-NEXT: [[TMP66:%.*]] = uitofp fast i32 [[TMP64]] to float
; CHECK-NEXT: [[TMP67:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP68:%.*]] = fmul fast float [[TMP65]], [[TMP67]]
+; CHECK-NEXT: [[TMP53:%.*]] = bitcast float [[TMP65]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP68:%.*]] = fmul fast float [[TMP55]], [[TMP67]]
; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.trunc.f32(float [[TMP68]])
-; CHECK-NEXT: [[TMP70:%.*]] = fneg fast float [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP70]], float [[TMP66]], float [[TMP65]])
; CHECK-NEXT: [[TMP72:%.*]] = fptoui float [[TMP69]] to i32
-; CHECK-NEXT: [[TMP73:%.*]] = call fast float @llvm.fabs.f32(float [[TMP71]])
-; CHECK-NEXT: [[TMP74:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP75:%.*]] = fcmp fast oge float [[TMP73]], [[TMP74]]
-; CHECK-NEXT: [[TMP76:%.*]] = select i1 [[TMP75]], i32 1, i32 0
-; CHECK-NEXT: [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP76]]
-; CHECK-NEXT: [[TMP79:%.*]] = trunc i32 [[TMP77]] to i16
+; CHECK-NEXT: [[TMP79:%.*]] = trunc i32 [[TMP72]] to i16
; CHECK-NEXT: [[TMP80:%.*]] = insertelement <4 x i16> [[TMP60]], i16 [[TMP79]], i64 3
; CHECK-NEXT: store <4 x i16> [[TMP80]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -2535,112 +2427,96 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_lshr_b32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v2, v3
; GFX6-NEXT: s_lshr_b32 s4, s11, 16
-; GFX6-NEXT: v_mad_f32 v3, -v1, v4, v5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
; GFX6-NEXT: s_lshr_b32 s4, s9, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4
-; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_rcp_f32_e32 v7, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mul_f32_e32 v3, v6, v7
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: v_rcp_f32_e32 v4, v4
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX6-NEXT: v_mad_f32 v3, -v3, v5, v6
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_lshr_b32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v5, v7
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_mad_f32 v3, -v1, v4, v5
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s0
-; GFX9-NEXT: v_rcp_f32_e32 v8, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
+; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2657,17 +2533,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i16
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> poison, i16 [[TMP21]], i64 0
@@ -2678,17 +2550,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i16
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[TMP43]], i64 1
@@ -2699,17 +2567,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <4 x i16> [[TMP44]], i16 [[TMP65]], i64 2
@@ -2720,17 +2584,13 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP71:%.*]] = uitofp fast i32 [[TMP69]] to float
; CHECK-NEXT: [[TMP72:%.*]] = uitofp fast i32 [[TMP70]] to float
; CHECK-NEXT: [[TMP73:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP72]])
-; CHECK-NEXT: [[TMP74:%.*]] = fmul fast float [[TMP71]], [[TMP73]]
+; CHECK-NEXT: [[TMP59:%.*]] = bitcast float [[TMP71]] to i32
+; CHECK-NEXT: [[TMP60:%.*]] = add i32 [[TMP59]], 1
+; CHECK-NEXT: [[TMP61:%.*]] = bitcast i32 [[TMP60]] to float
+; CHECK-NEXT: [[TMP74:%.*]] = fmul fast float [[TMP61]], [[TMP73]]
; CHECK-NEXT: [[TMP75:%.*]] = call fast float @llvm.trunc.f32(float [[TMP74]])
-; CHECK-NEXT: [[TMP76:%.*]] = fneg fast float [[TMP75]]
-; CHECK-NEXT: [[TMP77:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP76]], float [[TMP72]], float [[TMP71]])
; CHECK-NEXT: [[TMP78:%.*]] = fptoui float [[TMP75]] to i32
-; CHECK-NEXT: [[TMP79:%.*]] = call fast float @llvm.fabs.f32(float [[TMP77]])
-; CHECK-NEXT: [[TMP80:%.*]] = call fast float @llvm.fabs.f32(float [[TMP72]])
-; CHECK-NEXT: [[TMP81:%.*]] = fcmp fast oge float [[TMP79]], [[TMP80]]
-; CHECK-NEXT: [[TMP82:%.*]] = select i1 [[TMP81]], i32 1, i32 0
-; CHECK-NEXT: [[TMP83:%.*]] = add i32 [[TMP78]], [[TMP82]]
-; CHECK-NEXT: [[TMP84:%.*]] = mul i32 [[TMP83]], [[TMP70]]
+; CHECK-NEXT: [[TMP84:%.*]] = mul i32 [[TMP78]], [[TMP70]]
; CHECK-NEXT: [[TMP85:%.*]] = sub i32 [[TMP69]], [[TMP84]]
; CHECK-NEXT: [[TMP87:%.*]] = trunc i32 [[TMP85]] to i16
; CHECK-NEXT: [[TMP88:%.*]] = insertelement <4 x i16> [[TMP66]], i16 [[TMP87]], i64 3
@@ -2744,129 +2604,114 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: s_and_b32 s4, s10, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mad_f32 v1, -v1, v2, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: s_and_b32 s5, s9, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v1
-; GFX6-NEXT: s_lshr_b32 s4, s11, 16
-; GFX6-NEXT: v_mul_f32_e32 v1, v3, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: s_lshr_b32 s5, s9, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s5
-; GFX6-NEXT: v_rcp_f32_e32 v7, v4
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v3
+; GFX6-NEXT: s_and_b32 s6, s9, 0xffff
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, v6, v7
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s6, s9, 16
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: s_lshr_b32 s5, s11, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v6
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX6-NEXT: v_trunc_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s4, v1
+; GFX6-NEXT: v_mul_lo_u32 v1, v3, s5
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
-; GFX9-NEXT: s_and_b32 s5, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s5
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_lshr_b32 s2, s3, 16
-; GFX9-NEXT: v_mad_f32 v3, -v2, v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v4
+; GFX9-NEXT: s_and_b32 s9, s3, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: s_and_b32 s8, s1, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s8
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_rcp_f32_e32 v8, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s1
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s4
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
-; GFX9-NEXT: v_sub_u32_e32 v4, s0, v1
-; GFX9-NEXT: v_sub_u32_e32 v1, s5, v2
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v3
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
+; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = urem <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2880,89 +2725,61 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i16
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> poison, i16 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i16 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i16 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i16
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i16
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <4 x i16> [[TMP24]], i16 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <4 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i16 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i16 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i16
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <4 x i16> [[TMP48]], i16 [[TMP71]], i64 2
; CHECK-NEXT: [[TMP73:%.*]] = extractelement <4 x i16> [[X]], i64 3
; CHECK-NEXT: [[TMP74:%.*]] = extractelement <4 x i16> [[Y]], i64 3
; CHECK-NEXT: [[TMP75:%.*]] = sext i16 [[TMP73]] to i32
; CHECK-NEXT: [[TMP76:%.*]] = sext i16 [[TMP74]] to i32
-; CHECK-NEXT: [[TMP77:%.*]] = xor i32 [[TMP75]], [[TMP76]]
-; CHECK-NEXT: [[TMP78:%.*]] = ashr i32 [[TMP77]], 30
-; CHECK-NEXT: [[TMP79:%.*]] = or i32 [[TMP78]], 1
; CHECK-NEXT: [[TMP80:%.*]] = sitofp fast i32 [[TMP75]] to float
; CHECK-NEXT: [[TMP81:%.*]] = sitofp fast i32 [[TMP76]] to float
; CHECK-NEXT: [[TMP82:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP81]])
-; CHECK-NEXT: [[TMP83:%.*]] = fmul fast float [[TMP80]], [[TMP82]]
+; CHECK-NEXT: [[TMP53:%.*]] = bitcast float [[TMP80]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP83:%.*]] = fmul fast float [[TMP55]], [[TMP82]]
; CHECK-NEXT: [[TMP84:%.*]] = call fast float @llvm.trunc.f32(float [[TMP83]])
-; CHECK-NEXT: [[TMP85:%.*]] = fneg fast float [[TMP84]]
-; CHECK-NEXT: [[TMP86:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP85]], float [[TMP81]], float [[TMP80]])
; CHECK-NEXT: [[TMP87:%.*]] = fptosi float [[TMP84]] to i32
-; CHECK-NEXT: [[TMP88:%.*]] = call fast float @llvm.fabs.f32(float [[TMP86]])
-; CHECK-NEXT: [[TMP89:%.*]] = call fast float @llvm.fabs.f32(float [[TMP81]])
-; CHECK-NEXT: [[TMP90:%.*]] = fcmp fast oge float [[TMP88]], [[TMP89]]
-; CHECK-NEXT: [[TMP91:%.*]] = select i1 [[TMP90]], i32 [[TMP79]], i32 0
-; CHECK-NEXT: [[TMP92:%.*]] = add i32 [[TMP87]], [[TMP91]]
-; CHECK-NEXT: [[TMP95:%.*]] = trunc i32 [[TMP92]] to i16
+; CHECK-NEXT: [[TMP95:%.*]] = trunc i32 [[TMP87]] to i16
; CHECK-NEXT: [[TMP96:%.*]] = insertelement <4 x i16> [[TMP72]], i16 [[TMP95]], i64 3
; CHECK-NEXT: store <4 x i16> [[TMP96]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -2974,152 +2791,97 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s5, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
; GFX6-NEXT: s_sext_i32_i16 s4, s9
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s11, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s9, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v5, v5
-; GFX6-NEXT: v_mad_f32 v4, -v5, v0, v4
-; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v2, v3
+; GFX6-NEXT: s_ashr_i32 s4, s11, 16
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
+; GFX6-NEXT: s_ashr_i32 s4, s9, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_rcp_f32_e32 v4, v4
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX6-NEXT: v_trunc_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: s_xor_b32 s4, s5, s4
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s8, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s8, 0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_add_u32_e32 v3, s4, v3
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v4
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v1, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v1, -v5, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: s_ashr_i32 s2, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: v_add_u32_e32 v1, s0, v5
-; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v6, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
-; GFX9-NEXT: v_trunc_f32_e32 v6, v6
-; GFX9-NEXT: v_mad_f32 v5, -v6, v0, v5
-; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v6
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: s_ashr_i32 s0, s3, 16
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
+; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <4 x i16> %x, %y
@@ -3134,23 +2896,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i16
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> poison, i16 [[TMP25]], i64 0
@@ -3158,23 +2913,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i16 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i16 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i16
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i16> [[TMP26]], i16 [[TMP51]], i64 1
@@ -3182,23 +2930,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i16 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i16 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i16
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <4 x i16> [[TMP52]], i16 [[TMP77]], i64 2
@@ -3206,23 +2947,16 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; CHECK-NEXT: [[TMP80:%.*]] = extractelement <4 x i16> [[Y]], i64 3
; CHECK-NEXT: [[TMP81:%.*]] = sext i16 [[TMP79]] to i32
; CHECK-NEXT: [[TMP82:%.*]] = sext i16 [[TMP80]] to i32
-; CHECK-NEXT: [[TMP83:%.*]] = xor i32 [[TMP81]], [[TMP82]]
-; CHECK-NEXT: [[TMP84:%.*]] = ashr i32 [[TMP83]], 30
-; CHECK-NEXT: [[TMP85:%.*]] = or i32 [[TMP84]], 1
; CHECK-NEXT: [[TMP86:%.*]] = sitofp fast i32 [[TMP81]] to float
; CHECK-NEXT: [[TMP87:%.*]] = sitofp fast i32 [[TMP82]] to float
; CHECK-NEXT: [[TMP88:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP87]])
-; CHECK-NEXT: [[TMP89:%.*]] = fmul fast float [[TMP86]], [[TMP88]]
+; CHECK-NEXT: [[TMP59:%.*]] = bitcast float [[TMP86]] to i32
+; CHECK-NEXT: [[TMP65:%.*]] = add i32 [[TMP59]], 1
+; CHECK-NEXT: [[TMP66:%.*]] = bitcast i32 [[TMP65]] to float
+; CHECK-NEXT: [[TMP89:%.*]] = fmul fast float [[TMP66]], [[TMP88]]
; CHECK-NEXT: [[TMP90:%.*]] = call fast float @llvm.trunc.f32(float [[TMP89]])
-; CHECK-NEXT: [[TMP91:%.*]] = fneg fast float [[TMP90]]
-; CHECK-NEXT: [[TMP92:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP91]], float [[TMP87]], float [[TMP86]])
; CHECK-NEXT: [[TMP93:%.*]] = fptosi float [[TMP90]] to i32
-; CHECK-NEXT: [[TMP94:%.*]] = call fast float @llvm.fabs.f32(float [[TMP92]])
-; CHECK-NEXT: [[TMP95:%.*]] = call fast float @llvm.fabs.f32(float [[TMP87]])
-; CHECK-NEXT: [[TMP96:%.*]] = fcmp fast oge float [[TMP94]], [[TMP95]]
-; CHECK-NEXT: [[TMP97:%.*]] = select i1 [[TMP96]], i32 [[TMP85]], i32 0
-; CHECK-NEXT: [[TMP98:%.*]] = add i32 [[TMP93]], [[TMP97]]
-; CHECK-NEXT: [[TMP99:%.*]] = mul i32 [[TMP98]], [[TMP82]]
+; CHECK-NEXT: [[TMP99:%.*]] = mul i32 [[TMP93]], [[TMP82]]
; CHECK-NEXT: [[TMP100:%.*]] = sub i32 [[TMP81]], [[TMP99]]
; CHECK-NEXT: [[TMP103:%.*]] = trunc i32 [[TMP100]] to i16
; CHECK-NEXT: [[TMP104:%.*]] = insertelement <4 x i16> [[TMP78]], i16 [[TMP103]], i64 3
@@ -3238,170 +2972,115 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_sext_i32_i16 s4, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b32 s4, s10, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s6
+; GFX6-NEXT: s_ashr_i32 s6, s8, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_sext_i32_i16 s6, s11
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_sext_i32_i16 s6, s9
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s6
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: s_ashr_i32 s4, s11, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: s_ashr_i32 s4, s9, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s11
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s6, v1
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v2, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s11, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_ashr_i32 s5, s9, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s9, 16
-; GFX6-NEXT: s_lshr_b32 s7, s11, 16
-; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v5, v5
-; GFX6-NEXT: v_mad_f32 v4, -v5, v2, v4
-; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v5
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s7
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: s_lshr_b32 s4, s11, 16
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s5, v1
+; GFX6-NEXT: v_mul_lo_u32 v1, v2, s11
+; GFX6-NEXT: v_mul_lo_u32 v2, v3, s4
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: s_lshr_b32 s4, s9, 16
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s6, v2
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s8, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
-; GFX9-NEXT: s_xor_b32 s4, s9, s8
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s10, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v4
+; GFX9-NEXT: s_sext_i32_i16 s9, s3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
+; GFX9-NEXT: s_sext_i32_i16 s8, s1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
+; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s3
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s1
+; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v1, s4, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s10
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT: s_sext_i32_i16 s8, s1
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v4
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: s_xor_b32 s0, s8, s2
-; GFX9-NEXT: v_rcp_f32_e32 v5, v3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v4, -v5, v3, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v3|
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s3
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v5
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: s_ashr_i32 s2, s1, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s2
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
-; GFX9-NEXT: s_xor_b32 s0, s2, s3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
-; GFX9-NEXT: v_trunc_f32_e32 v6, v6
-; GFX9-NEXT: v_mad_f32 v5, -v6, v4, v5
-; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v4|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v6
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT: v_sub_u32_e32 v5, s9, v1
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_sub_u32_e32 v3, s2, v4
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = srem <4 x i16> %x, %y
@@ -3417,60 +3096,53 @@ define amdgpu_kernel void @udiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP15]] to i3
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i32 [[TMP10]] to i3
; CHECK-NEXT: store i3 [[TMP17]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: udiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0xb
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
+; GFX6-NEXT: s_and_b32 s3, s2, 7
+; GFX6-NEXT: s_bfe_u32 s2, s2, 0x30008
; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s4, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s3, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_and_b32 s2, s2, 7
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, s2
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX9-NEXT: global_store_byte v2, v0, s[0:1]
+; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i3 %x, %y
store i3 %r, ptr addrspace(1) %out
@@ -3485,17 +3157,13 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP10]], [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i3
; CHECK-NEXT: store i3 [[TMP19]], ptr addrspace(1) [[OUT]], align 1
@@ -3504,44 +3172,42 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX6-LABEL: urem_i3:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s3, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX6-NEXT: s_bfe_u32 s1, s6, 0x30008
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s1
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s0, s6, 7
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b32 s2, s6, 8
-; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: s_and_b32 s1, s2, 7
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s1
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_and_b32 s0, s2, 7
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: s_lshr_b32 s0, s2, 8
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
@@ -3559,75 +3225,57 @@ define amdgpu_kernel void @sdiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i3 [[X:%.*]], i3 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i3 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i3 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP18]] to i3
+; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP13]] to i3
; CHECK-NEXT: store i3 [[TMP21]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
; GFX6-LABEL: sdiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_bfe_i32 s5, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: s_bfe_i32 s1, s0, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bfe_i32 s1, s0, 0x30008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX9-NEXT: s_bfe_i32 s0, s0, 0x30000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s3, s2, 0x30008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_bfe_i32 s2, s2, 0x30000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv i3 %x, %y
@@ -3640,23 +3288,16 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i3 [[X:%.*]], i3 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i3 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i3 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP13]], [[TMP2]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP1]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i3
; CHECK-NEXT: store i3 [[TMP23]], ptr addrspace(1) [[OUT]], align 1
@@ -3665,56 +3306,44 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX6-LABEL: srem_i3:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_bfe_i32 s3, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_bfe_i32 s0, s6, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s0, s2, 0x30008
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_bfe_i32 s1, s2, 0x30000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GFX9-NEXT: s_xor_b32 s0, s1, s0
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_lshr_b32 s3, s2, 8
-; GFX9-NEXT: s_or_b32 s6, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s6, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x30000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s2, 8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -3735,17 +3364,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i16
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <3 x i16> poison, i16 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x i16> [[Y]], i64 1
@@ -3754,17 +3379,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i16
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i16
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <3 x i16> [[TMP20]], i16 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x i16> [[Y]], i64 2
@@ -3773,17 +3394,13 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i16
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i16
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <3 x i16> [[TMP40]], i16 [[TMP59]], i64 2
; CHECK-NEXT: store <3 x i16> [[TMP60]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -3797,87 +3414,75 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: s_lshr_b32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT: s_lshr_b32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
-; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_trunc_f32_e32 v2, v5
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_rcp_f32_e32 v7, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX9-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v6, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v6, v0, s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -3894,17 +3499,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i16
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x i16> poison, i16 [[TMP21]], i64 0
@@ -3915,17 +3516,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i16
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <3 x i16> [[TMP22]], i16 [[TMP43]], i64 1
@@ -3936,17 +3533,13 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <3 x i16> [[TMP44]], i16 [[TMP65]], i64 2
@@ -3960,100 +3553,90 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: s_and_b32 s4, s10, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_lshr_b32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
; GFX6-NEXT: s_and_b32 s6, s9, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s6
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_lshr_b32 s8, s0, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mad_f32 v2, -v5, v1, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s3
-; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1
-; GFX9-NEXT: v_rcp_f32_e32 v6, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_f32_e32 v2, v5, v6
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: s_and_b32 s9, s1, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX9-NEXT: v_mad_f32 v2, -v2, v3, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v1, s0, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v2
+; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4067,67 +3650,46 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i16
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i16
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x i16> poison, i16 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x i16> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i16 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i16 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i16
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i16
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <3 x i16> [[TMP24]], i16 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <3 x i16> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i16 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i16 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i16
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i16
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <3 x i16> [[TMP48]], i16 [[TMP71]], i64 2
; CHECK-NEXT: store <3 x i16> [[TMP72]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4139,118 +3701,77 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s5, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v3
; GFX6-NEXT: s_sext_i32_i16 s4, s9
; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v0, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
-; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX9-NEXT: s_xor_b32 s4, s5, s4
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s8, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s8, 0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: s_sext_i32_i16 s0, s3
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v4, -v5, v0, v4
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v5
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: global_store_short v1, v0, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v1, v2, s[6:7]
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4264,23 +3785,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i16> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i16
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x i16> poison, i16 [[TMP25]], i64 0
@@ -4288,23 +3802,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x i16> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i16 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i16 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i16
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <3 x i16> [[TMP26]], i16 [[TMP51]], i64 1
@@ -4312,23 +3819,16 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x i16> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i16 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i16 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i16
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <3 x i16> [[TMP52]], i16 [[TMP77]], i64 2
@@ -4344,130 +3844,90 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_sext_i32_i16 s4, s10
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
+; GFX6-NEXT: s_sext_i32_i16 s4, s8
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: s_ashr_i32 s5, s10, 16
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
-; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: s_ashr_i32 s4, s8, 16
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
-; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT: s_or_b32 s7, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s7, 0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v4
+; GFX6-NEXT: s_sext_i32_i16 s4, s9
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s4, s10, 16
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s4
; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_lshr_b32 s4, s8, 16
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s8, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
-; GFX9-NEXT: s_xor_b32 s4, s9, s8
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s10, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX9-NEXT: s_sext_i32_i16 s7, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s7
+; GFX9-NEXT: s_sext_i32_i16 s6, s0
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s6
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_ashr_i32 s8, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s8
+; GFX9-NEXT: s_sext_i32_i16 s3, s3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: s_sext_i32_i16 s9, s1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v1, s4, v2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s10
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_sext_i32_i16 s3, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: s_xor_b32 s0, s3, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s0, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT: v_sub_u32_e32 v2, s3, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4484,17 +3944,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP17]] to i15
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP12]] to i15
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <3 x i15> poison, i15 [[TMP19]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x i15> [[X]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x i15> [[Y]], i64 1
@@ -4503,17 +3959,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP25:%.*]] = uitofp fast i32 [[TMP23]] to float
; CHECK-NEXT: [[TMP26:%.*]] = uitofp fast i32 [[TMP24]] to float
; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast float [[TMP25]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP31]] to float
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast float [[TMP33]], [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fneg fast float [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP30]], float [[TMP26]], float [[TMP25]])
; CHECK-NEXT: [[TMP32:%.*]] = fptoui float [[TMP29]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.fabs.f32(float [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.fabs.f32(float [[TMP26]])
-; CHECK-NEXT: [[TMP35:%.*]] = fcmp fast oge float [[TMP33]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP35]], i32 1, i32 0
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP36]]
-; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP37]] to i15
+; CHECK-NEXT: [[TMP39:%.*]] = trunc i32 [[TMP32]] to i15
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <3 x i15> [[TMP20]], i15 [[TMP39]], i64 1
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x i15> [[X]], i64 2
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x i15> [[Y]], i64 2
@@ -4522,17 +3974,13 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP45:%.*]] = uitofp fast i32 [[TMP43]] to float
; CHECK-NEXT: [[TMP46:%.*]] = uitofp fast i32 [[TMP44]] to float
; CHECK-NEXT: [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP45]] to i32
+; CHECK-NEXT: [[TMP50:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i32 [[TMP50]] to float
+; CHECK-NEXT: [[TMP48:%.*]] = fmul fast float [[TMP51]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
-; CHECK-NEXT: [[TMP50:%.*]] = fneg fast float [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP50]], float [[TMP46]], float [[TMP45]])
; CHECK-NEXT: [[TMP52:%.*]] = fptoui float [[TMP49]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.fabs.f32(float [[TMP51]])
-; CHECK-NEXT: [[TMP54:%.*]] = call fast float @llvm.fabs.f32(float [[TMP46]])
-; CHECK-NEXT: [[TMP55:%.*]] = fcmp fast oge float [[TMP53]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i32 1, i32 0
-; CHECK-NEXT: [[TMP57:%.*]] = add i32 [[TMP52]], [[TMP56]]
-; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP57]] to i15
+; CHECK-NEXT: [[TMP59:%.*]] = trunc i32 [[TMP52]] to i15
; CHECK-NEXT: [[TMP60:%.*]] = insertelement <3 x i15> [[TMP40]], i15 [[TMP59]], i64 2
; CHECK-NEXT: store <3 x i15> [[TMP60]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4545,45 +3993,39 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_mov_b32 s0, s8
-; GFX6-NEXT: s_and_b32 s8, s6, 0x7fff
+; GFX6-NEXT: s_and_b32 s1, s6, 0x7fff
; GFX6-NEXT: s_and_b32 s6, s4, 0x7fff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_and_b32 s6, s10, 0x7fff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT: s_mov_b32 s0, s8
+; GFX6-NEXT: s_and_b32 s8, s10, 0x7fff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], 30
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
; GFX6-NEXT: s_bfe_u32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: s_bfe_u32 s9, s10, 0xf000f
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s9
-; GFX6-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
; GFX6-NEXT: s_and_b32 s5, s6, 0x7fff
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, v4, v5
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: s_bfe_u32 s10, s10, 0xf000f
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_mad_f32 v4, -v0, v3, v4
-; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX6-NEXT: v_rcp_f32_e32 v6, v1
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc
-; GFX6-NEXT: v_mul_f32_e32 v0, v5, v6
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s10
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0
-; GFX6-NEXT: v_mad_f32 v0, -v0, v1, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
@@ -4592,56 +4034,51 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: v_and_b32_e32 v0, 0x1fff, v1
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v3i15:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_and_b32 s3, s6, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
; GFX9-NEXT: s_and_b32 s5, s2, 0x7fff
; GFX9-NEXT: s_bfe_u32 s8, s2, 0xf000f
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[6:7], 30
+; GFX9-NEXT: s_and_b32 s2, s6, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[6:7], 30
; GFX9-NEXT: s_bfe_u32 s3, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_mul_f32_e32 v0, v5, v6
; GFX9-NEXT: s_and_b32 s4, s4, 0x7fff
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_mad_f32 v5, -v0, v4, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s4
-; GFX9-NEXT: v_rcp_f32_e32 v7, v1
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v0, vcc
-; GFX9-NEXT: v_mul_f32_e32 v0, v6, v7
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s8
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT: v_mul_f32_e32 v0, v3, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v0
-; GFX9-NEXT: v_mad_f32 v0, -v0, v1, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v5, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 15, v3
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: global_store_dword v4, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v4, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = udiv <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -4658,17 +4095,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP5:%.*]] = uitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP6:%.*]] = uitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = fneg fast float [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP10]], float [[TMP6]], float [[TMP5]])
; CHECK-NEXT: [[TMP12:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.fabs.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast oge float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 1, i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP17]], [[TMP4]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[TMP12]], [[TMP4]]
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP3]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP19]] to i15
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x i15> poison, i15 [[TMP21]], i64 0
@@ -4679,17 +4112,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP27:%.*]] = uitofp fast i32 [[TMP25]] to float
; CHECK-NEXT: [[TMP28:%.*]] = uitofp fast i32 [[TMP26]] to float
; CHECK-NEXT: [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast float [[TMP27]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i32 [[TMP33]] to float
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast float [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
-; CHECK-NEXT: [[TMP32:%.*]] = fneg fast float [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP32]], float [[TMP28]], float [[TMP27]])
; CHECK-NEXT: [[TMP34:%.*]] = fptoui float [[TMP31]] to i32
-; CHECK-NEXT: [[TMP35:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.fabs.f32(float [[TMP28]])
-; CHECK-NEXT: [[TMP37:%.*]] = fcmp fast oge float [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i32 1, i32 0
-; CHECK-NEXT: [[TMP39:%.*]] = add i32 [[TMP34]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP26]]
+; CHECK-NEXT: [[TMP40:%.*]] = mul i32 [[TMP34]], [[TMP26]]
; CHECK-NEXT: [[TMP41:%.*]] = sub i32 [[TMP25]], [[TMP40]]
; CHECK-NEXT: [[TMP43:%.*]] = trunc i32 [[TMP41]] to i15
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <3 x i15> [[TMP22]], i15 [[TMP43]], i64 1
@@ -4700,17 +4129,13 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP49:%.*]] = uitofp fast i32 [[TMP47]] to float
; CHECK-NEXT: [[TMP50:%.*]] = uitofp fast i32 [[TMP48]] to float
; CHECK-NEXT: [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP49]] to i32
+; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast i32 [[TMP54]] to float
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast float [[TMP55]], [[TMP51]]
; CHECK-NEXT: [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
-; CHECK-NEXT: [[TMP54:%.*]] = fneg fast float [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP54]], float [[TMP50]], float [[TMP49]])
; CHECK-NEXT: [[TMP56:%.*]] = fptoui float [[TMP53]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = call fast float @llvm.fabs.f32(float [[TMP55]])
-; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.fabs.f32(float [[TMP50]])
-; CHECK-NEXT: [[TMP59:%.*]] = fcmp fast oge float [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP59]], i32 1, i32 0
-; CHECK-NEXT: [[TMP61:%.*]] = add i32 [[TMP56]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], [[TMP48]]
+; CHECK-NEXT: [[TMP62:%.*]] = mul i32 [[TMP56]], [[TMP48]]
; CHECK-NEXT: [[TMP63:%.*]] = sub i32 [[TMP47]], [[TMP62]]
; CHECK-NEXT: [[TMP65:%.*]] = trunc i32 [[TMP63]] to i15
; CHECK-NEXT: [[TMP66:%.*]] = insertelement <3 x i15> [[TMP44]], i15 [[TMP65]], i64 2
@@ -4729,57 +4154,52 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_and_b32 s8, s6, 0x7fff
; GFX6-NEXT: s_and_b32 s6, s4, 0x7fff
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_and_b32 s6, s10, 0x7fff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], 30
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_and_b32 s5, s6, 0x7fff
-; GFX6-NEXT: s_bfe_u32 s6, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: s_bfe_u32 s11, s10, 0xf000f
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: v_rcp_f32_e32 v2, v3
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s8
-; GFX6-NEXT: s_lshr_b32 s4, s4, 15
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s10, v0
-; GFX6-NEXT: v_mul_f32_e32 v0, v1, v2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT: s_and_b32 s11, s10, 0x7fff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11
+; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], 30
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s5, s6, 0x7fff
+; GFX6-NEXT: s_bfe_u32 s6, s4, 0xf000f
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT: s_bfe_u32 s12, s10, 0xf000f
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s12
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
-; GFX6-NEXT: v_mad_f32 v1, -v0, v3, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_rcp_f32_e32 v6, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v3
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
-; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_mad_f32 v1, -v1, v2, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX6-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: s_lshr_b32 s6, s4, 15
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s6
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v2, s5
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: s_lshr_b32 s9, s10, 15
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v0
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v1
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 15, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_and_b32_e32 v0, 0x1fff, v1
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v3i15:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -4787,52 +4207,46 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_and_b32 s5, s6, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_and_b32 s9, s4, 0x7fff
+; GFX9-NEXT: s_and_b32 s4, s6, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
-; GFX9-NEXT: s_and_b32 s8, s4, 0x7fff
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], 30
-; GFX9-NEXT: s_bfe_u32 s5, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s5
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: s_bfe_u32 s9, s2, 0xf000f
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX9-NEXT: s_and_b32 s3, s4, 0x7fff
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s9
-; GFX9-NEXT: v_rcp_f32_e32 v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s3
-; GFX9-NEXT: v_mul_f32_e32 v1, v5, v6
-; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s8
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s6, 0xf000f
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX9-NEXT: s_bfe_u32 s8, s2, 0xf000f
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s3
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_rcp_f32_e32 v7, v3
-; GFX9-NEXT: v_mad_f32 v5, -v1, v4, v5
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v6, v7
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_f32 v4, -v4, v3, v6
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: s_lshr_b32 s4, s6, 15
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s4
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v5, vcc
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s3
; GFX9-NEXT: s_lshr_b32 s3, s2, 15
+; GFX9-NEXT: v_sub_u32_e32 v1, s3, v1
; GFX9-NEXT: v_sub_u32_e32 v4, s2, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s3, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v3
+; GFX9-NEXT: v_sub_u32_e32 v0, s9, v3
+; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v4
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v1
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
@@ -4851,67 +4265,46 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i15> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i15 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i15 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP20]] to i15
+; CHECK-NEXT: [[TMP23:%.*]] = trunc i32 [[TMP15]] to i15
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x i15> poison, i15 [[TMP23]], i64 0
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x i15> [[X]], i64 1
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x i15> [[Y]], i64 1
; CHECK-NEXT: [[TMP27:%.*]] = sext i15 [[TMP25]] to i32
; CHECK-NEXT: [[TMP28:%.*]] = sext i15 [[TMP26]] to i32
-; CHECK-NEXT: [[TMP29:%.*]] = xor i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i32 [[TMP29]], 30
-; CHECK-NEXT: [[TMP31:%.*]] = or i32 [[TMP30]], 1
; CHECK-NEXT: [[TMP32:%.*]] = sitofp fast i32 [[TMP27]] to float
; CHECK-NEXT: [[TMP33:%.*]] = sitofp fast i32 [[TMP28]] to float
; CHECK-NEXT: [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast float [[TMP32]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i32 [[TMP30]] to float
+; CHECK-NEXT: [[TMP35:%.*]] = fmul fast float [[TMP31]], [[TMP34]]
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fneg fast float [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP37]], float [[TMP33]], float [[TMP32]])
; CHECK-NEXT: [[TMP39:%.*]] = fptosi float [[TMP36]] to i32
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.fabs.f32(float [[TMP38]])
-; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.fabs.f32(float [[TMP33]])
-; CHECK-NEXT: [[TMP42:%.*]] = fcmp fast oge float [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP44:%.*]] = add i32 [[TMP39]], [[TMP43]]
-; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP44]] to i15
+; CHECK-NEXT: [[TMP47:%.*]] = trunc i32 [[TMP39]] to i15
; CHECK-NEXT: [[TMP48:%.*]] = insertelement <3 x i15> [[TMP24]], i15 [[TMP47]], i64 1
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <3 x i15> [[X]], i64 2
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x i15> [[Y]], i64 2
; CHECK-NEXT: [[TMP51:%.*]] = sext i15 [[TMP49]] to i32
; CHECK-NEXT: [[TMP52:%.*]] = sext i15 [[TMP50]] to i32
-; CHECK-NEXT: [[TMP53:%.*]] = xor i32 [[TMP51]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = ashr i32 [[TMP53]], 30
-; CHECK-NEXT: [[TMP55:%.*]] = or i32 [[TMP54]], 1
; CHECK-NEXT: [[TMP56:%.*]] = sitofp fast i32 [[TMP51]] to float
; CHECK-NEXT: [[TMP57:%.*]] = sitofp fast i32 [[TMP52]] to float
; CHECK-NEXT: [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast float [[TMP56]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[TMP38]], 1
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast i32 [[TMP41]] to float
+; CHECK-NEXT: [[TMP59:%.*]] = fmul fast float [[TMP40]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = fneg fast float [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP61]], float [[TMP57]], float [[TMP56]])
; CHECK-NEXT: [[TMP63:%.*]] = fptosi float [[TMP60]] to i32
-; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.fabs.f32(float [[TMP62]])
-; CHECK-NEXT: [[TMP65:%.*]] = call fast float @llvm.fabs.f32(float [[TMP57]])
-; CHECK-NEXT: [[TMP66:%.*]] = fcmp fast oge float [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP67:%.*]] = select i1 [[TMP66]], i32 [[TMP55]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = add i32 [[TMP63]], [[TMP67]]
-; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP68]] to i15
+; CHECK-NEXT: [[TMP71:%.*]] = trunc i32 [[TMP63]] to i15
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <3 x i15> [[TMP48]], i15 [[TMP71]], i64 2
; CHECK-NEXT: store <3 x i15> [[TMP72]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
@@ -4924,133 +4317,92 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_bfe_i32 s7, s4, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s7
+; GFX6-NEXT: s_bfe_i32 s1, s4, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s1, s10, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_bfe_i32 s1, s4, 0xf000f
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b64 s[12:13], s[4:5], 30
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_bfe_i32 s1, s10, 0xf000f
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s1
+; GFX6-NEXT: s_bfe_i32 s1, s12, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s1
+; GFX6-NEXT: s_bfe_i32 s1, s6, 0xf0000
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v1
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: s_mov_b32 s0, s8
-; GFX6-NEXT: s_mov_b32 s1, s9
-; GFX6-NEXT: s_lshr_b64 s[8:9], s[4:5], 30
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_xor_b32 s5, s5, s7
-; GFX6-NEXT: s_ashr_i32 s5, s5, 30
-; GFX6-NEXT: s_or_b32 s5, s5, 1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[12:13], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[12:13], s[12:13], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s5, s5, 0
-; GFX6-NEXT: s_bfe_i32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s5, v2
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s7, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_cselect_b32 s4, s7, 0
-; GFX6-NEXT: s_bfe_i32 s5, s8, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, s4, v3
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v4
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_and_b32_e32 v0, 0x1fff, v1
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v3i15:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_bfe_i32 s3, s6, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: s_bfe_i32 s5, s2, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: s_xor_b32 s3, s5, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 30
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
-; GFX9-NEXT: s_or_b32 s3, s3, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[10:11], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, 0
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
+; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf0000
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_bfe_i32 s2, s2, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s5
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: v_add_u32_e32 v3, s3, v3
-; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT: s_or_b32 s5, s2, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: s_cselect_b32 s2, s5, 0
-; GFX9-NEXT: s_bfe_i32 s3, s8, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
-; GFX9-NEXT: v_add_u32_e32 v4, s2, v4
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_i32 s2, s8, 0xf0000
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_bfe_i32 s2, s4, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v5, v1, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v1, -v5, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v5
+; GFX9-NEXT: v_rcp_f32_e32 v1, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v0, v2, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v3, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = sdiv <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -5064,23 +4416,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i15> [[Y]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = sext i15 [[TMP1]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = sext i15 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = ashr i32 [[TMP5]], 30
-; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sitofp fast i32 [[TMP3]] to float
; CHECK-NEXT: [[TMP9:%.*]] = sitofp fast i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast float [[TMP8]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast float [[TMP16]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = fneg fast float [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP13]], float [[TMP9]], float [[TMP8]])
; CHECK-NEXT: [[TMP15:%.*]] = fptosi float [[TMP12]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast oge float [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP20]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul i32 [[TMP15]], [[TMP4]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP3]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = trunc i32 [[TMP22]] to i15
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x i15> poison, i15 [[TMP25]], i64 0
@@ -5088,23 +4433,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x i15> [[Y]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = sext i15 [[TMP27]] to i32
; CHECK-NEXT: [[TMP30:%.*]] = sext i15 [[TMP28]] to i32
-; CHECK-NEXT: [[TMP31:%.*]] = xor i32 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i32 [[TMP31]], 30
-; CHECK-NEXT: [[TMP33:%.*]] = or i32 [[TMP32]], 1
; CHECK-NEXT: [[TMP34:%.*]] = sitofp fast i32 [[TMP29]] to float
; CHECK-NEXT: [[TMP35:%.*]] = sitofp fast i32 [[TMP30]] to float
; CHECK-NEXT: [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast float [[TMP34]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i32 [[TMP32]] to float
+; CHECK-NEXT: [[TMP37:%.*]] = fmul fast float [[TMP33]], [[TMP36]]
; CHECK-NEXT: [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = fneg fast float [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP39]], float [[TMP35]], float [[TMP34]])
; CHECK-NEXT: [[TMP41:%.*]] = fptosi float [[TMP38]] to i32
-; CHECK-NEXT: [[TMP42:%.*]] = call fast float @llvm.fabs.f32(float [[TMP40]])
-; CHECK-NEXT: [[TMP43:%.*]] = call fast float @llvm.fabs.f32(float [[TMP35]])
-; CHECK-NEXT: [[TMP44:%.*]] = fcmp fast oge float [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i32 [[TMP33]], i32 0
-; CHECK-NEXT: [[TMP46:%.*]] = add i32 [[TMP41]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP46]], [[TMP30]]
+; CHECK-NEXT: [[TMP47:%.*]] = mul i32 [[TMP41]], [[TMP30]]
; CHECK-NEXT: [[TMP48:%.*]] = sub i32 [[TMP29]], [[TMP47]]
; CHECK-NEXT: [[TMP51:%.*]] = trunc i32 [[TMP48]] to i15
; CHECK-NEXT: [[TMP52:%.*]] = insertelement <3 x i15> [[TMP26]], i15 [[TMP51]], i64 1
@@ -5112,23 +4450,16 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x i15> [[Y]], i64 2
; CHECK-NEXT: [[TMP55:%.*]] = sext i15 [[TMP53]] to i32
; CHECK-NEXT: [[TMP56:%.*]] = sext i15 [[TMP54]] to i32
-; CHECK-NEXT: [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = ashr i32 [[TMP57]], 30
-; CHECK-NEXT: [[TMP59:%.*]] = or i32 [[TMP58]], 1
; CHECK-NEXT: [[TMP60:%.*]] = sitofp fast i32 [[TMP55]] to float
; CHECK-NEXT: [[TMP61:%.*]] = sitofp fast i32 [[TMP56]] to float
; CHECK-NEXT: [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast float [[TMP60]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i32 [[TMP43]] to float
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast float [[TMP44]], [[TMP62]]
; CHECK-NEXT: [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
-; CHECK-NEXT: [[TMP65:%.*]] = fneg fast float [[TMP64]]
-; CHECK-NEXT: [[TMP66:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP65]], float [[TMP61]], float [[TMP60]])
; CHECK-NEXT: [[TMP67:%.*]] = fptosi float [[TMP64]] to i32
-; CHECK-NEXT: [[TMP68:%.*]] = call fast float @llvm.fabs.f32(float [[TMP66]])
-; CHECK-NEXT: [[TMP69:%.*]] = call fast float @llvm.fabs.f32(float [[TMP61]])
-; CHECK-NEXT: [[TMP70:%.*]] = fcmp fast oge float [[TMP68]], [[TMP69]]
-; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP72:%.*]] = add i32 [[TMP67]], [[TMP71]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP72]], [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = mul i32 [[TMP67]], [[TMP56]]
; CHECK-NEXT: [[TMP74:%.*]] = sub i32 [[TMP55]], [[TMP73]]
; CHECK-NEXT: [[TMP77:%.*]] = trunc i32 [[TMP74]] to i15
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <3 x i15> [[TMP52]], i15 [[TMP77]], i64 2
@@ -5147,149 +4478,108 @@ define amdgpu_kernel void @srem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
; GFX6-NEXT: s_lshr_b64 s[8:9], s[4:5], 30
; GFX6-NEXT: s_bfe_i32 s5, s4, 0xf0000
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_bfe_i32 s12, s10, 0xf0000
+; GFX6-NEXT: s_bfe_i32 s9, s10, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s9
+; GFX6-NEXT: s_bfe_i32 s12, s4, 0xf000f
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
+; GFX6-NEXT: s_lshr_b32 s9, s4, 15
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s12
-; GFX6-NEXT: s_xor_b32 s5, s12, s5
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: s_bfe_i32 s12, s10, 0xf000f
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s12
+; GFX6-NEXT: v_rcp_f32_e32 v1, v1
; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], 30
-; GFX6-NEXT: s_ashr_i32 s5, s5, 30
-; GFX6-NEXT: s_and_b32 s7, s6, 0x7fff
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_lshr_b32 s11, s10, 15
-; GFX6-NEXT: s_and_b32 s9, s8, 0x7fff
-; GFX6-NEXT: s_lshr_b32 s14, s4, 15
-; GFX6-NEXT: s_or_b32 s5, s5, 1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[12:13], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[12:13], s[12:13], exec
-; GFX6-NEXT: s_cselect_b32 s5, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s5, v2
; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: s_bfe_i32 s4, s4, 0xf000f
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_bfe_i32 s5, s10, 0xf000f
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s10, v0
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s10, s4, 1
-; GFX6-NEXT: v_mul_f32_e32 v3, v0, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v0, -v3, v1, v0
-; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s10, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v3
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
; GFX6-NEXT: s_bfe_i32 s4, s8, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_bfe_i32 s5, s6, 0xf0000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v4, v1
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s14
-; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT: v_trunc_f32_e32 v4, v4
-; GFX6-NEXT: v_mad_f32 v3, -v4, v1, v3
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v1|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
+; GFX6-NEXT: s_bfe_i32 s4, s6, 0xf0000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX6-NEXT: v_trunc_f32_e32 v1, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX6-NEXT: s_and_b32 s5, s8, 0x7fff
; GFX6-NEXT: v_mul_lo_u32 v1, v1, s9
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s11, v0
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v2, s5
+; GFX6-NEXT: s_lshr_b32 s11, s10, 15
+; GFX6-NEXT: s_and_b32 s7, s6, 0x7fff
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s11, v1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s7, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 15, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0x7fff, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s7, v1
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT: v_and_b32_e32 v2, 0x7fff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 15, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_and_b32_e32 v0, 0x1fff, v1
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 offset:4
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v3i15:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], 30
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_bfe_i32 s3, s6, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0xf0000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], 30
-; GFX9-NEXT: s_bfe_i32 s7, s2, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s7
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_xor_b32 s5, s7, s5
-; GFX9-NEXT: s_ashr_i32 s5, s5, 30
-; GFX9-NEXT: s_lshr_b32 s3, s2, 15
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: s_and_b32 s9, s4, 0x7fff
-; GFX9-NEXT: s_and_b32 s12, s8, 0x7fff
-; GFX9-NEXT: s_lshr_b32 s13, s6, 15
-; GFX9-NEXT: s_or_b32 s5, s5, 1
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[10:11], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
-; GFX9-NEXT: s_cselect_b32 s5, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s5, v2
-; GFX9-NEXT: s_bfe_i32 s5, s6, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
-; GFX9-NEXT: s_bfe_i32 s6, s2, 0xf000f
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s6
-; GFX9-NEXT: v_rcp_f32_e32 v3, v1
-; GFX9-NEXT: s_xor_b32 s5, s6, s5
-; GFX9-NEXT: s_ashr_i32 s5, s5, 30
-; GFX9-NEXT: s_or_b32 s5, s5, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v1, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v2|, |v1|
-; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX9-NEXT: s_cselect_b32 s5, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v1, s5, v3
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: s_bfe_i32 s9, s6, 0xf000f
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: s_bfe_i32 s5, s2, 0xf000f
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
; GFX9-NEXT: s_bfe_i32 s5, s8, 0xf0000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s5
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s3, s4, 0x7fff
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: s_bfe_i32 s4, s4, 0xf0000
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s5
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s13
-; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v4
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s12
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX9-NEXT: s_lshr_b32 s4, s6, 15
+; GFX9-NEXT: s_and_b32 s7, s8, 0x7fff
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s4
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s7
+; GFX9-NEXT: s_lshr_b32 s4, s2, 15
+; GFX9-NEXT: v_sub_u32_e32 v5, s4, v1
; GFX9-NEXT: v_sub_u32_e32 v4, s2, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s3, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s9, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0x7fff, v4
-; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v5
+; GFX9-NEXT: v_sub_u32_e32 v0, s3, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 0x7fff, v5
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 15, v4
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 15, v3
+; GFX9-NEXT: v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x1fff, v1
-; GFX9-NEXT: global_store_short v3, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_short v2, v0, s[0:1] offset:4
; GFX9-NEXT: s_endpgm
%r = srem <3 x i15> %x, %y
store <3 x i15> %r, ptr addrspace(1) %out
@@ -5318,6 +4608,7 @@ define amdgpu_kernel void @udiv_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 20, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i32_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -5355,6 +4646,7 @@ define amdgpu_kernel void @udiv_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -5390,6 +4682,7 @@ define amdgpu_kernel void @udiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -5431,6 +4724,7 @@ define amdgpu_kernel void @udiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -5476,6 +4770,7 @@ define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out, <
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i32_mixed_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -5622,6 +4917,7 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
@@ -5706,6 +5002,7 @@ define amdgpu_kernel void @urem_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i32_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -5745,6 +5042,7 @@ define amdgpu_kernel void @urem_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -5781,6 +5079,7 @@ define amdgpu_kernel void @urem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -5823,6 +5122,7 @@ define amdgpu_kernel void @urem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -5955,6 +5255,7 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
@@ -6032,6 +5333,7 @@ define amdgpu_kernel void @sdiv_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i32_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -6072,6 +5374,7 @@ define amdgpu_kernel void @sdiv_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -6136,6 +5439,7 @@ define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -6208,6 +5512,7 @@ define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -6262,6 +5567,7 @@ define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out,
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -6441,6 +5747,7 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s3, v1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
@@ -6538,6 +5845,7 @@ define amdgpu_kernel void @srem_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i32_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -6581,6 +5889,7 @@ define amdgpu_kernel void @srem_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -6642,6 +5951,7 @@ define amdgpu_kernel void @srem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -6713,6 +6023,7 @@ define amdgpu_kernel void @srem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -6879,6 +6190,7 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
@@ -6980,6 +6292,7 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i64_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7029,6 +6342,7 @@ define amdgpu_kernel void @udiv_i64_pow2k_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7067,6 +6381,7 @@ define amdgpu_kernel void @udiv_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
@@ -7112,6 +6427,7 @@ define amdgpu_kernel void @udiv_v2i64_pow2k_denom(ptr addrspace(1) %out, <2 x i6
; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -7179,6 +6495,7 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out, <
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i64_mixed_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -7250,6 +6567,7 @@ define amdgpu_kernel void @udiv_v2i64_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_mov_b32_e32 v3, s7
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: udiv_v2i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -7316,6 +6634,7 @@ define amdgpu_kernel void @urem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i64_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7372,6 +6691,7 @@ define amdgpu_kernel void @urem_i64_pow2k_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7411,6 +6731,7 @@ define amdgpu_kernel void @urem_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
@@ -7458,6 +6779,7 @@ define amdgpu_kernel void @urem_v2i64_pow2k_denom(ptr addrspace(1) %out, <2 x i6
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v2i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -7512,6 +6834,7 @@ define amdgpu_kernel void @urem_v2i64_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_mov_b32_e32 v3, s5
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: urem_v2i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -7584,6 +6907,7 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i64_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7649,6 +6973,7 @@ define amdgpu_kernel void @sdiv_i64_pow2k_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -7833,6 +7158,7 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
@@ -8008,6 +7334,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2k_denom(ptr addrspace(1) %out, <2 x i6
; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v2i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -8099,6 +7426,7 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: ssdiv_v2i64_mixed_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -8469,6 +7797,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: sdiv_v2i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -8788,6 +8117,7 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i64_oddk_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -8861,6 +8191,7 @@ define amdgpu_kernel void @srem_i64_pow2k_denom(ptr addrspace(1) %out, i64 %x) {
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -9047,6 +8378,7 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
@@ -9226,6 +8558,7 @@ define amdgpu_kernel void @srem_v2i64_pow2k_denom(ptr addrspace(1) %out, <2 x i6
; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v2i64_pow2k_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
@@ -9574,6 +8907,7 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
+;
; GFX9-LABEL: srem_v2i64_pow2_shl_denom:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
@@ -9856,6 +9190,7 @@ define <2 x i32> @v_sdiv_i32_exact(<2 x i32> %num) {
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 12, v0
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 10, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-LABEL: v_sdiv_i32_exact:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9883,6 +9218,7 @@ define <2 x i64> @v_sdiv_i64_exact(<2 x i64> %num) {
; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 12
; GFX6-NEXT: v_ashr_i64 v[2:3], v[2:3], 10
; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-LABEL: v_sdiv_i64_exact:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9910,6 +9246,7 @@ define <2 x i32> @v_udiv_i32_exact(<2 x i32> %num) {
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 12, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 10, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-LABEL: v_udiv_i32_exact:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9937,6 +9274,7 @@ define <2 x i64> @v_udiv_i64_exact(<2 x i64> %num) {
; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], 12
; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], 10
; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-LABEL: v_udiv_i64_exact:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -10002,7 +9340,6 @@ define i64 @udiv_i64_gt_smax(i8 %size) {
; CHECK-NEXT: [[MINUS:%.*]] = sub nuw nsw i64 -1, [[ESIZE]]
; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[MINUS]], 10
; CHECK-NEXT: ret i64 [[DIV]]
-;
%esize = sext i8 %size to i64
%minus = sub nuw nsw i64 -1, %esize
%div = udiv i64 %minus, 10
@@ -10016,14 +9353,11 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x41200000
-; GFX6-NEXT: v_mul_f32_e32 v1, 0x3dcccccd, v0
-; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX6-NEXT: v_mad_f32 v0, -v1, s4, v0
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4
; GFX6-NEXT: v_mov_b32_e32 v1, 0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x3dcccccd, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: udiv_i64_9divbits:
@@ -10032,14 +9366,11 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_add_u32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x41200000
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x3dcccccd, v0
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX9-NEXT: v_mad_f32 v0, -v1, s4, v0
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x3dcccccd, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
; CHECK-LABEL: define i64 @udiv_i64_9divbits(
; CHECK-SAME: i8 [[SIZE:%.*]]) #[[ATTR0]] {
@@ -10050,19 +9381,14 @@ define i64 @udiv_i64_9divbits(i8 %size) {
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.amdgcn.rcp.f32(float 1.000000e+01)
-; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP10]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP9]], [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.trunc.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fneg float [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.fmad.ftz.f32(float [[TMP9]], float 1.000000e+01, float [[TMP5]])
; CHECK-NEXT: [[TMP11:%.*]] = fptoui float [[TMP8]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = call float @llvm.fabs.f32(float [[TMP10]])
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.fabs.f32(float 1.000000e+01)
-; CHECK-NEXT: [[TMP14:%.*]] = fcmp oge float [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i32 1, i32 0
-; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP11]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP16]] to i64
+; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: ret i64 [[TMP17]]
-;
%zextend = zext i8 %size to i64
%num = add nuw nsw i64 1, %zextend
%div = udiv i64 %num, 10
@@ -10083,7 +9409,6 @@ define <2 x i64> @srem_zero_zero() {
; CHECK-SAME: ) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: ret <2 x i64> poison
-;
entry:
%B = srem <2 x i64> zeroinitializer, zeroinitializer
ret <2 x i64> %B
@@ -10095,25 +9420,17 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_bfe_i32 s5, s5, 0x170000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_bfe_i32 s4, s4, 0x170000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_bfe_i32 s3, s3, 0x170000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX6-NEXT: s_bfe_i32 s2, s2, 0x170000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s2
; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7
@@ -10123,24 +9440,17 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; GFX9-LABEL: sdiv23:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s3, s3, 0x170000
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX9-NEXT: s_bfe_i32 s2, s2, 0x170000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2
@@ -10149,26 +9459,18 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i23 [[X:%.*]], i23 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = sext i23 [[X]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sext i23 [[Y]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = ashr i32 [[TMP3]], 30
-; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP7:%.*]] = sitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast float [[TMP6]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP14]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fneg fast float [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP11]], float [[TMP7]], float [[TMP6]])
; CHECK-NEXT: [[TMP13:%.*]] = fptosi float [[TMP10]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = call fast float @llvm.fabs.f32(float [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = call fast float @llvm.fabs.f32(float [[TMP7]])
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp fast oge float [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP13]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP18]] to i23
+; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP13]] to i23
; CHECK-NEXT: store i23 [[TMP19]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
-;
%r = sdiv i23 %x, %y
store i23 %r, ptr addrspace(1) %out
ret void
@@ -10300,7 +9602,6 @@ define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) {
; CHECK-NEXT: [[TMP41:%.*]] = trunc i32 [[TMP40]] to i24
; CHECK-NEXT: store i24 [[TMP41]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
-;
%r = sdiv i24 %x, %y
store i24 %r, ptr addrspace(1) %out
ret void
@@ -10317,14 +9618,12 @@ define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) {
; GFX6-NEXT: s_and_b32 s2, s2, 0x3fffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2
; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_trunc_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 6
@@ -10334,22 +9633,20 @@ define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) {
; GFX9-LABEL: udiv22:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s3, s3, 0x3fffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
; GFX9-NEXT: s_and_b32 s2, s2, 0x3fffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
-; GFX9-NEXT: global_store_short v3, v0, s[0:1]
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: v_and_b32_e32 v0, 0x3fffff, v0
-; GFX9-NEXT: global_store_byte_d16_hi v3, v0, s[0:1] offset:2
+; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2
; GFX9-NEXT: s_endpgm
; CHECK-LABEL: define amdgpu_kernel void @udiv22(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i22 [[X:%.*]], i22 [[Y:%.*]]) #[[ATTR0]] {
@@ -10358,20 +9655,15 @@ define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) {
; CHECK-NEXT: [[TMP3:%.*]] = uitofp fast i32 [[TMP1]] to float
; CHECK-NEXT: [[TMP4:%.*]] = uitofp fast i32 [[TMP2]] to float
; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast float [[TMP3]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP8]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = fneg fast float [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP8]], float [[TMP4]], float [[TMP3]])
; CHECK-NEXT: [[TMP10:%.*]] = fptoui float [[TMP7]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.fabs.f32(float [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.fabs.f32(float [[TMP4]])
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp fast oge float [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP15]] to i22
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP10]] to i22
; CHECK-NEXT: store i22 [[TMP16]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
-;
%r = udiv i22 %x, %y
store i22 %r, ptr addrspace(1) %out
ret void
@@ -10484,7 +9776,6 @@ define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) {
; CHECK-NEXT: [[TMP32:%.*]] = trunc i32 [[TMP31]] to i23
; CHECK-NEXT: store i23 [[TMP32]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
-;
%r = udiv i23 %x, %y
store i23 %r, ptr addrspace(1) %out
ret void
@@ -10595,7 +9886,6 @@ define amdgpu_kernel void @udiv24(ptr addrspace(1) %out, i24 %x, i24 %y) {
; CHECK-NEXT: [[TMP32:%.*]] = trunc i32 [[TMP31]] to i24
; CHECK-NEXT: store i24 [[TMP32]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
-;
%r = udiv i24 %x, %y
store i24 %r, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
index d60dc563ff020..5b2fd1f971b0f 100644
--- a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
+++ b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
@@ -10,18 +10,14 @@ define amdgpu_kernel void @divrem24_assume(ptr addrspace(1) %arg, i32 %arg1) {
; CHECK-NEXT: [[TMP0:%.*]] = uitofp fast i32 [[TMP]] to float
; CHECK-NEXT: [[TMP1:%.*]] = uitofp fast i32 [[ARG1]] to float
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast float [[TMP0]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP6]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP8]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP5]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call fast float @llvm.trunc.f32(float [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = fneg fast float [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.amdgcn.fmad.ftz.f32(float [[TMP5]], float [[TMP1]], float [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = fptoui float [[TMP4]] to i32
-; CHECK-NEXT: [[TMP8:%.*]] = call fast float @llvm.fabs.f32(float [[TMP6]])
-; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.fabs.f32(float [[TMP1]])
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp fast oge float [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP7]], [[TMP11]]
-; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG:%.*]], i64 [[TMP8]]
; CHECK-NEXT: store i32 0, ptr addrspace(1) [[TMP5]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index aeccbbf6c2d57..82ae949d0c8f5 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -605,24 +605,22 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB4_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX9-NEXT: s_add_i32 s2, s2, 1
; GFX9-NEXT: s_lshl_b32 s3, s3, 1
; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v5, vcc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -636,23 +634,21 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX10-NEXT: s_mov_b32 s2, 0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB4_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_and_b32 s3, 0xffff, s2
; GFX10-NEXT: s_add_i32 s2, s2, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX10-NEXT: s_lshl_b32 s3, s3, 1
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
+; GFX10-NEXT: v_mov_b32_e32 v2, s3
; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: s_cmpk_eq_i32 s3, 0x400
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_store_short v4, v2, s[0:1]
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB4_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -667,28 +663,24 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .LBB4_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_and_b32 s3, 0xffff, s2
; GFX11-NEXT: s_add_i32 s2, s2, 1
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX11-NEXT: s_lshl_b32 s3, s3, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mov_b32_e32 v2, s3
; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x400
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v3, vcc_lo
-; GFX11-NEXT: global_store_b16 v4, v2, s[0:1]
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB4_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -718,26 +710,24 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB5_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_and_b32 s4, 0xffff, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_add_i32 s3, s3, 1
; GFX9-NEXT: s_lshl_b32 s5, s4, 1
; GFX9-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NEXT: s_cmpk_eq_i32 s6, 0x400
-; GFX9-NEXT: v_sub_u32_e32 v2, s4, v2
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB5_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -751,25 +741,23 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB5_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_and_b32 s4, 0xffff, s3
; GFX10-NEXT: s_add_i32 s3, s3, 1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX10-NEXT: s_lshl_b32 s5, s4, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB5_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -784,31 +772,28 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB5_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_and_b32 s4, 0xffff, s3
; GFX11-NEXT: s_add_i32 s3, s3, 1
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX11-NEXT: s_lshl_b32 s5, s4, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_add_nc_u32 v1, 1, v1
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v0
-; GFX11-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v3, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v3, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX11-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB5_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -833,35 +818,28 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-LABEL: sdiv16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s2, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB6_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_sext_i32_i16 s4, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
-; GFX9-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX9-NEXT: s_add_i32 s3, s3, 1
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT: s_lshl_b32 s4, s5, 1
-; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
+; GFX9-NEXT: s_sext_i32_i16 s3, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX9-NEXT: s_add_i32 s2, s2, 1
+; GFX9-NEXT: s_lshl_b32 s3, s3, 1
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB6_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -871,33 +849,26 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: s_mov_b32 s2, 0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB6_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_sext_i32_i16 s4, s3
-; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX10-NEXT: s_xor_b32 s4, s4, s2
-; GFX10-NEXT: s_ashr_i32 s4, s4, 30
-; GFX10-NEXT: s_or_b32 s4, s4, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 s5, |v2|, |v0|
-; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_cselect_b32 s4, s4, 0
-; GFX10-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX10-NEXT: s_add_i32 s3, s3, 1
-; GFX10-NEXT: s_lshl_b32 s5, s5, 1
-; GFX10-NEXT: v_add_nc_u32_e32 v2, s4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: s_sext_i32_i16 s3, s2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX10-NEXT: s_add_i32 s2, s2, 1
+; GFX10-NEXT: s_lshl_b32 s3, s3, 1
+; GFX10-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX10-NEXT: s_cmpk_eq_i32 s3, 0x400
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB6_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -907,39 +878,32 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_sext_i32_i16 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: s_mov_b32 s2, 0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB6_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_sext_i32_i16 s4, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX11-NEXT: s_xor_b32 s4, s4, s2
-; GFX11-NEXT: s_ashr_i32 s4, s4, 30
+; GFX11-NEXT: s_sext_i32_i16 s3, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, s3
+; GFX11-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX11-NEXT: s_add_i32 s2, s2, 1
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_add_nc_u32 v1, 1, v1
+; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s4, s4, 1
+; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x400
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cmp_ge_f32_e64 s5, |v2|, |v0|
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, s4, 0
-; GFX11-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX11-NEXT: s_add_i32 s3, s3, 1
-; GFX11-NEXT: s_lshl_b32 s5, s5, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_add_nc_u32 v2, s4, v2
-; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -969,32 +933,25 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_sext_i32_i16 s2, s0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_rcp_f32_e32 v1, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: .LBB7_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_sext_i32_i16 s6, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s6
-; GFX9-NEXT: s_xor_b32 s4, s6, s2
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s7, s4, 1
-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s7, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_and_b32 s5, 0xffff, s3
; GFX9-NEXT: s_add_i32 s3, s3, 1
-; GFX9-NEXT: s_lshl_b32 s4, s5, 1
-; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
-; GFX9-NEXT: v_sub_u32_e32 v2, s6, v2
+; GFX9-NEXT: s_lshl_b32 s5, s5, 1
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v3, v2, s[0:1]
+; GFX9-NEXT: global_store_short v2, v1, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
@@ -1008,31 +965,24 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: .LBB7_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_sext_i32_i16 s4, s3
-; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX10-NEXT: s_xor_b32 s5, s4, s2
-; GFX10-NEXT: s_ashr_i32 s5, s5, 30
-; GFX10-NEXT: s_or_b32 s5, s5, 1
-; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 s6, |v2|, |v0|
-; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s5, s5, 0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, s5, v2
; GFX10-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX10-NEXT: s_add_i32 s3, s3, 1
+; GFX10-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX10-NEXT: s_lshl_b32 s5, s5, 1
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX10-NEXT: s_add_i32 s3, s3, 1
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX10-NEXT: s_and_b32 s4, s3, 0xffff
; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX10-NEXT: global_store_short v3, v2, s[0:1]
+; GFX10-NEXT: global_store_short v2, v1, s[0:1]
; GFX10-NEXT: s_cbranch_scc0 .LBB7_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
@@ -1047,39 +997,28 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_sext_i32_i16 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, v0
+; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_sext_i32_i16 s4, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX11-NEXT: s_xor_b32 s5, s4, s2
-; GFX11-NEXT: s_ashr_i32 s5, s5, 30
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s5, s5, 1
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v2, -v3, v0, v2
-; GFX11-NEXT: v_cmp_ge_f32_e64 s6, |v2|, |v0|
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s5, s5, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_nc_u32_e32 v2, s5, v2
; GFX11-NEXT: s_and_b32 s5, 0xffff, s3
-; GFX11-NEXT: s_add_i32 s3, s3, 1
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX11-NEXT: s_lshl_b32 s5, s5, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX11-NEXT: v_mov_b32_e32 v3, s5
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, s4, v2
+; GFX11-NEXT: s_add_i32 s3, s3, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_add_nc_u32 v1, 1, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v0
+; GFX11-NEXT: v_trunc_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX11-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
; GFX11-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
-; GFX11-NEXT: global_store_b16 v3, v2, s[0:1]
+; GFX11-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX11-NEXT: s_cbranch_scc0 .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index ecdae6ab20d3e..d15f296d7764a 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -1512,61 +1512,36 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v14, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v15, v1
-; GFX10-NEXT: v_rcp_f32_e32 v16, v10
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v19, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v17, v12
-; GFX10-NEXT: v_xor_b32_sdwa v0, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX10-NEXT: v_rcp_f32_e32 v18, v14
-; GFX10-NEXT: v_xor_b32_sdwa v3, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX10-NEXT: v_xor_b32_sdwa v11, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX10-NEXT: v_xor_b32_sdwa v13, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_3
-; GFX10-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GFX10-NEXT: v_mul_f32_e32 v15, v2, v15
-; GFX10-NEXT: v_mul_f32_e32 v16, v19, v16
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 30, v3
-; GFX10-NEXT: v_mul_f32_e32 v17, v2, v17
-; GFX10-NEXT: v_or_b32_e32 v0, 1, v0
-; GFX10-NEXT: v_trunc_f32_e32 v15, v15
-; GFX10-NEXT: v_trunc_f32_e32 v16, v16
-; GFX10-NEXT: v_mul_f32_e32 v18, v1, v18
-; GFX10-NEXT: v_trunc_f32_e32 v17, v17
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 30, v11
-; GFX10-NEXT: v_mad_f32 v20, -v15, v1, v2
-; GFX10-NEXT: v_mad_f32 v19, -v16, v10, v19
-; GFX10-NEXT: v_or_b32_e32 v3, 1, v3
-; GFX10-NEXT: v_trunc_f32_e32 v18, v18
-; GFX10-NEXT: v_mad_f32 v2, -v17, v12, v2
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v20|, |v1|
-; GFX10-NEXT: v_ashrrev_i32_e32 v13, 30, v13
-; GFX10-NEXT: v_or_b32_e32 v11, 1, v11
-; GFX10-NEXT: v_mad_f32 v21, -v18, v14, v1
-; GFX10-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v19|, |v10|
-; GFX10-NEXT: v_or_b32_e32 v13, 1, v13
-; GFX10-NEXT: v_cvt_i32_f32_e32 v16, v16
-; GFX10-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX10-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, |v12|
-; GFX10-NEXT: v_add_nc_u32_e32 v0, v15, v0
-; GFX10-NEXT: v_add_nc_u32_sdwa v1, v16, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v21|, |v14|
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_add_nc_u32_e32 v2, v17, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v13, vcc_lo
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_add_nc_u32_sdwa v3, v18, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_rcp_f32_e32 v12, v0
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_rcp_f32_e32 v11, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_mul_f32_e32 v12, v1, v12
+; GFX10-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v10
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v11
+; GFX10-NEXT: v_trunc_f32_e32 v3, v12
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x60706
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1585,64 +1560,39 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x60706
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v9, v4, s4
-; GFX9-NEXT: v_xor_b32_sdwa v1, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_xor_b32_sdwa v10, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_xor_b32_sdwa v9, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_xor_b32_sdwa v14, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_3
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_rcp_f32_e32 v15, v2
-; GFX9-NEXT: v_rcp_f32_e32 v16, v12
-; GFX9-NEXT: v_rcp_f32_e32 v17, v13
-; GFX9-NEXT: v_rcp_f32_e32 v18, v4
-; GFX9-NEXT: v_mul_f32_e32 v15, v3, v15
-; GFX9-NEXT: v_mul_f32_e32 v16, v11, v16
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 30, v1
-; GFX9-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX9-NEXT: v_mul_f32_e32 v18, v2, v18
-; GFX9-NEXT: v_trunc_f32_e32 v16, v16
-; GFX9-NEXT: v_mad_f32 v19, -v15, v2, v3
-; GFX9-NEXT: v_ashrrev_i32_e32 v10, 30, v10
-; GFX9-NEXT: v_or_b32_e32 v1, 1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_trunc_f32_e32 v18, v18
-; GFX9-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX9-NEXT: v_mad_f32 v11, -v16, v12, v11
-; GFX9-NEXT: v_cvt_i32_f32_e32 v16, v16
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, |v2|
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 30, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 1, v10
-; GFX9-NEXT: v_mad_f32 v3, -v17, v13, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX9-NEXT: v_mad_f32 v2, -v18, v4, v2
-; GFX9-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v11|, |v12|
-; GFX9-NEXT: v_ashrrev_i32_e32 v14, 30, v14
-; GFX9-NEXT: v_or_b32_e32 v9, 1, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v13|
-; GFX9-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v4|
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v14, vcc
-; GFX9-NEXT: v_add_u32_e32 v1, v15, v1
-; GFX9-NEXT: v_add_u32_sdwa v4, v16, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_add_u32_e32 v3, v17, v3
-; GFX9-NEXT: v_add_u32_sdwa v2, v18, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_rcp_f32_e32 v12, v0
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v10, v10
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_mul_f32_e32 v12, v1, v12
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v10
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v11
+; GFX9-NEXT: v_trunc_f32_e32 v3, v12
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: global_store_dword v[5:6], v1, off
-; GFX9-NEXT: global_store_dword v[7:8], v0, off
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: global_store_dword v[5:6], v0, off
+; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1869,71 +1819,47 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v12, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v15, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v17, v2
-; GFX10-NEXT: v_rcp_f32_e32 v18, v13
-; GFX10-NEXT: v_rcp_f32_e32 v19, v3
-; GFX10-NEXT: v_xor_b32_sdwa v1, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX10-NEXT: v_rcp_f32_e32 v20, v15
-; GFX10-NEXT: v_xor_b32_sdwa v11, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v21, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_xor_b32_sdwa v14, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_2
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 30, v1
-; GFX10-NEXT: v_xor_b32_sdwa v16, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_3
-; GFX10-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX10-NEXT: v_mul_f32_e32 v18, v12, v18
-; GFX10-NEXT: v_mul_f32_e32 v19, v15, v19
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 30, v11
-; GFX10-NEXT: v_or_b32_e32 v1, 1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v17, v17
-; GFX10-NEXT: v_trunc_f32_e32 v18, v18
-; GFX10-NEXT: v_mul_f32_e32 v20, v21, v20
-; GFX10-NEXT: v_trunc_f32_e32 v19, v19
-; GFX10-NEXT: v_ashrrev_i32_e32 v14, 30, v14
-; GFX10-NEXT: v_mad_f32 v22, -v17, v2, v3
-; GFX10-NEXT: v_mad_f32 v12, -v18, v13, v12
-; GFX10-NEXT: v_or_b32_e32 v11, 1, v11
-; GFX10-NEXT: v_trunc_f32_e32 v20, v20
-; GFX10-NEXT: v_mad_f32 v23, -v19, v3, v15
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v22|, |v2|
-; GFX10-NEXT: v_ashrrev_i32_e32 v16, 30, v16
-; GFX10-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX10-NEXT: v_mad_f32 v21, -v20, v15, v21
-; GFX10-NEXT: v_cvt_i32_f32_e32 v17, v17
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v12|, |v13|
-; GFX10-NEXT: v_or_b32_e32 v16, 1, v16
-; GFX10-NEXT: v_cvt_i32_f32_e32 v18, v18
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
-; GFX10-NEXT: v_cvt_i32_f32_e32 v19, v19
-; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v23|, |v3|
-; GFX10-NEXT: v_cvt_i32_f32_e32 v20, v20
-; GFX10-NEXT: v_add_nc_u32_e32 v1, v17, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v2, v18, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v14, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v21|, |v15|
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_rcp_f32_e32 v12, v2
+; GFX10-NEXT: v_rcp_f32_e32 v13, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_mul_f32_e32 v0, v2, v0
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v3, v10, v12
+; GFX10-NEXT: v_mul_f32_e32 v10, v11, v13
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 8, v4
+; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_trunc_f32_e32 v10, v10
+; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v4
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, v10
-; GFX10-NEXT: v_add_nc_u32_e32 v3, v19, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v11, 0, v16, vcc_lo
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX10-NEXT: v_add_nc_u32_e32 v11, v20, v11
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, v0, v1
-; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_mul_lo_u32 v10, v11, v12
+; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_i32_f32_e32 v10, v10
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v11
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v2
+; GFX10-NEXT: v_mul_lo_u32 v10, v10, v12
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v2, v0
+; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX10-NEXT: v_sub_nc_u32_e32 v2, v12, v3
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2070306
; GFX10-NEXT: global_store_dword v[5:6], v0, off
@@ -1953,68 +1879,44 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x2070306
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v14, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_rcp_f32_e32 v17, v3
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v16, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_xor_b32_sdwa v15, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_2
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
-; GFX9-NEXT: v_mul_f32_e32 v17, v10, v17
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_mad_f32 v19, -v17, v3, v10
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, |v3|
-; GFX9-NEXT: v_rcp_f32_e32 v3, v14
-; GFX9-NEXT: v_rcp_f32_e32 v19, v10
-; GFX9-NEXT: v_xor_b32_sdwa v2, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
-; GFX9-NEXT: v_xor_b32_sdwa v12, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
-; GFX9-NEXT: v_mul_f32_e32 v3, v13, v3
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_rcp_f32_e32 v12, v12
+; GFX9-NEXT: v_add_u32_e32 v16, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_add_u32_e32 v17, 1, v13
+; GFX9-NEXT: v_rcp_f32_e32 v13, v13
+; GFX9-NEXT: v_add_u32_e32 v11, 1, v11
+; GFX9-NEXT: v_add_u32_e32 v15, 1, v15
+; GFX9-NEXT: v_mul_f32_e32 v2, v16, v2
+; GFX9-NEXT: v_mul_f32_e32 v11, v11, v12
+; GFX9-NEXT: v_mul_f32_e32 v3, v17, v3
+; GFX9-NEXT: v_mul_f32_e32 v12, v15, v13
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v11, v11
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_mad_f32 v13, -v3, v14, v13
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v13|, |v14|
-; GFX9-NEXT: v_ashrrev_i32_e32 v14, 30, v15
-; GFX9-NEXT: v_mul_f32_e32 v15, v16, v19
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_mad_f32 v19, -v15, v10, v16
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v19|, |v10|
-; GFX9-NEXT: v_rcp_f32_e32 v10, v16
-; GFX9-NEXT: v_cvt_i32_f32_e32 v17, v17
+; GFX9-NEXT: v_trunc_f32_e32 v12, v12
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GFX9-NEXT: v_mul_f32_e32 v10, v13, v10
-; GFX9-NEXT: v_trunc_f32_e32 v10, v10
-; GFX9-NEXT: v_ashrrev_i32_e32 v12, 30, v12
-; GFX9-NEXT: v_xor_b32_sdwa v19, sext(v9), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v15, v15
-; GFX9-NEXT: v_mad_f32 v13, -v10, v16, v13
-; GFX9-NEXT: v_cvt_i32_f32_e32 v10, v10
-; GFX9-NEXT: v_or_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_or_b32_e32 v12, 1, v12
-; GFX9-NEXT: v_ashrrev_i32_e32 v19, 30, v19
-; GFX9-NEXT: v_or_b32_e32 v14, 1, v14
-; GFX9-NEXT: v_or_b32_e32 v19, 1, v19
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v13|, |v16|
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 8, v4
-; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, v14, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v13, 0, v19, vcc
-; GFX9-NEXT: v_add_u32_e32 v2, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v3, v3, v12
+; GFX9-NEXT: v_cvt_i32_f32_e32 v12, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v18, 24, v4
-; GFX9-NEXT: v_add_u32_e32 v12, v15, v14
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v13
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v11
-; GFX9-NEXT: v_mul_lo_u32 v4, v12, v0
-; GFX9-NEXT: v_mul_lo_u32 v10, v10, v18
+; GFX9-NEXT: v_mul_lo_u32 v4, v11, v10
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX9-NEXT: v_mul_lo_u32 v10, v12, v14
; GFX9-NEXT: v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_sub_u32_e32 v3, v18, v4
+; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_e32 v3, v14, v3
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -2235,53 +2137,43 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v2
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v14, v0
-; GFX10-NEXT: v_rcp_f32_e32 v11, v3
-; GFX10-NEXT: v_rcp_f32_e32 v10, v1
-; GFX10-NEXT: v_rcp_f32_e32 v12, v4
-; GFX10-NEXT: v_rcp_f32_e32 v13, v9
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v15, v0
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x40207
-; GFX10-NEXT: v_mul_f32_e32 v11, v4, v11
-; GFX10-NEXT: v_mul_f32_e32 v10, v14, v10
-; GFX10-NEXT: v_mul_f32_e32 v12, v15, v12
-; GFX10-NEXT: v_mul_f32_e32 v13, v1, v13
-; GFX10-NEXT: v_trunc_f32_e32 v11, v11
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
-; GFX10-NEXT: v_trunc_f32_e32 v12, v12
-; GFX10-NEXT: v_trunc_f32_e32 v13, v13
-; GFX10-NEXT: v_mad_f32 v16, -v11, v3, v4
-; GFX10-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX10-NEXT: v_mad_f32 v14, -v10, v1, v14
-; GFX10-NEXT: v_cvt_u32_f32_e32 v10, v10
-; GFX10-NEXT: v_mad_f32 v17, -v13, v9, v1
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v16|, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX10-NEXT: v_mad_f32 v15, -v12, v4, v15
-; GFX10-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v14|, v1
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v17|, v9
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v15|, v4
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v2, v9
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v3, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v10, v4
+; GFX10-NEXT: v_rcp_f32_e32 v11, v0
+; GFX10-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v12, v9
+; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v12, 1, v12
+; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v11
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX10-NEXT: v_mul_f32_e32 v3, v12, v13
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v10
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
-; GFX10-NEXT: v_or_b32_sdwa v3, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX10-NEXT: global_store_dword v[5:6], v1, off
-; GFX10-NEXT: global_store_dword v[7:8], v0, off
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x40207
+; GFX10-NEXT: global_store_dword v[5:6], v0, off
+; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: udiv:
@@ -2293,54 +2185,44 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: global_load_dword v4, v[2:3], off
-; GFX9-NEXT: global_load_dword v9, v[0:1], off
+; GFX9-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-NEXT: global_load_dword v9, v[2:3], off
; GFX9-NEXT: s_mov_b32 s4, 0x40207
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v11, v2
-; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
-; GFX9-NEXT: v_rcp_f32_e32 v12, v3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v1, v9
-; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v10, v4
-; GFX9-NEXT: v_rcp_f32_e32 v13, v10
-; GFX9-NEXT: v_mul_f32_e32 v11, v1, v11
-; GFX9-NEXT: v_perm_b32 v0, v9, v4, s4
-; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v11, v11
-; GFX9-NEXT: v_rcp_f32_e32 v14, v4
-; GFX9-NEXT: v_mul_f32_e32 v12, v10, v12
-; GFX9-NEXT: v_mad_f32 v1, -v11, v2, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v9, v9
-; GFX9-NEXT: v_trunc_f32_e32 v12, v12
-; GFX9-NEXT: v_mul_f32_e32 v13, v9, v13
-; GFX9-NEXT: v_mad_f32 v15, -v12, v3, v10
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX9-NEXT: v_trunc_f32_e32 v13, v13
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
-; GFX9-NEXT: v_mul_f32_e32 v14, v2, v14
-; GFX9-NEXT: v_mad_f32 v9, -v13, v10, v9
-; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v14, v14
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v15|, v3
-; GFX9-NEXT: v_mad_f32 v16, -v14, v4, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v14, v14
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v12, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v9|, v10
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v13, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v16|, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v14, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: global_store_dword v[5:6], v1, off
-; GFX9-NEXT: global_store_dword v[7:8], v0, off
+; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v0, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v9
+; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v2, v9
+; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v3, v9
+; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v11, v9
+; GFX9-NEXT: v_rcp_f32_e32 v12, v1
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v13, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v10, v4
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_add_u32_e32 v10, 1, v10
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v12
+; GFX9-NEXT: v_mul_f32_e32 v2, v13, v2
+; GFX9-NEXT: v_mul_f32_e32 v3, v10, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v11
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: global_store_dword v[5:6], v0, off
+; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2433,50 +2315,40 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v15, v0
-; GFX10-NEXT: v_rcp_f32_e32 v10, v1
-; GFX10-NEXT: v_rcp_f32_e32 v11, v3
-; GFX10-NEXT: v_rcp_f32_e32 v12, v4
-; GFX10-NEXT: v_rcp_f32_e32 v13, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v16, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v17, 24, v2
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v11, v0
+; GFX10-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-NEXT: v_rcp_f32_e32 v10, v3
+; GFX10-NEXT: v_rcp_f32_e32 v4, v4
+; GFX10-NEXT: v_rcp_f32_e32 v9, v9
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX10-NEXT: v_mul_f32_e32 v10, v3, v10
-; GFX10-NEXT: v_mul_f32_e32 v11, v3, v11
-; GFX10-NEXT: v_mul_f32_e32 v12, v3, v12
-; GFX10-NEXT: v_mul_f32_e32 v13, v15, v13
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
-; GFX10-NEXT: v_trunc_f32_e32 v11, v11
-; GFX10-NEXT: v_trunc_f32_e32 v12, v12
-; GFX10-NEXT: v_trunc_f32_e32 v13, v13
-; GFX10-NEXT: v_mad_f32 v18, -v10, v1, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v10, v10
-; GFX10-NEXT: v_mad_f32 v19, -v11, v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v11, v11
-; GFX10-NEXT: v_mad_f32 v20, -v12, v4, v3
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v18|, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v12, v12
-; GFX10-NEXT: v_mad_f32 v15, -v13, v9, v15
-; GFX10-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v19|, v3
+; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_mul_f32_e32 v4, v11, v9
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v9, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v2
+; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_trunc_f32_e32 v4, v4
+; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_u32_f32_e32 v9, v9
+; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v4
; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v20|, v4
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v16
-; GFX10-NEXT: v_sub_nc_u32_e32 v1, v16, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v15|, v9
-; GFX10-NEXT: v_mul_lo_u32 v4, v4, v14
-; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v16, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_mul_lo_u32 v9, v9, v17
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, v16, v4
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_sub_nc_u32_sdwa v9, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mul_lo_u32 v9, v9, v10
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v12
+; GFX10-NEXT: v_mul_lo_u32 v4, v4, v11
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, v10, v1
+; GFX10-NEXT: v_sub_nc_u32_sdwa v9, v10, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, v10, v3
+; GFX10-NEXT: v_sub_nc_u32_sdwa v4, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x2050505
-; GFX10-NEXT: v_or_b32_sdwa v3, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX10-NEXT: global_store_dword v[5:6], v1, off
; GFX10-NEXT: global_store_dword v[7:8], v0, off
@@ -2496,53 +2368,43 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: s_mov_b32 s4, 0x2050505
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v4
-; GFX9-NEXT: v_rcp_f32_e32 v15, v2
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
-; GFX9-NEXT: v_rcp_f32_e32 v16, v3
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v11, v4
-; GFX9-NEXT: v_rcp_f32_e32 v17, v11
-; GFX9-NEXT: v_mul_f32_e32 v15, v3, v15
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v14, v4
-; GFX9-NEXT: v_trunc_f32_e32 v15, v15
-; GFX9-NEXT: v_rcp_f32_e32 v18, v14
-; GFX9-NEXT: v_mul_f32_e32 v16, v3, v16
-; GFX9-NEXT: v_mad_f32 v19, -v15, v2, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v15, v15
-; GFX9-NEXT: v_trunc_f32_e32 v16, v16
-; GFX9-NEXT: v_mul_f32_e32 v17, v3, v17
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, v2
-; GFX9-NEXT: v_mad_f32 v2, -v16, v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v16, v16
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v15, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: v_rcp_f32_e32 v11, v11
+; GFX9-NEXT: v_rcp_f32_e32 v14, v14
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v13, v9
-; GFX9-NEXT: v_trunc_f32_e32 v17, v17
-; GFX9-NEXT: v_mul_f32_e32 v18, v13, v18
-; GFX9-NEXT: v_mad_f32 v19, -v17, v11, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v17, v17
-; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT: v_trunc_f32_e32 v18, v18
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT: v_mad_f32 v13, -v18, v14, v13
-; GFX9-NEXT: v_cvt_u32_f32_e32 v18, v18
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v16, vcc
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v19|, v11
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v17, vcc
+; GFX9-NEXT: v_add_u32_e32 v13, 1, v13
+; GFX9-NEXT: v_mul_f32_e32 v2, v15, v2
+; GFX9-NEXT: v_mul_f32_e32 v3, v15, v3
+; GFX9-NEXT: v_mul_f32_e32 v11, v15, v11
+; GFX9-NEXT: v_mul_f32_e32 v13, v13, v14
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v11, v11
+; GFX9-NEXT: v_trunc_f32_e32 v13, v13
+; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
+; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v13|, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, 0, v18, vcc
-; GFX9-NEXT: v_mul_lo_u32 v4, v15, v4
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, v10
-; GFX9-NEXT: v_mul_lo_u32 v0, v3, v0
-; GFX9-NEXT: v_mul_lo_u32 v3, v11, v12
-; GFX9-NEXT: v_sub_u32_e32 v4, v10, v4
-; GFX9-NEXT: v_sub_u32_sdwa v2, v10, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT: v_mul_lo_u32 v0, v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v4, v13, v12
+; GFX9-NEXT: v_sub_u32_e32 v2, v10, v2
+; GFX9-NEXT: v_sub_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v0, v10, v0
-; GFX9-NEXT: v_sub_u32_sdwa v3, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-NEXT: global_store_dword v[5:6], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index c1de1bd96bd5b..1b826496efd66 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -201,25 +201,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_ashr_i32 s5, s5, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GCN-NEXT: s_ashr_i32 s4, s4, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GCN-NEXT: s_xor_b32 s4, s4, s5
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_ashr_i32 s4, s4, 30
-; GCN-NEXT: s_or_b32 s6, s4, 1
+; GCN-NEXT: s_ashr_i32 s3, s3, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GCN-NEXT: s_ashr_i32 s2, s2, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2
; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, s6, 0
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
@@ -227,25 +219,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; TONGA: ; %bb.0:
; TONGA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: s_mov_b64 s[4:5], s[2:3]
-; TONGA-NEXT: s_ashr_i32 s5, s5, 10
-; TONGA-NEXT: v_cvt_f32_i32_e32 v0, s5
-; TONGA-NEXT: s_ashr_i32 s4, s4, 10
-; TONGA-NEXT: v_cvt_f32_i32_e32 v1, s4
-; TONGA-NEXT: s_xor_b32 s4, s4, s5
-; TONGA-NEXT: v_rcp_f32_e32 v2, v0
-; TONGA-NEXT: s_ashr_i32 s4, s4, 30
-; TONGA-NEXT: s_or_b32 s6, s4, 1
+; TONGA-NEXT: s_ashr_i32 s3, s3, 10
+; TONGA-NEXT: v_cvt_f32_i32_e32 v0, s3
+; TONGA-NEXT: s_ashr_i32 s2, s2, 10
+; TONGA-NEXT: v_cvt_f32_i32_e32 v1, s2
; TONGA-NEXT: s_mov_b32 s3, 0xf000
-; TONGA-NEXT: v_mul_f32_e32 v2, v1, v2
-; TONGA-NEXT: v_trunc_f32_e32 v2, v2
-; TONGA-NEXT: v_mad_f32 v1, -v2, v0, v1
-; TONGA-NEXT: v_cvt_i32_f32_e32 v2, v2
-; TONGA-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; TONGA-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; TONGA-NEXT: s_cselect_b32 s4, s6, 0
+; TONGA-NEXT: v_rcp_f32_e32 v0, v0
; TONGA-NEXT: s_mov_b32 s2, -1
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, s4, v2
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; TONGA-NEXT: v_mul_f32_e32 v0, v1, v0
+; TONGA-NEXT: v_trunc_f32_e32 v0, v0
+; TONGA-NEXT: v_cvt_i32_f32_e32 v0, v0
; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0
; TONGA-NEXT: s_endpgm
;
@@ -253,25 +237,17 @@ define amdgpu_kernel void @s_test_sdiv22_32(ptr addrspace(1) %out, i32 %x, i32 %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_ashr_i32 s5, s5, 10
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX9-NEXT: s_ashr_i32 s4, s4, 10
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX9-NEXT: s_xor_b32 s4, s4, s5
-; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s6, s4, 1
+; GFX9-NEXT: s_ashr_i32 s3, s3, 10
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_ashr_i32 s2, s2, 10
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s6, 0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_add_u32_e32 v0, s4, v2
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -1579,20 +1555,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GCN-NEXT: s_mov_b32 s4, s0
; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_i32_e32 v3, v1
-; GCN-NEXT: v_xor_b32_e32 v0, v1, v0
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GCN-NEXT: v_rcp_f32_e32 v4, v2
-; GCN-NEXT: v_or_b32_e32 v0, 1, v0
-; GCN-NEXT: v_mul_f32_e32 v1, v3, v4
-; GCN-NEXT: v_trunc_f32_e32 v1, v1
-; GCN-NEXT: v_mad_f32 v3, -v1, v2, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 8
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
@@ -1612,20 +1582,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; TONGA-NEXT: s_mov_b32 s4, s0
; TONGA-NEXT: s_mov_b32 s5, s1
; TONGA-NEXT: s_waitcnt vmcnt(1)
-; TONGA-NEXT: v_cvt_f32_i32_e32 v2, v0
+; TONGA-NEXT: v_cvt_f32_i32_e32 v0, v0
; TONGA-NEXT: s_waitcnt vmcnt(0)
-; TONGA-NEXT: v_cvt_f32_i32_e32 v3, v1
-; TONGA-NEXT: v_xor_b32_e32 v0, v1, v0
-; TONGA-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; TONGA-NEXT: v_rcp_f32_e32 v4, v2
-; TONGA-NEXT: v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT: v_mul_f32_e32 v1, v3, v4
-; TONGA-NEXT: v_trunc_f32_e32 v1, v1
-; TONGA-NEXT: v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT: v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT: v_cvt_f32_i32_e32 v1, v1
+; TONGA-NEXT: v_rcp_f32_e32 v0, v0
+; TONGA-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; TONGA-NEXT: v_mul_f32_e32 v0, v1, v0
+; TONGA-NEXT: v_trunc_f32_e32 v0, v0
+; TONGA-NEXT: v_cvt_i32_f32_e32 v0, v0
; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 8
; TONGA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; TONGA-NEXT: s_endpgm
@@ -1645,20 +1609,14 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GFX9-NEXT: s_mov_b32 s4, s0
; GFX9-NEXT: s_mov_b32 s5, s1
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v1
-; GFX9-NEXT: v_xor_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v2, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2|
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
@@ -1711,20 +1669,20 @@ define amdgpu_kernel void @v_sdiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GCN-LABEL: v_sdiv_i23:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_mov_b32 s10, s2
-; GCN-NEXT: s_mov_b32 s11, s3
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s10, s6
+; GCN-NEXT: s_mov_b32 s11, s7
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s8, s6
-; GCN-NEXT: s_mov_b32 s9, s7
+; GCN-NEXT: s_mov_b32 s8, s2
+; GCN-NEXT: s_mov_b32 s9, s3
; GCN-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:2
; GCN-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:6
; GCN-NEXT: buffer_load_ushort v2, off, s[8:11], 0 offset:4
; GCN-NEXT: buffer_load_ushort v3, off, s[8:11], 0
-; GCN-NEXT: s_mov_b32 s0, s4
-; GCN-NEXT: s_mov_b32 s1, s5
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: s_waitcnt vmcnt(2)
@@ -1732,42 +1690,36 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: s_waitcnt vmcnt(1)
; GCN-NEXT: v_or_b32_e32 v1, v2, v1
; GCN-NEXT: v_bfe_i32 v1, v1, 0, 23
-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v1
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_or_b32_e32 v0, v3, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 23
-; GCN-NEXT: v_cvt_f32_i32_e32 v3, v0
-; GCN-NEXT: v_rcp_f32_e32 v4, v2
-; GCN-NEXT: v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GCN-NEXT: v_or_b32_e32 v0, 1, v0
-; GCN-NEXT: v_mul_f32_e32 v1, v3, v4
-; GCN-NEXT: v_trunc_f32_e32 v1, v1
-; GCN-NEXT: v_mad_f32 v3, -v1, v2, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GCN-NEXT: v_rcp_f32_e32 v1, v1
+; GCN-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 23
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; TONGA-LABEL: v_sdiv_i23:
; TONGA: ; %bb.0:
-; TONGA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; TONGA-NEXT: s_mov_b32 s3, 0xf000
-; TONGA-NEXT: s_mov_b32 s2, -1
-; TONGA-NEXT: s_mov_b32 s10, s2
-; TONGA-NEXT: s_mov_b32 s11, s3
+; TONGA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; TONGA-NEXT: s_mov_b32 s7, 0xf000
+; TONGA-NEXT: s_mov_b32 s6, -1
+; TONGA-NEXT: s_mov_b32 s10, s6
+; TONGA-NEXT: s_mov_b32 s11, s7
; TONGA-NEXT: s_waitcnt lgkmcnt(0)
-; TONGA-NEXT: s_mov_b32 s8, s6
-; TONGA-NEXT: s_mov_b32 s9, s7
+; TONGA-NEXT: s_mov_b32 s8, s2
+; TONGA-NEXT: s_mov_b32 s9, s3
; TONGA-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:2
; TONGA-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:6
; TONGA-NEXT: buffer_load_ushort v2, off, s[8:11], 0 offset:4
; TONGA-NEXT: buffer_load_ushort v3, off, s[8:11], 0
-; TONGA-NEXT: s_mov_b32 s0, s4
-; TONGA-NEXT: s_mov_b32 s1, s5
+; TONGA-NEXT: s_mov_b32 s4, s0
+; TONGA-NEXT: s_mov_b32 s5, s1
; TONGA-NEXT: s_waitcnt vmcnt(3)
; TONGA-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; TONGA-NEXT: s_waitcnt vmcnt(2)
@@ -1775,42 +1727,36 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
; TONGA-NEXT: s_waitcnt vmcnt(1)
; TONGA-NEXT: v_or_b32_e32 v1, v2, v1
; TONGA-NEXT: v_bfe_i32 v1, v1, 0, 23
-; TONGA-NEXT: v_cvt_f32_i32_e32 v2, v1
+; TONGA-NEXT: v_cvt_f32_i32_e32 v1, v1
; TONGA-NEXT: s_waitcnt vmcnt(0)
; TONGA-NEXT: v_or_b32_e32 v0, v3, v0
; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 23
-; TONGA-NEXT: v_cvt_f32_i32_e32 v3, v0
-; TONGA-NEXT: v_rcp_f32_e32 v4, v2
-; TONGA-NEXT: v_xor_b32_e32 v0, v0, v1
-; TONGA-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; TONGA-NEXT: v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT: v_mul_f32_e32 v1, v3, v4
-; TONGA-NEXT: v_trunc_f32_e32 v1, v1
-; TONGA-NEXT: v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT: v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT: v_cvt_f32_i32_e32 v0, v0
+; TONGA-NEXT: v_rcp_f32_e32 v1, v1
+; TONGA-NEXT: v_add_u32_e32 v0, vcc, 1, v0
+; TONGA-NEXT: v_mul_f32_e32 v0, v0, v1
+; TONGA-NEXT: v_trunc_f32_e32 v0, v0
+; TONGA-NEXT: v_cvt_i32_f32_e32 v0, v0
; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 23
-; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; TONGA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; TONGA-NEXT: s_endpgm
;
; GFX9-LABEL: v_sdiv_i23:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: s_mov_b32 s6, s2
-; GFX9-NEXT: s_mov_b32 s7, s3
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s10, s6
+; GFX9-NEXT: s_mov_b32 s11, s7
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s10
-; GFX9-NEXT: s_mov_b32 s5, s11
-; GFX9-NEXT: buffer_load_ubyte v0, off, s[4:7], 0 offset:2
-; GFX9-NEXT: buffer_load_ubyte v1, off, s[4:7], 0 offset:6
-; GFX9-NEXT: buffer_load_ushort v2, off, s[4:7], 0 offset:4
-; GFX9-NEXT: buffer_load_ushort v3, off, s[4:7], 0
-; GFX9-NEXT: s_mov_b32 s0, s8
-; GFX9-NEXT: s_mov_b32 s1, s9
+; GFX9-NEXT: s_mov_b32 s8, s2
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:2
+; GFX9-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:6
+; GFX9-NEXT: buffer_load_ushort v2, off, s[8:11], 0 offset:4
+; GFX9-NEXT: buffer_load_ushort v3, off, s[8:11], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -1818,24 +1764,18 @@ define amdgpu_kernel void @v_sdiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 23
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 23
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v0
-; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; GFX9-NEXT: v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v4
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v1
-; GFX9-NEXT: v_mad_f32 v1, -v1, v2, v3
-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2|
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 23
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v_sdiv_i23:
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 9554a03942579..bfd5d557190a8 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -484,62 +484,42 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
define amdgpu_kernel void @s_test_sdiv22_64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-LABEL: s_test_sdiv22_64:
; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_ashr_i32 s0, s2, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: s_ashr_i32 s1, s3, 10
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_xor_b32 s0, s1, s0
-; GCN-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-NEXT: s_or_b32 s2, s0, 1
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT: s_ashr_i32 s2, s6, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT: s_ashr_i32 s2, s3, 10
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GCN-IR-LABEL: s_test_sdiv22_64:
; GCN-IR: ; %bb.0:
+; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-IR-NEXT: s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT: s_mov_b32 s6, -1
-; GCN-IR-NEXT: s_mov_b32 s4, s0
-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_ashr_i32 s0, s2, 10
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-IR-NEXT: s_mov_b32 s5, s1
-; GCN-IR-NEXT: s_ashr_i32 s1, s3, 10
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-IR-NEXT: s_xor_b32 s0, s1, s0
-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-IR-NEXT: s_or_b32 s2, s0, 1
-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT: s_ashr_i32 s2, s6, 10
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-IR-NEXT: s_ashr_i32 s2, s3, 10
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-IR-NEXT: s_mov_b32 s2, -1
+; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-IR-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-IR-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 42
%2 = ashr i64 %y, 42
@@ -1059,62 +1039,42 @@ define amdgpu_kernel void @s_test_sdiv31_64(ptr addrspace(1) %out, i64 %x, i64 %
define amdgpu_kernel void @s_test_sdiv23_64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-LABEL: s_test_sdiv23_64:
; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_ashr_i32 s0, s2, 9
-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: s_ashr_i32 s1, s3, 9
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_xor_b32 s0, s1, s0
-; GCN-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-NEXT: s_or_b32 s2, s0, 1
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT: s_ashr_i32 s2, s6, 9
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT: s_ashr_i32 s2, s3, 9
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GCN-IR-LABEL: s_test_sdiv23_64:
; GCN-IR: ; %bb.0:
+; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe
-; GCN-IR-NEXT: s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT: s_mov_b32 s6, -1
-; GCN-IR-NEXT: s_mov_b32 s4, s0
-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_ashr_i32 s0, s2, 9
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GCN-IR-NEXT: s_mov_b32 s5, s1
-; GCN-IR-NEXT: s_ashr_i32 s1, s3, 9
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-IR-NEXT: s_xor_b32 s0, s1, s0
-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30
-; GCN-IR-NEXT: s_or_b32 s2, s0, 1
-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT: s_ashr_i32 s2, s6, 9
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GCN-IR-NEXT: s_ashr_i32 s2, s3, 9
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2
+; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-IR-NEXT: s_mov_b32 s2, -1
+; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-IR-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-IR-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 41
%2 = ashr i64 %y, 41
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index 082620a68e1b5..84a898215892f 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -473,62 +473,44 @@ define amdgpu_kernel void @s_test_srem23_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_ashr_i32 s4, s6, 9
; GCN-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GCN-NEXT: s_ashr_i32 s8, s3, 9
-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s8
-; GCN-NEXT: s_xor_b32 s2, s8, s4
-; GCN-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-NEXT: s_ashr_i32 s2, s2, 30
-; GCN-NEXT: s_or_b32 s5, s2, 1
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GCN-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GCN-NEXT: s_cselect_b32 s2, s5, 0
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v2
+; GCN-NEXT: s_ashr_i32 s5, s3, 9
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: v_mul_lo_u32 v0, v0, s4
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GCN-NEXT: v_sub_i32_e32 v0, vcc, s5, v0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GCN-IR-LABEL: s_test_srem23_64:
; GCN-IR: ; %bb.0:
; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IR-NEXT: s_mov_b32 s7, 0xf000
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT: s_mov_b32 s2, -1
; GCN-IR-NEXT: s_ashr_i32 s4, s6, 9
; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GCN-IR-NEXT: s_ashr_i32 s8, s3, 9
-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s8
-; GCN-IR-NEXT: s_xor_b32 s2, s8, s4
-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0
-; GCN-IR-NEXT: s_ashr_i32 s2, s2, 30
-; GCN-IR-NEXT: s_or_b32 s5, s2, 1
-; GCN-IR-NEXT: s_mov_b32 s6, -1
-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2
-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GCN-IR-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GCN-IR-NEXT: s_cselect_b32 s2, s5, 0
-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s2, v2
+; GCN-IR-NEXT: s_ashr_i32 s5, s3, 9
+; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s5
+; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GCN-IR-NEXT: v_mul_f32_e32 v0, v1, v0
+; GCN-IR-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-IR-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4
-; GCN-IR-NEXT: s_mov_b32 s4, s0
-; GCN-IR-NEXT: s_mov_b32 s5, s1
-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s5, v0
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 41
%2 = ashr i64 %y, 41
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 8dcdf95c0ead7..fb7eb6d1e0d0f 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -1404,15 +1404,13 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -1433,15 +1431,13 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: v_and_b32_e32 v0, 0xff, v0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -1455,19 +1451,17 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NEXT: flat_load_ushort v2, v[0:1]
+; GCN-NEXT: flat_load_ushort v0, v[0:1]
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT: v_rcp_f32_e32 v1, v1
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT: v_add_u32_e32 v0, vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GCN-NEXT: v_rcp_f32_e32 v4, v3
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
-; GCN-NEXT: v_mul_f32_e32 v4, v2, v4
-; GCN-NEXT: v_trunc_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GCN-NEXT: v_mad_f32 v2, -v4, v3, v2
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
; GCN-NEXT: v_and_b32_e32 v2, 0xff, v2
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -1481,13 +1475,11 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: v_cvt_f32_ubyte1_e32 v2, v1
; GFX1030-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; GFX1030-NEXT: v_rcp_f32_e32 v3, v2
-; GFX1030-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3
-; GFX1030-NEXT: v_fma_f32 v1, -v3, v2, v1
-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v1|, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX1030-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v1, v1
+; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX1030-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT: s_endpgm
@@ -1549,13 +1541,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v0
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_rcp_f32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -1578,13 +1568,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; VI-NEXT: v_rcp_f32_e32 v2, v0
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -1599,18 +1587,16 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3
; GCN-NEXT: flat_load_dword v0, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GCN-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GCN-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GCN-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GCN-NEXT: v_rcp_f32_e32 v1, v1
+; GCN-NEXT: v_add_u32_e32 v0, vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_trunc_f32_e32 v0, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_rcp_f32_e32 v4, v2
-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4
-; GCN-NEXT: v_trunc_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -1624,13 +1610,11 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX1030-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX1030-NEXT: v_rcp_f32_e32 v3, v2
-; GFX1030-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3
-; GFX1030-NEXT: v_fma_f32 v1, -v3, v2, v1
-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v1|, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX1030-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v1, v1
+; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX1030-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1030-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT: s_endpgm
@@ -2458,20 +2442,14 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_i32_e32 v2, v0
+; SI-NEXT: v_cvt_f32_i32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; SI-NEXT: v_xor_b32_e32 v0, v1, v0
-; SI-NEXT: v_rcp_f32_e32 v4, v2
-; SI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; SI-NEXT: v_or_b32_e32 v0, 1, v0
-; SI-NEXT: v_mul_f32_e32 v1, v3, v4
-; SI-NEXT: v_trunc_f32_e32 v1, v1
-; SI-NEXT: v_mad_f32 v3, -v1, v2, v3
-; SI-NEXT: v_cvt_i32_f32_e32 v1, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; SI-NEXT: v_cvt_f32_i32_e32 v1, v1
+; SI-NEXT: v_rcp_f32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_i32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -2487,20 +2465,14 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; VI-NEXT: s_mov_b32 s7, s3
; VI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_i32_e32 v2, v0
+; VI-NEXT: v_cvt_f32_i32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; VI-NEXT: v_xor_b32_e32 v0, v1, v0
-; VI-NEXT: v_rcp_f32_e32 v4, v2
-; VI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
-; VI-NEXT: v_or_b32_e32 v0, 1, v0
-; VI-NEXT: v_mul_f32_e32 v1, v3, v4
-; VI-NEXT: v_trunc_f32_e32 v1, v1
-; VI-NEXT: v_mad_f32 v3, -v1, v2, v3
-; VI-NEXT: v_cvt_i32_f32_e32 v1, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; VI-NEXT: v_cvt_f32_i32_e32 v1, v1
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_i32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -2518,47 +2490,35 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: flat_load_sbyte v3, v[0:1]
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_cvt_f32_i32_e32 v4, v2
+; GCN-NEXT: v_cvt_f32_i32_e32 v2, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cvt_f32_i32_e32 v5, v3
-; GCN-NEXT: v_rcp_f32_e32 v6, v4
-; GCN-NEXT: v_xor_b32_e32 v2, v3, v2
-; GCN-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GCN-NEXT: v_or_b32_e32 v2, 1, v2
-; GCN-NEXT: v_mul_f32_e32 v3, v5, v6
-; GCN-NEXT: v_trunc_f32_e32 v3, v3
-; GCN-NEXT: v_mad_f32 v5, -v3, v4, v5
-; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, |v4|
-; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v3
+; GCN-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GCN-NEXT: v_rcp_f32_e32 v2, v2
+; GCN-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; GCN-NEXT: v_mul_f32_e32 v2, v3, v2
+; GCN-NEXT: v_trunc_f32_e32 v2, v2
+; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
;
; GFX1030-LABEL: fdiv_test_denormals:
; GFX1030: ; %bb.0: ; %bb
; GFX1030-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX1030-NEXT: v_mov_b32_e32 v2, 0
; GFX1030-NEXT: v_mov_b32_e32 v0, 0
; GFX1030-NEXT: v_mov_b32_e32 v1, 0
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: global_load_sbyte v2, v0, s[0:1]
-; GFX1030-NEXT: v_mov_b32_e32 v0, 0
+; GFX1030-NEXT: global_load_sbyte v2, v2, s[0:1]
; GFX1030-NEXT: global_load_sbyte v3, v[0:1], off
; GFX1030-NEXT: s_waitcnt vmcnt(1)
-; GFX1030-NEXT: v_cvt_f32_i32_e32 v4, v2
-; GFX1030-NEXT: v_rcp_f32_e32 v5, v4
+; GFX1030-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX1030-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-NEXT: v_cvt_f32_i32_e32 v6, v3
-; GFX1030-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX1030-NEXT: v_ashrrev_i32_e32 v2, 30, v2
-; GFX1030-NEXT: v_mul_f32_e32 v5, v6, v5
-; GFX1030-NEXT: v_or_b32_e32 v2, 1, v2
-; GFX1030-NEXT: v_trunc_f32_e32 v3, v5
-; GFX1030-NEXT: v_fma_f32 v5, -v3, v4, v6
-; GFX1030-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v5|, |v4|
-; GFX1030-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc_lo
-; GFX1030-NEXT: v_add_nc_u32_e32 v2, v3, v2
+; GFX1030-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX1030-NEXT: v_rcp_f32_e32 v2, v2
+; GFX1030-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX1030-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX1030-NEXT: v_trunc_f32_e32 v2, v2
+; GFX1030-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX1030-NEXT: global_store_byte v[0:1], v2, off
; GFX1030-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
index a465da4101a47..e8e1a88a46174 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem24.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
@@ -20,15 +20,13 @@ define amdgpu_kernel void @udiv24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -48,15 +46,13 @@ define amdgpu_kernel void @udiv24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -123,15 +119,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in_out(ptr addrspace(1) %out,
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -151,15 +145,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in_out(ptr addrspace(1) %out,
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -226,15 +218,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in(ptr addrspace(1) %out, ptr
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -254,15 +244,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_in(ptr addrspace(1) %out, ptr
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -329,15 +317,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_out(ptr addrspace(1) %out, ptr
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: v_rcp_f32_e32 v2, v0
+; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -357,15 +343,13 @@ define amdgpu_kernel void @udiv24_i8_denorm_flush_out(ptr addrspace(1) %out, ptr
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: v_rcp_f32_e32 v2, v0
+; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -434,13 +418,11 @@ define amdgpu_kernel void @udiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; SI-NEXT: v_rcp_f32_e32 v2, v0
-; SI-NEXT: v_mul_f32_e32 v2, v1, v2
-; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; SI-NEXT: v_rcp_f32_e32 v0, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_trunc_f32_e32 v0, v0
+; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -462,13 +444,11 @@ define amdgpu_kernel void @udiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v1, v1
-; VI-NEXT: v_rcp_f32_e32 v2, v0
-; VI-NEXT: v_mul_f32_e32 v2, v1, v2
-; VI-NEXT: v_trunc_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2
-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; VI-NEXT: v_rcp_f32_e32 v0, v0
+; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; VI-NEXT: v_mul_f32_e32 v0, v1, v0
+; VI-NEXT: v_trunc_f32_e32 v0, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
@@ -1349,15 +1329,13 @@ define amdgpu_kernel void @urem24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
-; SI-NEXT: v_rcp_f32_e32 v3, v2
+; SI-NEXT: v_rcp_f32_e32 v2, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cvt_f32_ubyte0_e32 v4, v1
-; SI-NEXT: v_mul_f32_e32 v3, v4, v3
-; SI-NEXT: v_trunc_f32_e32 v3, v3
-; SI-NEXT: v_cvt_u32_f32_e32 v5, v3
-; SI-NEXT: v_mad_f32 v3, -v3, v2, v4
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; SI-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; SI-NEXT: v_mul_f32_e32 v2, v3, v2
+; SI-NEXT: v_trunc_f32_e32 v2, v2
+; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
; SI-NEXT: v_mul_lo_u32 v0, v2, v0
; SI-NEXT: v_subrev_i32_e32 v0, vcc, v0, v1
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
@@ -1379,15 +1357,13 @@ define amdgpu_kernel void @urem24_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
-; VI-NEXT: v_rcp_f32_e32 v3, v2
+; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v4, v1
-; VI-NEXT: v_mul_f32_e32 v3, v4, v3
-; VI-NEXT: v_trunc_f32_e32 v3, v3
-; VI-NEXT: v_cvt_u32_f32_e32 v5, v3
-; VI-NEXT: v_mad_f32 v3, -v3, v2, v4
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; VI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
+; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; VI-NEXT: v_mul_f32_e32 v2, v3, v2
+; VI-NEXT: v_trunc_f32_e32 v2, v2
+; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
; VI-NEXT: v_mul_lo_u32 v0, v2, v0
; VI-NEXT: v_subrev_u32_e32 v0, vcc, v0, v1
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
@@ -1460,13 +1436,11 @@ define amdgpu_kernel void @urem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_cvt_f32_u32_e32 v2, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_u32_e32 v3, v1
-; SI-NEXT: v_rcp_f32_e32 v4, v2
-; SI-NEXT: v_mul_f32_e32 v4, v3, v4
-; SI-NEXT: v_trunc_f32_e32 v4, v4
-; SI-NEXT: v_cvt_u32_f32_e32 v5, v4
-; SI-NEXT: v_mad_f32 v3, -v4, v2, v3
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; SI-NEXT: v_rcp_f32_e32 v2, v2
+; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; SI-NEXT: v_mul_f32_e32 v2, v3, v2
+; SI-NEXT: v_trunc_f32_e32 v2, v2
+; SI-NEXT: v_cvt_u32_f32_e32 v2, v2
; SI-NEXT: v_mul_lo_u32 v0, v2, v0
; SI-NEXT: v_subrev_i32_e32 v0, vcc, v0, v1
; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
@@ -1490,13 +1464,11 @@ define amdgpu_kernel void @urem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_u32_e32 v2, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_u32_e32 v3, v1
-; VI-NEXT: v_rcp_f32_e32 v4, v2
-; VI-NEXT: v_mul_f32_e32 v4, v3, v4
-; VI-NEXT: v_trunc_f32_e32 v4, v4
-; VI-NEXT: v_cvt_u32_f32_e32 v5, v4
-; VI-NEXT: v_mad_f32 v3, -v4, v2, v3
-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; VI-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; VI-NEXT: v_rcp_f32_e32 v2, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v3
+; VI-NEXT: v_mul_f32_e32 v2, v3, v2
+; VI-NEXT: v_trunc_f32_e32 v2, v2
+; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
; VI-NEXT: v_mul_lo_u32 v0, v2, v0
; VI-NEXT: v_subrev_u32_e32 v0, vcc, v0, v1
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
More information about the llvm-commits
mailing list