[llvm] [AMDGPU] Fix legalization of frem(-0.0, y) (PR #70448)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 27 05:53:08 PDT 2023
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/70448
>From 8c60b73fbe2e16a769f0bacc8e8dc19cfd56ac57 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 27 Oct 2023 13:09:47 +0100
Subject: [PATCH 1/2] [AMDGPU] Fix legalization of frem(-0.0, y)
We legalized frem(x, y) -> x - y * trunc(x / y). When x is -0.0 this
evaluates to +0.0 but the result should be -0.0.
Fix this by legalizing to copysign(x - y * trunc(x / y), x).
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 8 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 14 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll | 185 +++-
llvm/test/CodeGen/AMDGPU/frem.ll | 880 +++++++++++-------
llvm/test/CodeGen/AMDGPU/wave32.ll | 12 +-
5 files changed, 735 insertions(+), 364 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index adf4e0139e03c1d..28f6ad62f2c37ed 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -2267,19 +2267,23 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
return DAG.getMergeValues(Res, DL);
}
-// (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x)
+// (frem x, y) -> (fcopysign (fma (fneg (ftrunc (fdiv x, y))), y, x), x)
+// The fcopysign is only required to get the correct result -0.0 when x is -0.0
+// (and y is non-zero). With NSZ it can be dropped.
SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
EVT VT = Op.getValueType();
auto Flags = Op->getFlags();
SDValue X = Op.getOperand(0);
SDValue Y = Op.getOperand(1);
+ bool NSZ = mayIgnoreSignedZero(Op);
SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags);
SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags);
SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
// TODO: For f32 use FMAD instead if !hasFastFMA32?
- return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
+ SDValue FMA = DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
+ return NSZ ? FMA : DAG.getNode(ISD::FCOPYSIGN , SL, VT, FMA, X);
}
SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 3d70ed150df12f8..0775d8a3c2396f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2341,6 +2341,14 @@ bool AMDGPULegalizerInfo::legalizeFceil(
return true;
}
+static bool allowNoSignedZeros(const MachineFunction &MF, unsigned Flags) {
+ return (Flags & MachineInstr::FmNsz) ||
+ MF.getTarget().Options.NoSignedZerosFPMath;
+}
+
+// Legalize frem(x, y) -> copysign(x - y * trunc(x / y), x)
+// The copysign is only required to get the correct result -0.0 when x is -0.0
+// (and y is non-zero). With NSZ it can be dropped.
bool AMDGPULegalizerInfo::legalizeFrem(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
@@ -2349,11 +2357,15 @@ bool AMDGPULegalizerInfo::legalizeFrem(
Register Src1Reg = MI.getOperand(2).getReg();
auto Flags = MI.getFlags();
LLT Ty = MRI.getType(DstReg);
+ bool NSZ = allowNoSignedZeros(B.getMF(), Flags);
auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
auto Neg = B.buildFNeg(Ty, Trunc, Flags);
- B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
+ DstOp FMADst = NSZ ? DstOp(DstReg) : DstOp(Ty);
+ auto FMA = B.buildFMA(FMADst, Neg, Src1Reg, Src0Reg, Flags);
+ if (!NSZ)
+ B.buildFCopysign(DstReg, FMA, Src0Reg);
MI.eraseFromParent();
return true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
index 0417b97a53c0ff3..da7fca95d5eef07 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
@@ -29,7 +29,10 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
@@ -46,12 +49,15 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
; VI-NEXT: v_cvt_f32_f16_e32 v2, s0
; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: v_mul_f32_e32 v0, v0, v2
; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
; VI-NEXT: v_div_fixup_f16 v0, v0, v1, s2
; VI-NEXT: v_trunc_f16_e32 v0, v0
-; VI-NEXT: v_fma_f16 v2, -v0, v1, s2
+; VI-NEXT: v_fma_f16 v0, -v0, v1, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_short v[0:1], v2
@@ -126,7 +132,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_rcp_f32_e32 v2, v1
; CI-NEXT: v_mul_f32_e32 v2, v0, v2
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
@@ -143,7 +152,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_rcp_f16_e32 v0, s0
; VI-NEXT: v_mul_f16_e32 v0, s2, v0
; VI-NEXT: v_trunc_f16_e32 v0, v0
-; VI-NEXT: v_fma_f16 v2, -v0, s0, v1
+; VI-NEXT: v_fma_f16 v0, -v0, s0, v1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_short v[0:1], v2
@@ -178,11 +190,14 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; CI-NEXT: v_trunc_f32_e32 v1, v1
; CI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -207,9 +222,12 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; VI-NEXT: v_trunc_f32_e32 v1, v1
-; VI-NEXT: v_fma_f32 v2, -v1, v0, s2
+; VI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -282,6 +300,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_mul_f32_e32 v0, s2, v0
; CI-NEXT: v_trunc_f32_e32 v0, v0
; CI-NEXT: v_fma_f32 v0, -v0, s0, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -297,7 +318,10 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_rcp_f32_e32 v0, s0
; VI-NEXT: v_mul_f32_e32 v0, s2, v0
; VI-NEXT: v_trunc_f32_e32 v0, v0
-; VI-NEXT: v_fma_f32 v2, -v0, s0, v1
+; VI-NEXT: v_fma_f32 v0, -v0, s0, v1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -325,6 +349,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3]
; CI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3]
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_brev_b32 s1, 1
+; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; CI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -336,6 +363,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[0:1], s[2:3]
; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
; CI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3]
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
+; CI-NEXT: v_or_b32_e32 v1, s1, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -351,6 +381,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3]
; VI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3]
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_brev_b32 s1, 1
+; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; VI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; VI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; VI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -364,6 +397,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3]
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_or_b32_e32 v1, s1, v1
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
%r0 = load double, ptr addrspace(1) %in1, align 8
@@ -453,6 +489,12 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
; CI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; CI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3]
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_brev_b32 s1, 1
+; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
+; CI-NEXT: v_or_b32_e32 v1, s1, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -476,8 +518,14 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
; VI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; VI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3]
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_brev_b32 s1, 1
+; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_or_b32_e32 v1, s1, v1
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
ptr addrspace(1) %in2) #1 {
@@ -515,7 +563,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f32_f16_e32 v1, s3
; CI-NEXT: v_cvt_f32_f16_e32 v2, s6
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -536,7 +587,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v1, -v3, v2, v1
+; CI-NEXT: v_fma_f32 v2, -v3, v2, v1
+; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; CI-NEXT: v_or_b32_e32 v1, v2, v1
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; CI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -559,6 +613,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: s_lshr_b32 s1, s2, 16
; VI-NEXT: v_rcp_f32_e32 v3, v3
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v0, v0, v2
; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
; VI-NEXT: v_mov_b32_e32 v2, s3
@@ -566,13 +621,19 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_trunc_f16_e32 v0, v0
; VI-NEXT: v_fma_f16 v0, -v0, v1, s2
; VI-NEXT: v_cvt_f32_f16_e32 v1, s1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: s_and_b32 s0, s1, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v1, v1, v3
; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s1
; VI-NEXT: v_trunc_f16_e32 v1, v1
; VI-NEXT: v_fma_f16 v1, -v1, v2, s1
+; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_or_b32_e32 v2, v0, v1
+; VI-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -614,7 +675,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f32_f16_e32 v1, s8
; CI-NEXT: v_cvt_f32_f16_e32 v2, s10
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -633,7 +697,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v1, -v3, v2, v1
+; CI-NEXT: v_fma_f32 v2, -v3, v2, v1
+; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; CI-NEXT: v_or_b32_e32 v1, v2, v1
; CI-NEXT: v_cvt_f32_f16_e32 v2, s3
; CI-NEXT: v_cvt_f32_f16_e32 v3, s1
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
@@ -651,7 +718,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; CI-NEXT: v_div_fixup_f32 v4, v4, v3, v2
; CI-NEXT: v_trunc_f32_e32 v4, v4
-; CI-NEXT: v_fma_f32 v2, -v4, v3, v2
+; CI-NEXT: v_fma_f32 v3, -v4, v3, v2
+; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CI-NEXT: v_and_b32_e32 v2, 0x80000000, v2
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_cvt_f32_f16_e32 v3, s9
; CI-NEXT: v_cvt_f32_f16_e32 v4, s11
; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -673,7 +743,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v5, v5, v4, v3
; CI-NEXT: v_trunc_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v3, -v5, v4, v3
+; CI-NEXT: v_fma_f32 v4, -v5, v4, v3
+; CI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v4
+; CI-NEXT: v_and_b32_e32 v3, 0x80000000, v3
+; CI-NEXT: v_or_b32_e32 v3, v4, v3
; CI-NEXT: v_cvt_f16_f32_e32 v3, v3
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; CI-NEXT: v_or_b32_e32 v1, v2, v1
@@ -708,30 +781,44 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_f16_e32 v1, s6
; VI-NEXT: v_cvt_f32_f16_e32 v5, s9
; VI-NEXT: s_lshr_b32 s7, s3, 16
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; VI-NEXT: v_mul_f32_e32 v1, v1, v3
; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_rcp_f32_e32 v5, v5
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s6
; VI-NEXT: v_trunc_f16_e32 v1, v1
; VI-NEXT: v_fma_f16 v1, -v1, v2, s6
; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; VI-NEXT: s_and_b32 s0, s6, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v2, v2, v4
; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
; VI-NEXT: v_mov_b32_e32 v4, s9
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
+; VI-NEXT: s_and_b32 s0, s3, 0xffff8000
; VI-NEXT: v_div_fixup_f16 v2, v2, v3, s3
; VI-NEXT: v_trunc_f16_e32 v2, v2
; VI-NEXT: v_fma_f16 v2, -v2, v3, s3
; VI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; VI-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; VI-NEXT: v_or_b32_e32 v2, s0, v2
+; VI-NEXT: s_and_b32 s0, s7, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v3, v3, v5
; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_div_fixup_f16 v3, v3, v4, s7
; VI-NEXT: v_trunc_f16_e32 v3, v3
; VI-NEXT: v_fma_f16 v3, -v3, v4, s7
-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-NEXT: v_and_b32_e32 v3, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v3, s0, v3
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
@@ -766,10 +853,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; CI-NEXT: v_trunc_f32_e32 v1, v1
; CI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3
; CI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3
; CI-NEXT: v_rcp_f32_e32 v4, v2
@@ -782,11 +872,14 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v2, -v2, v5, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; CI-NEXT: s_and_b32 s0, s3, 0x80000000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, s3
; CI-NEXT: v_trunc_f32_e32 v2, v2
; CI-NEXT: v_fma_f32 v1, -v2, v1, s3
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v1, s0, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -811,10 +904,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; VI-NEXT: v_trunc_f32_e32 v1, v1
; VI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
; VI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3
; VI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3
; VI-NEXT: v_rcp_f32_e32 v4, v2
@@ -827,10 +923,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v2, -v2, v5, v3
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; VI-NEXT: s_and_b32 s0, s3, 0x80000000
; VI-NEXT: v_div_fixup_f32 v2, v2, v1, s3
; VI-NEXT: v_trunc_f32_e32 v2, v2
; VI-NEXT: v_fma_f32 v1, -v2, v1, s3
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
@@ -869,6 +968,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v0, -v1, v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s9
; CI-NEXT: v_div_scale_f32 v2, s[6:7], v1, v1, s1
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; CI-NEXT: s_and_b32 s0, s0, 0x80000000
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: v_div_scale_f32 v3, vcc, s1, v1, s1
; CI-NEXT: v_rcp_f32_e32 v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -880,10 +982,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v2, -v2, v5, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; CI-NEXT: s_and_b32 s0, s1, 0x80000000
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, s1
; CI-NEXT: v_trunc_f32_e32 v2, v2
; CI-NEXT: v_fma_f32 v1, -v2, v1, s1
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
; CI-NEXT: v_mov_b32_e32 v2, s10
+; CI-NEXT: v_or_b32_e32 v1, s0, v1
; CI-NEXT: v_div_scale_f32 v3, s[0:1], v2, v2, s2
; CI-NEXT: v_div_scale_f32 v4, vcc, s2, v2, s2
; CI-NEXT: v_rcp_f32_e32 v5, v3
@@ -896,10 +1001,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v3, -v3, v6, v4
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, s2
; CI-NEXT: v_trunc_f32_e32 v3, v3
; CI-NEXT: v_fma_f32 v2, -v3, v2, s2
+; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
; CI-NEXT: v_mov_b32_e32 v3, s11
+; CI-NEXT: v_or_b32_e32 v2, s0, v2
; CI-NEXT: v_div_scale_f32 v4, s[0:1], v3, v3, s3
; CI-NEXT: v_div_scale_f32 v5, vcc, s3, v3, s3
; CI-NEXT: v_rcp_f32_e32 v6, v4
@@ -912,11 +1020,14 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v4, -v4, v7, v5
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
+; CI-NEXT: s_and_b32 s0, s3, 0x80000000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v4, v4, v3, s3
; CI-NEXT: v_trunc_f32_e32 v4, v4
; CI-NEXT: v_fma_f32 v3, -v4, v3, s3
+; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CI-NEXT: v_or_b32_e32 v3, s0, v3
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -946,6 +1057,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v0, -v1, v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_div_scale_f32 v2, s[6:7], v1, v1, s1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; VI-NEXT: s_and_b32 s0, s0, 0x80000000
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
; VI-NEXT: v_div_scale_f32 v3, vcc, s1, v1, s1
; VI-NEXT: v_rcp_f32_e32 v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -957,10 +1071,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v2, -v2, v5, v3
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; VI-NEXT: s_and_b32 s0, s1, 0x80000000
; VI-NEXT: v_div_fixup_f32 v2, v2, v1, s1
; VI-NEXT: v_trunc_f32_e32 v2, v2
; VI-NEXT: v_fma_f32 v1, -v2, v1, s1
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
; VI-NEXT: v_div_scale_f32 v3, s[0:1], v2, v2, s2
; VI-NEXT: v_div_scale_f32 v4, vcc, s2, v2, s2
; VI-NEXT: v_rcp_f32_e32 v5, v3
@@ -973,10 +1090,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v3, -v3, v6, v4
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_div_fixup_f32 v3, v3, v2, s2
; VI-NEXT: v_trunc_f32_e32 v3, v3
; VI-NEXT: v_fma_f32 v2, -v3, v2, s2
+; VI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
; VI-NEXT: v_mov_b32_e32 v3, s11
+; VI-NEXT: v_or_b32_e32 v2, s0, v2
; VI-NEXT: v_div_scale_f32 v4, s[0:1], v3, v3, s3
; VI-NEXT: v_div_scale_f32 v5, vcc, s3, v3, s3
; VI-NEXT: v_rcp_f32_e32 v6, v4
@@ -989,10 +1109,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v4, -v4, v7, v5
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
+; VI-NEXT: s_and_b32 s0, s3, 0x80000000
; VI-NEXT: v_div_fixup_f32 v4, v4, v3, s3
; VI-NEXT: v_trunc_f32_e32 v4, v4
; VI-NEXT: v_fma_f32 v3, -v4, v3, s3
+; VI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_or_b32_e32 v3, s0, v3
; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
@@ -1017,8 +1140,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_mov_b32_e32 v1, s9
; CI-NEXT: v_div_scale_f64 v[2:3], s[6:7], v[0:1], v[0:1], s[0:1]
; CI-NEXT: v_div_scale_f64 v[8:9], vcc, s[0:1], v[0:1], s[0:1]
-; CI-NEXT: s_mov_b32 s6, -1
-; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: s_brev_b32 s7, 1
; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; CI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -1031,9 +1154,16 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
; CI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1]
; CI-NEXT: v_mov_b32_e32 v2, s10
+; CI-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
; CI-NEXT: v_mov_b32_e32 v3, s11
-; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3]
; CI-NEXT: v_div_scale_f64 v[10:11], vcc, s[2:3], v[2:3], s[2:3]
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
+; CI-NEXT: v_or_b32_e32 v1, s1, v1
+; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3]
+; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7]
+; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5]
; CI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
; CI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
@@ -1045,6 +1175,9 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], s[2:3]
; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], s[2:3]
+; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CI-NEXT: v_or_b32_e32 v2, s0, v2
+; CI-NEXT: v_or_b32_e32 v3, s1, v3
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -1060,6 +1193,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_div_scale_f64 v[2:3], s[6:7], v[0:1], v[0:1], s[0:1]
; VI-NEXT: v_div_scale_f64 v[8:9], vcc, s[0:1], v[0:1], s[0:1]
+; VI-NEXT: s_mov_b32 s6, 0
+; VI-NEXT: s_brev_b32 s7, 1
; VI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; VI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; VI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -1072,9 +1207,14 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
; VI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1]
; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
; VI-NEXT: v_mov_b32_e32 v3, s11
-; VI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3]
; VI-NEXT: v_div_scale_f64 v[10:11], vcc, s[2:3], v[2:3], s[2:3]
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_or_b32_e32 v1, s1, v1
+; VI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3]
+; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7]
; VI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5]
; VI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
; VI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
@@ -1088,6 +1228,9 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], s[2:3]
; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; VI-NEXT: v_or_b32_e32 v2, s0, v2
+; VI-NEXT: v_or_b32_e32 v3, s1, v3
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
%gep2 = getelementptr <2 x double>, ptr addrspace(1) %in2, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 7d4393b653a756c..39a58dde8f1a25c 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -43,7 +43,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; SI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; SI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; SI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_bfi_b32 v0, s0, v1, v0
; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
@@ -72,6 +74,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; CI-NEXT: v_div_scale_f32 v3, s[0:1], v1, v1, v0
; CI-NEXT: v_div_scale_f32 v2, vcc, v0, v1, v0
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: v_rcp_f32_e32 v4, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; CI-NEXT: v_fma_f32 v5, -v3, v4, 1.0
@@ -84,7 +87,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_bfi_b32 v0, s0, v1, v0
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[8:11], 0
@@ -103,6 +107,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_load_ushort v2, v[2:3]
+; VI-NEXT: s_movk_i32 s0, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(1)
@@ -115,6 +120,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4
; VI-NEXT: v_trunc_f16_e32 v3, v3
; VI-NEXT: v_fma_f16 v2, -v3, v2, v4
+; VI-NEXT: v_bfi_b32 v2, s0, v2, v4
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -123,6 +129,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] offset:8
@@ -132,7 +139,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: v_mad_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
; GFX9-NEXT: v_div_fixup_f16 v3, v3, v2, v1
; GFX9-NEXT: v_trunc_f16_e32 v3, v3
-; GFX9-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX9-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1
; GFX9-NEXT: global_store_short v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -152,7 +160,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX10-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
; GFX10-NEXT: v_div_fixup_f16 v3, v3, v2, v1
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
-; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX10-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX10-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -175,7 +184,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX11-NEXT: v_div_fixup_f16 v3, v3, v2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f16_e32 v3, v3
-; GFX11-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX11-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX11-NEXT: global_store_b16 v0, v1, s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -201,7 +212,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1150-NEXT: v_trunc_f16_e32 v3, v3
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2
+; GFX1150-NEXT: v_fma_f16 v2, v3, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX1150-NEXT: global_store_b16 v0, v1, s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -409,7 +422,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: v_rcp_f32_e32 v2, v1
; SI-NEXT: v_mul_f32_e32 v2, v0, v2
; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; SI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_bfi_b32 v0, s0, v1, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
@@ -431,6 +446,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: s_mov_b32 s6, s10
; CI-NEXT: s_mov_b32 s7, s11
; CI-NEXT: buffer_load_ushort v0, off, s[4:7], 0
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: s_waitcnt vmcnt(1)
; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; CI-NEXT: v_rcp_f32_e32 v2, v1
@@ -438,7 +454,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; CI-NEXT: v_mul_f32_e32 v2, v0, v2
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_bfi_b32 v0, s0, v1, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[8:11], 0
; CI-NEXT: s_endpgm
@@ -456,6 +473,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_load_ushort v2, v[2:3]
+; VI-NEXT: s_movk_i32 s0, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -463,6 +481,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_mul_f16_e32 v3, v4, v3
; VI-NEXT: v_trunc_f16_e32 v3, v3
; VI-NEXT: v_fma_f16 v2, -v3, v2, v4
+; VI-NEXT: v_bfi_b32 v2, s0, v2, v4
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -471,6 +490,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] offset:8
@@ -478,7 +498,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: v_rcp_f16_e32 v3, v2
; GFX9-NEXT: v_mul_f16_e32 v3, v1, v3
; GFX9-NEXT: v_trunc_f16_e32 v3, v3
-; GFX9-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX9-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1
; GFX9-NEXT: global_store_short v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -496,7 +517,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: v_rcp_f16_e32 v3, v2
; GFX10-NEXT: v_mul_f16_e32 v3, v1, v3
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
-; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX10-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX10-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -516,7 +538,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX11-NEXT: v_mul_f16_e32 v3, v1, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f16_e32 v3, v3
-; GFX11-NEXT: v_fma_f16 v1, -v3, v2, v1
+; GFX11-NEXT: v_fma_f16 v2, -v3, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX11-NEXT: global_store_b16 v0, v1, s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -539,7 +563,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-NEXT: v_trunc_f16_e32 v3, v3
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2
+; GFX1150-NEXT: v_fma_f16 v2, v3, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX1150-NEXT: global_store_b16 v0, v1, s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -586,7 +612,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; SI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; SI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; SI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_bfi_b32 v0, s0, v1, v0
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
;
@@ -610,6 +638,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_div_scale_f32 v3, s[0:1], v1, v1, v0
; CI-NEXT: v_div_scale_f32 v2, vcc, v0, v1, v0
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: v_rcp_f32_e32 v4, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; CI-NEXT: v_fma_f32 v5, -v3, v4, 1.0
@@ -622,7 +651,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_bfi_b32 v0, s0, v1, v0
; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; CI-NEXT: s_endpgm
;
@@ -644,6 +674,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_div_scale_f32 v5, s[0:1], v2, v2, v4
; VI-NEXT: v_div_scale_f32 v3, vcc, v4, v2, v4
+; VI-NEXT: s_brev_b32 s0, -2
; VI-NEXT: v_rcp_f32_e32 v6, v5
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; VI-NEXT: v_fma_f32 v7, -v5, v6, 1.0
@@ -657,6 +688,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_div_fixup_f32 v3, v3, v2, v4
; VI-NEXT: v_trunc_f32_e32 v3, v3
; VI-NEXT: v_fma_f32 v2, -v3, v2, v4
+; VI-NEXT: v_bfi_b32 v2, s0, v2, v4
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -671,6 +703,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_div_scale_f32 v4, s[0:1], v2, v2, v1
; GFX9-NEXT: v_div_scale_f32 v3, vcc, v1, v2, v1
+; GFX9-NEXT: s_brev_b32 s0, -2
; GFX9-NEXT: v_rcp_f32_e32 v5, v4
; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX9-NEXT: v_fma_f32 v6, -v4, v5, 1.0
@@ -683,7 +716,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; GFX9-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -712,7 +746,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX10-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; GFX10-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -747,7 +782,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -785,7 +822,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1150-NEXT: v_trunc_f32_e32 v3, v3
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; GFX1150-NEXT: v_fmac_f32_e32 v1, v3, v2
+; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -982,7 +1021,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: v_rcp_f32_e32 v2, v1
; SI-NEXT: v_mul_f32_e32 v2, v0, v2
; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; SI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_bfi_b32 v0, s0, v1, v0
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
;
@@ -1003,11 +1044,13 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: s_mov_b32 s3, s11
; CI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; CI-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:16
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_rcp_f32_e32 v2, v1
; CI-NEXT: v_mul_f32_e32 v2, v0, v2
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_bfi_b32 v0, s0, v1, v0
; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; CI-NEXT: s_endpgm
;
@@ -1024,6 +1067,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_load_dword v2, v[2:3]
+; VI-NEXT: s_brev_b32 s0, -2
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -1031,6 +1075,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_mul_f32_e32 v3, v4, v3
; VI-NEXT: v_trunc_f32_e32 v3, v3
; VI-NEXT: v_fma_f32 v2, -v3, v2, v4
+; VI-NEXT: v_bfi_b32 v2, s0, v2, v4
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -1039,6 +1084,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_brev_b32 s0, -2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:16
@@ -1046,7 +1092,8 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: v_rcp_f32_e32 v3, v2
; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -1064,7 +1111,8 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-NEXT: v_mul_f32_e32 v3, v1, v3
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -1084,7 +1132,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX11-NEXT: v_mul_f32_e32 v3, v1, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: v_fma_f32 v1, -v3, v2, v1
+; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -1107,7 +1157,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-NEXT: v_trunc_f32_e32 v3, v3
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; GFX1150-NEXT: v_fmac_f32_e32 v1, v3, v2
+; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -1176,8 +1228,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; SI-NEXT: v_mov_b32_e32 v7, s2
; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_bfi_b32 v3, s0, v3, v1
+; SI-NEXT: buffer_store_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; CI-LABEL: frem_f64:
@@ -1199,6 +1253,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1]
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5]
; CI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
; CI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
@@ -1211,8 +1266,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
; CI-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; CI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; CI-NEXT: v_bfi_b32 v3, s0, v3, v1
+; CI-NEXT: buffer_store_dwordx2 v[2:3], off, s[8:11], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: frem_f64:
@@ -1230,6 +1286,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[2:3]
+; VI-NEXT: s_brev_b32 s0, -2
; VI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
; VI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
; VI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
@@ -1242,8 +1299,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
; VI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[2:3]
; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT: v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
-; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: v_fma_f64 v[4:5], -v[6:7], v[4:5], v[2:3]
+; VI-NEXT: v_bfi_b32 v5, s0, v5, v3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: frem_f64:
@@ -1256,6 +1314,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: global_load_dwordx2 v[2:3], v12, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1]
+; GFX9-NEXT: s_brev_b32 s0, -2
; GFX9-NEXT: v_rcp_f64_e32 v[6:7], v[4:5]
; GFX9-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
; GFX9-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
@@ -1268,8 +1327,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX9-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
; GFX9-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX9-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX9-NEXT: global_store_dwordx2 v12, v[0:1], s[4:5]
+; GFX9-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1
+; GFX9-NEXT: global_store_dwordx2 v12, v[2:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: frem_f64:
@@ -1295,8 +1355,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX10-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
; GFX10-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
; GFX10-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX10-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX10-NEXT: global_store_dwordx2 v12, v[0:1], s[4:5]
+; GFX10-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX10-NEXT: global_store_dwordx2 v12, v[2:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: frem_f64:
@@ -1328,8 +1389,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX11-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX11-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX11-NEXT: global_store_b64 v12, v[0:1], s[4:5]
+; GFX11-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX11-NEXT: global_store_b64 v12, v[2:3], s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
@@ -1362,9 +1425,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
; GFX1150-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX1150-NEXT: global_store_b64 v12, v[0:1], s[4:5]
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX1150-NEXT: global_store_b64 v12, v[2:3], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX1150-NEXT: s_endpgm
@@ -1649,8 +1713,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: v_mov_b32_e32 v7, s6
; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT: s_brev_b32 s4, -2
+; SI-NEXT: v_bfi_b32 v3, s4, v3, v1
+; SI-NEXT: buffer_store_dwordx2 v[2:3], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; CI-LABEL: unsafe_frem_f64:
@@ -1670,6 +1736,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: s_mov_b32 s3, s11
; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; CI-NEXT: s_brev_b32 s0, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
@@ -1680,8 +1747,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
; CI-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; CI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; CI-NEXT: v_bfi_b32 v3, s0, v3, v1
+; CI-NEXT: buffer_store_dwordx2 v[2:3], off, s[8:11], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: unsafe_frem_f64:
@@ -1695,6 +1763,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; VI-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
+; VI-NEXT: s_brev_b32 s0, -2
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -1707,8 +1776,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_fma_f64 v[10:11], -v[4:5], v[8:9], v[2:3]
; VI-NEXT: v_fma_f64 v[6:7], v[10:11], v[6:7], v[8:9]
; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT: v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3]
-; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: v_fma_f64 v[4:5], -v[6:7], v[4:5], v[2:3]
+; VI-NEXT: v_bfi_b32 v5, s0, v5, v3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: unsafe_frem_f64:
@@ -1716,6 +1786,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v10, 0
+; GFX9-NEXT: s_brev_b32 s0, -2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v10, s[6:7]
; GFX9-NEXT: global_load_dwordx2 v[2:3], v10, s[2:3]
@@ -1729,8 +1800,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
; GFX9-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX9-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX9-NEXT: global_store_dwordx2 v10, v[0:1], s[4:5]
+; GFX9-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1
+; GFX9-NEXT: global_store_dwordx2 v10, v[2:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: unsafe_frem_f64:
@@ -1753,8 +1825,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
; GFX10-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
; GFX10-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX10-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX10-NEXT: global_store_dwordx2 v10, v[0:1], s[4:5]
+; GFX10-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX10-NEXT: global_store_dwordx2 v10, v[2:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: unsafe_frem_f64:
@@ -1782,8 +1855,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; GFX11-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX11-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX11-NEXT: global_store_b64 v10, v[0:1], s[4:5]
+; GFX11-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX11-NEXT: global_store_b64 v10, v[2:3], s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
@@ -1812,9 +1887,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
; GFX1150-NEXT: v_trunc_f64_e32 v[4:5], v[4:5]
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
-; GFX1150-NEXT: global_store_b64 v10, v[0:1], s[4:5]
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1]
+; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1
+; GFX1150-NEXT: global_store_b64 v10, v[2:3], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX1150-NEXT: s_endpgm
@@ -1866,7 +1942,9 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; SI-NEXT: v_div_fixup_f32 v4, v4, v2, v0
; SI-NEXT: v_trunc_f32_e32 v4, v4
-; SI-NEXT: v_fma_f32 v0, -v4, v2, v0
+; SI-NEXT: v_fma_f32 v2, -v4, v2, v0
+; SI-NEXT: s_brev_b32 s6, -2
+; SI-NEXT: v_bfi_b32 v0, s6, v2, v0
; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
@@ -1884,7 +1962,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v2, v2, v5, v6
; SI-NEXT: v_div_fixup_f32 v2, v2, v3, v1
; SI-NEXT: v_trunc_f32_e32 v2, v2
-; SI-NEXT: v_fma_f32 v1, -v2, v3, v1
+; SI-NEXT: v_fma_f32 v2, -v2, v3, v1
+; SI-NEXT: v_bfi_b32 v1, s6, v2, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1907,6 +1986,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s11, s3
; CI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; CI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:16
+; CI-NEXT: s_brev_b32 s6, -2
; CI-NEXT: s_waitcnt vmcnt(1)
; CI-NEXT: v_cvt_f32_f16_e32 v1, v0
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -1929,8 +2009,9 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; CI-NEXT: v_div_fixup_f32 v4, v4, v2, v0
; CI-NEXT: v_trunc_f32_e32 v4, v4
-; CI-NEXT: v_fma_f32 v0, -v4, v2, v0
+; CI-NEXT: v_fma_f32 v2, -v4, v2, v0
; CI-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v1
+; CI-NEXT: v_bfi_b32 v0, s6, v2, v0
; CI-NEXT: v_div_scale_f32 v2, vcc, v1, v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -1947,7 +2028,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v2, v2, v5, v6
; CI-NEXT: v_div_fixup_f32 v2, v2, v3, v1
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v1, -v2, v3, v1
+; CI-NEXT: v_fma_f32 v2, -v2, v3, v1
+; CI-NEXT: v_bfi_b32 v1, s6, v2, v1
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1966,30 +2048,33 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_load_dword v2, v[2:3]
+; VI-NEXT: s_movk_i32 s0, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v3
+; VI-NEXT: v_cvt_f32_f16_e32 v3, v4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; VI-NEXT: v_cvt_f32_f16_e32 v7, v6
-; VI-NEXT: v_rcp_f32_e32 v7, v7
-; VI-NEXT: v_mul_f32_e32 v5, v5, v7
-; VI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; VI-NEXT: v_div_fixup_f16 v5, v5, v6, v3
-; VI-NEXT: v_trunc_f16_e32 v5, v5
-; VI-NEXT: v_fma_f16 v3, -v5, v6, v3
+; VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; VI-NEXT: v_rcp_f32_e32 v5, v5
+; VI-NEXT: v_mul_f32_e32 v3, v3, v5
+; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4
+; VI-NEXT: v_trunc_f16_e32 v3, v3
+; VI-NEXT: v_fma_f16 v3, -v3, v2, v4
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; VI-NEXT: v_cvt_f32_f16_e32 v6, v2
+; VI-NEXT: v_bfi_b32 v3, s0, v3, v4
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; VI-NEXT: v_cvt_f32_f16_e32 v5, v4
-; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_rcp_f32_e32 v6, v6
; VI-NEXT: v_mul_f32_e32 v5, v5, v6
; VI-NEXT: v_cvt_f16_f32_e32 v5, v5
; VI-NEXT: v_div_fixup_f16 v5, v5, v2, v4
; VI-NEXT: v_trunc_f16_e32 v5, v5
; VI-NEXT: v_fma_f16 v2, -v5, v2, v4
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-NEXT: v_bfi_b32 v2, s0, v2, v4
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -1998,6 +2083,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:16
@@ -2010,12 +2096,14 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_fma_f16 v3, -v3, v2, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1
; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_mad_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_div_fixup_f16 v4, v4, v2, v1
; GFX9-NEXT: v_trunc_f16_e32 v4, v4
-; GFX9-NEXT: v_fma_f16 v1, -v4, v2, v1
+; GFX9-NEXT: v_fma_f16 v2, -v4, v2, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1
; GFX9-NEXT: v_pack_b32_f16 v1, v3, v1
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
@@ -2038,13 +2126,15 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
; GFX10-NEXT: v_fma_f16 v3, -v3, v2, v1
; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_bfi_b32 v3, 0x7fff, v3, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v4, v2
; GFX10-NEXT: v_rcp_f32_e32 v4, v4
; GFX10-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_div_fixup_f16 v4, v4, v2, v1
; GFX10-NEXT: v_trunc_f16_e32 v4, v4
-; GFX10-NEXT: v_fma_f16 v1, -v4, v2, v1
+; GFX10-NEXT: v_fma_f16 v2, -v4, v2, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX10-NEXT: v_pack_b32_f16 v1, v3, v1
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
@@ -2070,17 +2160,20 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_trunc_f16_e32 v3, v3
; GFX11-NEXT: v_fma_f16 v3, -v3, v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v3, 0x7fff, v3, v1
; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v4, v4
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fixup_f16 v4, v4, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f16_e32 v4, v4
+; GFX11-NEXT: v_fma_f16 v2, -v4, v2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f16 v1, -v4, v2, v1
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX11-NEXT: v_pack_b32_f16 v1, v3, v1
; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX11-NEXT: s_nop 0
@@ -2110,19 +2203,22 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_trunc_f16_e32 v4, v4
; GFX1150-NEXT: v_xor_b32_e32 v4, 0x8000, v4
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fmac_f16_e32 v5, v4, v3
-; GFX1150-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX1150-NEXT: v_rcp_f32_e32 v3, v3
-; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
-; GFX1150-NEXT: v_div_fixup_f16 v3, v3, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_fma_f16 v3, v4, v3, v5
+; GFX1150-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX1150-NEXT: v_bfi_b32 v3, 0x7fff, v3, v5
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: v_rcp_f32_e32 v4, v4
+; GFX1150-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel_hi:[1,0,0]
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_trunc_f16_e32 v3, v3
-; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3
+; GFX1150-NEXT: v_div_fixup_f16 v4, v4, v2, v1
+; GFX1150-NEXT: v_trunc_f16_e32 v4, v4
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2
-; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v5
+; GFX1150-NEXT: v_xor_b32_e32 v4, 0x8000, v4
+; GFX1150-NEXT: v_fma_f16 v2, v4, v2, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
+; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -2183,6 +2279,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fixup_f32 v8, v8, v1, v5
; SI-NEXT: v_trunc_f32_e32 v8, v8
; SI-NEXT: v_fma_f32 v1, -v8, v1, v5
+; SI-NEXT: s_brev_b32 s6, -2
+; SI-NEXT: v_bfi_b32 v1, s6, v1, v5
; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
@@ -2200,7 +2298,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v5, v5, v9, v10
; SI-NEXT: v_div_fixup_f32 v5, v5, v7, v4
; SI-NEXT: v_trunc_f32_e32 v5, v5
-; SI-NEXT: v_fma_f32 v4, -v5, v7, v4
+; SI-NEXT: v_fma_f32 v5, -v5, v7, v4
+; SI-NEXT: v_bfi_b32 v4, s6, v5, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v4, v1
; SI-NEXT: v_div_scale_f32 v4, vcc, v3, v0, v3
@@ -2218,6 +2317,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fixup_f32 v4, v4, v0, v3
; SI-NEXT: v_trunc_f32_e32 v4, v4
; SI-NEXT: v_fma_f32 v0, -v4, v0, v3
+; SI-NEXT: v_bfi_b32 v0, s6, v0, v3
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_div_scale_f32 v3, vcc, v2, v6, v2
@@ -2234,7 +2334,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v3, v3, v5, v7
; SI-NEXT: v_div_fixup_f32 v3, v3, v6, v2
; SI-NEXT: v_trunc_f32_e32 v3, v3
-; SI-NEXT: v_fma_f32 v2, -v3, v6, v2
+; SI-NEXT: v_fma_f32 v3, -v3, v6, v2
+; SI-NEXT: v_bfi_b32 v2, s6, v3, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_or_b32_e32 v0, v2, v0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2256,6 +2357,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s7, s3
; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_mov_b32 s11, s3
+; CI-NEXT: s_brev_b32 s6, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -2287,6 +2389,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_trunc_f32_e32 v8, v8
; CI-NEXT: v_fma_f32 v1, -v8, v1, v5
; CI-NEXT: v_div_scale_f32 v8, s[4:5], v7, v7, v4
+; CI-NEXT: v_bfi_b32 v1, s6, v1, v5
; CI-NEXT: v_div_scale_f32 v5, vcc, v4, v7, v4
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
@@ -2303,7 +2406,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v5, v5, v9, v10
; CI-NEXT: v_div_fixup_f32 v5, v5, v7, v4
; CI-NEXT: v_trunc_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v4, -v5, v7, v4
+; CI-NEXT: v_fma_f32 v5, -v5, v7, v4
+; CI-NEXT: v_bfi_b32 v4, s6, v5, v4
; CI-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, v3
; CI-NEXT: v_cvt_f16_f32_e32 v4, v4
; CI-NEXT: v_or_b32_e32 v1, v4, v1
@@ -2322,6 +2426,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_trunc_f32_e32 v4, v4
; CI-NEXT: v_fma_f32 v0, -v4, v0, v3
; CI-NEXT: v_div_scale_f32 v4, s[4:5], v6, v6, v2
+; CI-NEXT: v_bfi_b32 v0, s6, v0, v3
; CI-NEXT: v_div_scale_f32 v3, vcc, v2, v6, v2
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
@@ -2337,7 +2442,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v7
; CI-NEXT: v_div_fixup_f32 v3, v3, v6, v2
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v2, -v3, v6, v2
+; CI-NEXT: v_fma_f32 v3, -v3, v6, v2
+; CI-NEXT: v_bfi_b32 v2, s6, v3, v2
; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
; CI-NEXT: v_or_b32_e32 v0, v2, v0
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2356,50 +2462,55 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: flat_load_dwordx2 v[4:5], v[4:5]
; VI-NEXT: v_mov_b32_e32 v3, s7
; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
+; VI-NEXT: s_movk_i32 s0, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v5
-; VI-NEXT: v_cvt_f32_f16_e32 v9, v8
+; VI-NEXT: v_cvt_f32_f16_e32 v7, v5
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v7, v6
-; VI-NEXT: v_rcp_f32_e32 v9, v9
-; VI-NEXT: v_mul_f32_e32 v7, v7, v9
-; VI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; VI-NEXT: v_div_fixup_f16 v7, v7, v8, v6
-; VI-NEXT: v_trunc_f16_e32 v7, v7
-; VI-NEXT: v_fma_f16 v6, -v7, v8, v6
+; VI-NEXT: v_cvt_f32_f16_e32 v6, v3
+; VI-NEXT: v_rcp_f32_e32 v7, v7
+; VI-NEXT: v_mul_f32_e32 v6, v6, v7
+; VI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; VI-NEXT: v_div_fixup_f16 v6, v6, v5, v3
+; VI-NEXT: v_trunc_f16_e32 v6, v6
+; VI-NEXT: v_fma_f16 v6, -v6, v5, v3
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; VI-NEXT: v_cvt_f32_f16_e32 v8, v5
+; VI-NEXT: v_bfi_b32 v6, s0, v6, v3
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; VI-NEXT: v_cvt_f32_f16_e32 v7, v3
-; VI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; VI-NEXT: v_rcp_f32_e32 v8, v8
; VI-NEXT: v_mul_f32_e32 v7, v7, v8
; VI-NEXT: v_cvt_f16_f32_e32 v7, v7
; VI-NEXT: v_div_fixup_f16 v7, v7, v5, v3
; VI-NEXT: v_trunc_f16_e32 v7, v7
-; VI-NEXT: v_fma_f16 v3, -v7, v5, v3
-; VI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v8, v7
-; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; VI-NEXT: v_or_b32_e32 v3, v3, v6
-; VI-NEXT: v_cvt_f32_f16_e32 v6, v5
-; VI-NEXT: v_rcp_f32_e32 v8, v8
-; VI-NEXT: v_mul_f32_e32 v6, v6, v8
-; VI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; VI-NEXT: v_div_fixup_f16 v6, v6, v7, v5
-; VI-NEXT: v_trunc_f16_e32 v6, v6
-; VI-NEXT: v_fma_f16 v5, -v6, v7, v5
+; VI-NEXT: v_fma_f16 v5, -v7, v5, v3
+; VI-NEXT: v_bfi_b32 v3, s0, v5, v3
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT: v_cvt_f32_f16_e32 v6, v4
+; VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; VI-NEXT: v_rcp_f32_e32 v6, v6
+; VI-NEXT: v_mul_f32_e32 v5, v5, v6
+; VI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; VI-NEXT: v_div_fixup_f16 v5, v5, v4, v2
+; VI-NEXT: v_trunc_f16_e32 v5, v5
+; VI-NEXT: v_fma_f16 v5, -v5, v4, v2
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; VI-NEXT: v_cvt_f32_f16_e32 v7, v4
+; VI-NEXT: v_bfi_b32 v5, s0, v5, v2
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; VI-NEXT: v_cvt_f32_f16_e32 v6, v2
-; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_rcp_f32_e32 v7, v7
; VI-NEXT: v_mul_f32_e32 v6, v6, v7
; VI-NEXT: v_cvt_f16_f32_e32 v6, v6
; VI-NEXT: v_div_fixup_f16 v6, v6, v4, v2
; VI-NEXT: v_trunc_f16_e32 v6, v6
-; VI-NEXT: v_fma_f16 v2, -v6, v4, v2
-; VI-NEXT: v_or_b32_e32 v2, v2, v5
+; VI-NEXT: v_fma_f16 v4, -v6, v4, v2
+; VI-NEXT: v_bfi_b32 v2, s0, v4, v2
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; VI-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
@@ -2408,6 +2519,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:32
@@ -2420,12 +2532,14 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_fma_f16 v5, -v5, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_cvt_f32_f16_e32 v6, v3
+; GFX9-NEXT: v_bfi_b32 v5, s0, v5, v1
; GFX9-NEXT: v_rcp_f32_e32 v6, v6
; GFX9-NEXT: v_mad_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_div_fixup_f16 v6, v6, v3, v1
; GFX9-NEXT: v_trunc_f16_e32 v6, v6
-; GFX9-NEXT: v_fma_f16 v1, -v6, v3, v1
+; GFX9-NEXT: v_fma_f16 v3, -v6, v3, v1
+; GFX9-NEXT: v_bfi_b32 v1, s0, v3, v1
; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-NEXT: v_pack_b32_f16 v1, v5, v1
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
@@ -2435,12 +2549,14 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_fma_f16 v3, -v3, v2, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v0
; GFX9-NEXT: v_rcp_f32_e32 v5, v5
; GFX9-NEXT: v_mad_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_div_fixup_f16 v5, v5, v2, v0
; GFX9-NEXT: v_trunc_f16_e32 v5, v5
-; GFX9-NEXT: v_fma_f16 v0, -v5, v2, v0
+; GFX9-NEXT: v_fma_f16 v2, -v5, v2, v0
+; GFX9-NEXT: v_bfi_b32 v0, s0, v2, v0
; GFX9-NEXT: v_pack_b32_f16 v0, v3, v0
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
@@ -2463,13 +2579,15 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_trunc_f16_e32 v5, v5
; GFX10-NEXT: v_fma_f16 v5, -v5, v3, v1
; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_bfi_b32 v5, 0x7fff, v5, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v3
; GFX10-NEXT: v_rcp_f32_e32 v6, v6
; GFX10-NEXT: v_fma_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_div_fixup_f16 v6, v6, v3, v1
; GFX10-NEXT: v_trunc_f16_e32 v6, v6
-; GFX10-NEXT: v_fma_f16 v1, -v6, v3, v1
+; GFX10-NEXT: v_fma_f16 v3, -v6, v3, v1
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX10-NEXT: v_pack_b32_f16 v1, v5, v1
; GFX10-NEXT: v_rcp_f32_e32 v3, v3
@@ -2478,13 +2596,15 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
; GFX10-NEXT: v_fma_f16 v3, -v3, v2, v0
; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_bfi_b32 v3, 0x7fff, v3, v0
; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX10-NEXT: v_rcp_f32_e32 v5, v5
; GFX10-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_div_fixup_f16 v5, v5, v2, v0
; GFX10-NEXT: v_trunc_f16_e32 v5, v5
-; GFX10-NEXT: v_fma_f16 v0, -v5, v2, v0
+; GFX10-NEXT: v_fma_f16 v2, -v5, v2, v0
+; GFX10-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0
; GFX10-NEXT: v_pack_b32_f16 v0, v3, v0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
@@ -2510,17 +2630,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_trunc_f16_e32 v5, v5
; GFX11-NEXT: v_fma_f16 v5, -v5, v3, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v5, 0x7fff, v5, v1
; GFX11-NEXT: v_cvt_f32_f16_e32 v6, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v6, v6
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fixup_f16 v6, v6, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f16_e32 v6, v6
+; GFX11-NEXT: v_fma_f16 v3, -v6, v3, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f16 v1, -v6, v3, v1
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1
; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX11-NEXT: v_pack_b32_f16 v1, v5, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -2532,17 +2655,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_trunc_f16_e32 v3, v3
; GFX11-NEXT: v_fma_f16 v3, -v3, v2, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v3, 0x7fff, v3, v0
; GFX11-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v5, v5
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fixup_f16 v5, v5, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f16_e32 v5, v5
+; GFX11-NEXT: v_fma_f16 v2, -v5, v2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f16 v0, -v5, v2, v0
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0
; GFX11-NEXT: v_pack_b32_f16 v0, v3, v0
; GFX11-NEXT: global_store_b64 v4, v[0:1], s[4:5]
; GFX11-NEXT: s_nop 0
@@ -2572,21 +2698,24 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_trunc_f16_e32 v6, v6
; GFX1150-NEXT: v_xor_b32_e32 v6, 0x8000, v6
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1150-NEXT: v_fmac_f16_e32 v7, v6, v5
-; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX1150-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX1150-NEXT: v_rcp_f32_e32 v5, v5
-; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel_hi:[1,0,0]
-; GFX1150-NEXT: v_div_fixup_f16 v5, v5, v2, v0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_trunc_f16_e32 v5, v5
-; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1150-NEXT: v_fma_f16 v0, v5, v2, v0
+; GFX1150-NEXT: v_fma_f16 v5, v6, v5, v7
+; GFX1150-NEXT: v_cvt_f32_f16_e32 v6, v2
+; GFX1150-NEXT: v_bfi_b32 v5, 0x7fff, v5, v7
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: v_rcp_f32_e32 v6, v6
+; GFX1150-NEXT: v_fma_mixlo_f16 v6, v0, v6, 0 op_sel_hi:[1,0,0]
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_div_fixup_f16 v6, v6, v2, v0
+; GFX1150-NEXT: v_trunc_f16_e32 v6, v6
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_xor_b32_e32 v6, 0x8000, v6
+; GFX1150-NEXT: v_fma_f16 v2, v6, v2, v0
+; GFX1150-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0
; GFX1150-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX1150-NEXT: v_pack_b32_f16 v0, v0, v7
+; GFX1150-NEXT: v_pack_b32_f16 v0, v0, v5
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX1150-NEXT: v_rcp_f32_e32 v5, v5
@@ -2596,19 +2725,22 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_trunc_f16_e32 v5, v5
; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fmac_f16_e32 v6, v5, v2
-; GFX1150-NEXT: v_cvt_f32_f16_e32 v2, v3
-; GFX1150-NEXT: v_rcp_f32_e32 v2, v2
-; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_mixlo_f16 v2, v1, v2, 0 op_sel_hi:[1,0,0]
-; GFX1150-NEXT: v_div_fixup_f16 v2, v2, v3, v1
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_fma_f16 v2, v5, v2, v6
+; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v3
+; GFX1150-NEXT: v_bfi_b32 v2, 0x7fff, v2, v6
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: v_rcp_f32_e32 v5, v5
+; GFX1150-NEXT: v_fma_mixlo_f16 v5, v1, v5, 0 op_sel_hi:[1,0,0]
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_trunc_f16_e32 v2, v2
-; GFX1150-NEXT: v_xor_b32_e32 v2, 0x8000, v2
+; GFX1150-NEXT: v_div_fixup_f16 v5, v5, v3, v1
+; GFX1150-NEXT: v_trunc_f16_e32 v5, v5
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5
+; GFX1150-NEXT: v_fma_f16 v3, v5, v3, v1
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fmac_f16_e32 v1, v2, v3
-; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v6
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1
+; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v2
; GFX1150-NEXT: global_store_b64 v4, v[0:1], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -2655,7 +2787,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; SI-NEXT: v_div_fixup_f32 v4, v4, v3, v1
; SI-NEXT: v_trunc_f32_e32 v4, v4
-; SI-NEXT: v_fma_f32 v1, -v4, v3, v1
+; SI-NEXT: v_fma_f32 v3, -v4, v3, v1
+; SI-NEXT: s_brev_b32 s6, -2
+; SI-NEXT: v_bfi_b32 v1, s6, v3, v1
; SI-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0
; SI-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v0
; SI-NEXT: v_rcp_f32_e32 v5, v4
@@ -2670,7 +2804,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; SI-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; SI-NEXT: v_trunc_f32_e32 v3, v3
-; SI-NEXT: v_fma_f32 v0, -v3, v2, v0
+; SI-NEXT: v_fma_f32 v2, -v3, v2, v0
+; SI-NEXT: v_bfi_b32 v0, s6, v2, v0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -2691,6 +2826,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s11, s3
; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[8:11], 0 offset:32
+; CI-NEXT: s_brev_b32 s6, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_div_scale_f32 v5, s[4:5], v3, v3, v1
; CI-NEXT: v_div_scale_f32 v4, vcc, v1, v3, v1
@@ -2706,8 +2842,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; CI-NEXT: v_div_fixup_f32 v4, v4, v3, v1
; CI-NEXT: v_trunc_f32_e32 v4, v4
-; CI-NEXT: v_fma_f32 v1, -v4, v3, v1
+; CI-NEXT: v_fma_f32 v3, -v4, v3, v1
; CI-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; CI-NEXT: v_bfi_b32 v1, s6, v3, v1
; CI-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0
; CI-NEXT: v_rcp_f32_e32 v5, v4
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -2721,7 +2858,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v0, -v3, v2, v0
+; CI-NEXT: v_fma_f32 v2, -v3, v2, v0
+; CI-NEXT: v_bfi_b32 v0, s6, v2, v0
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
@@ -2729,6 +2867,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT: s_brev_b32 s2, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: s_add_u32 s0, s0, 32
@@ -2755,8 +2894,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v6, v6, v8, v9
; VI-NEXT: v_div_fixup_f32 v6, v6, v5, v3
; VI-NEXT: v_trunc_f32_e32 v6, v6
-; VI-NEXT: v_fma_f32 v3, -v6, v5, v3
+; VI-NEXT: v_fma_f32 v5, -v6, v5, v3
; VI-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v2
+; VI-NEXT: v_bfi_b32 v3, s2, v5, v3
; VI-NEXT: v_div_scale_f32 v5, vcc, v2, v4, v2
; VI-NEXT: v_rcp_f32_e32 v7, v6
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -2770,7 +2910,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v5, v5, v7, v8
; VI-NEXT: v_div_fixup_f32 v5, v5, v4, v2
; VI-NEXT: v_trunc_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v2, -v5, v4, v2
+; VI-NEXT: v_fma_f32 v4, -v5, v4, v2
+; VI-NEXT: v_bfi_b32 v2, s2, v4, v2
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
@@ -2782,6 +2923,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7]
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:32
+; GFX9-NEXT: s_brev_b32 s2, -2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_div_scale_f32 v6, s[0:1], v3, v3, v1
; GFX9-NEXT: v_div_scale_f32 v5, vcc, v1, v3, v1
@@ -2797,8 +2939,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v5, v5, v7, v8
; GFX9-NEXT: v_div_fixup_f32 v5, v5, v3, v1
; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_fma_f32 v1, -v5, v3, v1
+; GFX9-NEXT: v_fma_f32 v3, -v5, v3, v1
; GFX9-NEXT: v_div_scale_f32 v5, s[0:1], v2, v2, v0
+; GFX9-NEXT: v_bfi_b32 v1, s2, v3, v1
; GFX9-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0
; GFX9-NEXT: v_rcp_f32_e32 v6, v5
; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -2812,7 +2955,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v3, v3, v6, v7
; GFX9-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_fma_f32 v0, -v3, v2, v0
+; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v0
+; GFX9-NEXT: v_bfi_b32 v0, s2, v2, v0
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -2841,10 +2985,11 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v5, v5, v7, v8
; GFX10-NEXT: v_div_fixup_f32 v5, v5, v3, v1
; GFX10-NEXT: v_trunc_f32_e32 v5, v5
-; GFX10-NEXT: v_fma_f32 v1, -v5, v3, v1
+; GFX10-NEXT: v_fma_f32 v3, -v5, v3, v1
; GFX10-NEXT: v_div_scale_f32 v5, s0, v2, v2, v0
-; GFX10-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1
; GFX10-NEXT: v_rcp_f32_e32 v6, v5
+; GFX10-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v7, -v5, v6, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v6, v7, v6
@@ -2856,7 +3001,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v3, v3, v6, v7
; GFX10-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_fma_f32 v0, -v3, v2, v0
+; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v0
+; GFX10-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -2891,28 +3037,31 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_div_fixup_f32 v5, v5, v3, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v5, v5
-; GFX11-NEXT: v_fma_f32 v1, -v5, v3, v1
+; GFX11-NEXT: v_fma_f32 v3, -v5, v3, v1
; GFX11-NEXT: v_div_scale_f32 v5, null, v2, v2, v0
-; GFX11-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1
; GFX11-NEXT: v_rcp_f32_e32 v6, v5
+; GFX11-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_f32 v7, -v5, v6, 1.0
-; GFX11-NEXT: v_fmac_f32_e32 v6, v7, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v6, v7, v6
; GFX11-NEXT: v_mul_f32_e32 v7, v3, v6
-; GFX11-NEXT: v_fma_f32 v8, -v5, v7, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v8, -v5, v7, v3
; GFX11-NEXT: v_fmac_f32_e32 v7, v8, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v3, -v5, v7, v3
; GFX11-NEXT: s_denorm_mode 12
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fmas_f32 v3, v3, v6, v7
-; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v0
; GFX11-NEXT: v_trunc_f32_e32 v3, v3
-; GFX11-NEXT: v_fma_f32 v0, -v3, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v0
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
; GFX11-NEXT: global_store_b64 v4, v[0:1], s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -2950,29 +3099,32 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: v_trunc_f32_e32 v5, v5
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1150-NEXT: v_fma_f32 v1, v5, v3, v1
+; GFX1150-NEXT: v_fma_f32 v3, v5, v3, v1
; GFX1150-NEXT: v_div_scale_f32 v5, null, v2, v2, v0
-; GFX1150-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1
; GFX1150-NEXT: v_rcp_f32_e32 v6, v5
+; GFX1150-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0
; GFX1150-NEXT: s_denorm_mode 15
+; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v7, -v5, v6, 1.0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v6, v7, v6
-; GFX1150-NEXT: v_mul_f32_e32 v7, v3, v6
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_mul_f32_e32 v7, v3, v6
; GFX1150-NEXT: v_fma_f32 v8, -v5, v7, v3
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v7, v8, v6
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v3, -v5, v7, v3
; GFX1150-NEXT: s_denorm_mode 12
-; GFX1150-NEXT: v_div_fmas_f32 v3, v3, v6, v7
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_div_fmas_f32 v3, v3, v6, v7
; GFX1150-NEXT: v_div_fixup_f32 v3, v3, v2, v0
-; GFX1150-NEXT: v_trunc_f32_e32 v3, v3
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_trunc_f32_e32 v3, v3
; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; GFX1150-NEXT: v_fmac_f32_e32 v0, v3, v2
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v0
+; GFX1150-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
; GFX1150-NEXT: global_store_b64 v4, v[0:1], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -3019,7 +3171,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v8, v8, v10, v11
; SI-NEXT: v_div_fixup_f32 v8, v8, v7, v3
; SI-NEXT: v_trunc_f32_e32 v8, v8
-; SI-NEXT: v_fma_f32 v3, -v8, v7, v3
+; SI-NEXT: v_fma_f32 v7, -v8, v7, v3
+; SI-NEXT: s_brev_b32 s6, -2
+; SI-NEXT: v_bfi_b32 v3, s6, v7, v3
; SI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2
; SI-NEXT: v_div_scale_f32 v8, s[4:5], v6, v6, v2
; SI-NEXT: v_rcp_f32_e32 v9, v8
@@ -3034,7 +3188,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v7, v7, v9, v10
; SI-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; SI-NEXT: v_trunc_f32_e32 v7, v7
-; SI-NEXT: v_fma_f32 v2, -v7, v6, v2
+; SI-NEXT: v_fma_f32 v6, -v7, v6, v2
+; SI-NEXT: v_bfi_b32 v2, s6, v6, v2
; SI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1
; SI-NEXT: v_div_scale_f32 v7, s[4:5], v5, v5, v1
; SI-NEXT: v_rcp_f32_e32 v8, v7
@@ -3049,7 +3204,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v6, v6, v8, v9
; SI-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; SI-NEXT: v_trunc_f32_e32 v6, v6
-; SI-NEXT: v_fma_f32 v1, -v6, v5, v1
+; SI-NEXT: v_fma_f32 v5, -v6, v5, v1
+; SI-NEXT: v_bfi_b32 v1, s6, v5, v1
; SI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0
; SI-NEXT: v_div_scale_f32 v6, s[4:5], v4, v4, v0
; SI-NEXT: v_rcp_f32_e32 v7, v6
@@ -3064,7 +3220,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_div_fmas_f32 v5, v5, v7, v8
; SI-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; SI-NEXT: v_trunc_f32_e32 v5, v5
-; SI-NEXT: v_fma_f32 v0, -v5, v4, v0
+; SI-NEXT: v_fma_f32 v4, -v5, v4, v0
+; SI-NEXT: v_bfi_b32 v0, s6, v4, v0
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -3085,6 +3242,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s11, s3
; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:64
+; CI-NEXT: s_brev_b32 s6, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_div_scale_f32 v9, s[4:5], v7, v7, v3
; CI-NEXT: v_div_scale_f32 v8, vcc, v3, v7, v3
@@ -3100,8 +3258,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v8, v8, v10, v11
; CI-NEXT: v_div_fixup_f32 v8, v8, v7, v3
; CI-NEXT: v_trunc_f32_e32 v8, v8
-; CI-NEXT: v_fma_f32 v3, -v8, v7, v3
+; CI-NEXT: v_fma_f32 v7, -v8, v7, v3
; CI-NEXT: v_div_scale_f32 v8, s[4:5], v6, v6, v2
+; CI-NEXT: v_bfi_b32 v3, s6, v7, v3
; CI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2
; CI-NEXT: v_rcp_f32_e32 v9, v8
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3115,8 +3274,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v7, v7, v9, v10
; CI-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; CI-NEXT: v_trunc_f32_e32 v7, v7
-; CI-NEXT: v_fma_f32 v2, -v7, v6, v2
+; CI-NEXT: v_fma_f32 v6, -v7, v6, v2
; CI-NEXT: v_div_scale_f32 v7, s[4:5], v5, v5, v1
+; CI-NEXT: v_bfi_b32 v2, s6, v6, v2
; CI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1
; CI-NEXT: v_rcp_f32_e32 v8, v7
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3130,8 +3290,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v6, v6, v8, v9
; CI-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; CI-NEXT: v_trunc_f32_e32 v6, v6
-; CI-NEXT: v_fma_f32 v1, -v6, v5, v1
+; CI-NEXT: v_fma_f32 v5, -v6, v5, v1
; CI-NEXT: v_div_scale_f32 v6, s[4:5], v4, v4, v0
+; CI-NEXT: v_bfi_b32 v1, s6, v5, v1
; CI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0
; CI-NEXT: v_rcp_f32_e32 v7, v6
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3145,7 +3306,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v5, v5, v7, v8
; CI-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; CI-NEXT: v_trunc_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v0, -v5, v4, v0
+; CI-NEXT: v_fma_f32 v4, -v5, v4, v0
+; CI-NEXT: v_bfi_b32 v0, s6, v4, v0
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
@@ -3153,6 +3315,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT: s_brev_b32 s2, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_add_u32 s0, s0, 64
@@ -3179,8 +3342,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v10, v10, v12, v13
; VI-NEXT: v_div_fixup_f32 v10, v10, v7, v3
; VI-NEXT: v_trunc_f32_e32 v10, v10
-; VI-NEXT: v_fma_f32 v3, -v10, v7, v3
+; VI-NEXT: v_fma_f32 v7, -v10, v7, v3
; VI-NEXT: v_div_scale_f32 v10, s[0:1], v6, v6, v2
+; VI-NEXT: v_bfi_b32 v3, s2, v7, v3
; VI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2
; VI-NEXT: v_rcp_f32_e32 v11, v10
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3194,8 +3358,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v7, v7, v11, v12
; VI-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; VI-NEXT: v_trunc_f32_e32 v7, v7
-; VI-NEXT: v_fma_f32 v2, -v7, v6, v2
+; VI-NEXT: v_fma_f32 v6, -v7, v6, v2
; VI-NEXT: v_div_scale_f32 v7, s[0:1], v5, v5, v1
+; VI-NEXT: v_bfi_b32 v2, s2, v6, v2
; VI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1
; VI-NEXT: v_rcp_f32_e32 v10, v7
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3209,8 +3374,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
; VI-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; VI-NEXT: v_trunc_f32_e32 v6, v6
-; VI-NEXT: v_fma_f32 v1, -v6, v5, v1
+; VI-NEXT: v_fma_f32 v5, -v6, v5, v1
; VI-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v0
+; VI-NEXT: v_bfi_b32 v1, s2, v5, v1
; VI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0
; VI-NEXT: v_rcp_f32_e32 v7, v6
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3224,7 +3390,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f32 v5, v5, v7, v10
; VI-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; VI-NEXT: v_trunc_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v0, -v5, v4, v0
+; VI-NEXT: v_fma_f32 v4, -v5, v4, v0
+; VI-NEXT: v_bfi_b32 v0, s2, v4, v0
; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; VI-NEXT: s_endpgm
;
@@ -3236,6 +3403,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
; GFX9-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3] offset:64
+; GFX9-NEXT: s_brev_b32 s2, -2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_div_scale_f32 v10, s[0:1], v7, v7, v3
; GFX9-NEXT: v_div_scale_f32 v9, vcc, v3, v7, v3
@@ -3251,8 +3419,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v9, v9, v11, v12
; GFX9-NEXT: v_div_fixup_f32 v9, v9, v7, v3
; GFX9-NEXT: v_trunc_f32_e32 v9, v9
-; GFX9-NEXT: v_fma_f32 v3, -v9, v7, v3
+; GFX9-NEXT: v_fma_f32 v7, -v9, v7, v3
; GFX9-NEXT: v_div_scale_f32 v9, s[0:1], v6, v6, v2
+; GFX9-NEXT: v_bfi_b32 v3, s2, v7, v3
; GFX9-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2
; GFX9-NEXT: v_rcp_f32_e32 v10, v9
; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3266,8 +3435,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v7, v7, v10, v11
; GFX9-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; GFX9-NEXT: v_trunc_f32_e32 v7, v7
-; GFX9-NEXT: v_fma_f32 v2, -v7, v6, v2
+; GFX9-NEXT: v_fma_f32 v6, -v7, v6, v2
; GFX9-NEXT: v_div_scale_f32 v7, s[0:1], v5, v5, v1
+; GFX9-NEXT: v_bfi_b32 v2, s2, v6, v2
; GFX9-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1
; GFX9-NEXT: v_rcp_f32_e32 v9, v7
; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3281,8 +3451,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v6, v6, v9, v10
; GFX9-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; GFX9-NEXT: v_trunc_f32_e32 v6, v6
-; GFX9-NEXT: v_fma_f32 v1, -v6, v5, v1
+; GFX9-NEXT: v_fma_f32 v5, -v6, v5, v1
; GFX9-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v0
+; GFX9-NEXT: v_bfi_b32 v1, s2, v5, v1
; GFX9-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0
; GFX9-NEXT: v_rcp_f32_e32 v7, v6
; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
@@ -3296,7 +3467,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f32 v5, v5, v7, v9
; GFX9-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_fma_f32 v0, -v5, v4, v0
+; GFX9-NEXT: v_fma_f32 v4, -v5, v4, v0
+; GFX9-NEXT: v_bfi_b32 v0, s2, v4, v0
; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
@@ -3325,10 +3497,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v9, v9, v11, v12
; GFX10-NEXT: v_div_fixup_f32 v9, v9, v7, v3
; GFX10-NEXT: v_trunc_f32_e32 v9, v9
-; GFX10-NEXT: v_fma_f32 v3, -v9, v7, v3
+; GFX10-NEXT: v_fma_f32 v7, -v9, v7, v3
; GFX10-NEXT: v_div_scale_f32 v9, s0, v6, v6, v2
-; GFX10-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
+; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3
; GFX10-NEXT: v_rcp_f32_e32 v10, v9
+; GFX10-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v11, -v9, v10, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v10, v11, v10
@@ -3340,10 +3513,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v7, v7, v10, v11
; GFX10-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; GFX10-NEXT: v_trunc_f32_e32 v7, v7
-; GFX10-NEXT: v_fma_f32 v2, -v7, v6, v2
+; GFX10-NEXT: v_fma_f32 v6, -v7, v6, v2
; GFX10-NEXT: v_div_scale_f32 v7, s0, v5, v5, v1
-; GFX10-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
+; GFX10-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2
; GFX10-NEXT: v_rcp_f32_e32 v9, v7
+; GFX10-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v10, -v7, v9, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v9, v10, v9
@@ -3355,10 +3529,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v6, v6, v9, v10
; GFX10-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; GFX10-NEXT: v_trunc_f32_e32 v6, v6
-; GFX10-NEXT: v_fma_f32 v1, -v6, v5, v1
+; GFX10-NEXT: v_fma_f32 v5, -v6, v5, v1
; GFX10-NEXT: v_div_scale_f32 v6, s0, v4, v4, v0
-; GFX10-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
+; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1
; GFX10-NEXT: v_rcp_f32_e32 v7, v6
+; GFX10-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v9, -v6, v7, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v7, v9, v7
@@ -3370,7 +3545,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f32 v5, v5, v7, v9
; GFX10-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; GFX10-NEXT: v_trunc_f32_e32 v5, v5
-; GFX10-NEXT: v_fma_f32 v0, -v5, v4, v0
+; GFX10-NEXT: v_fma_f32 v4, -v5, v4, v0
+; GFX10-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
@@ -3405,32 +3581,35 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_div_fixup_f32 v9, v9, v7, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v9, v9
-; GFX11-NEXT: v_fma_f32 v3, -v9, v7, v3
+; GFX11-NEXT: v_fma_f32 v7, -v9, v7, v3
; GFX11-NEXT: v_div_scale_f32 v9, null, v6, v6, v2
-; GFX11-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3
; GFX11-NEXT: v_rcp_f32_e32 v10, v9
+; GFX11-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; GFX11-NEXT: v_fmac_f32_e32 v10, v11, v10
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v10, v11, v10
; GFX11-NEXT: v_mul_f32_e32 v11, v7, v10
-; GFX11-NEXT: v_fma_f32 v12, -v9, v11, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v12, -v9, v11, v7
; GFX11-NEXT: v_fmac_f32_e32 v11, v12, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v7, -v9, v11, v7
; GFX11-NEXT: s_denorm_mode 12
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fmas_f32 v7, v7, v10, v11
-; GFX11-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v7, v7, v6, v2
; GFX11-NEXT: v_trunc_f32_e32 v7, v7
-; GFX11-NEXT: v_fma_f32 v2, -v7, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fma_f32 v6, -v7, v6, v2
; GFX11-NEXT: v_div_scale_f32 v7, null, v5, v5, v1
-; GFX11-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v9, v7
+; GFX11-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_f32 v10, -v7, v9, 1.0
@@ -3447,28 +3626,31 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_div_fixup_f32 v6, v6, v5, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v6, v6
-; GFX11-NEXT: v_fma_f32 v1, -v6, v5, v1
+; GFX11-NEXT: v_fma_f32 v5, -v6, v5, v1
; GFX11-NEXT: v_div_scale_f32 v6, null, v4, v4, v0
-; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1
; GFX11-NEXT: v_rcp_f32_e32 v7, v6
+; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_fma_f32 v9, -v6, v7, 1.0
-; GFX11-NEXT: v_fmac_f32_e32 v7, v9, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v7, v9, v7
; GFX11-NEXT: v_mul_f32_e32 v9, v5, v7
-; GFX11-NEXT: v_fma_f32 v10, -v6, v9, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v10, -v6, v9, v5
; GFX11-NEXT: v_fmac_f32_e32 v9, v10, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v5, -v6, v9, v5
; GFX11-NEXT: s_denorm_mode 12
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fmas_f32 v5, v5, v7, v9
-; GFX11-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v5, v5, v4, v0
; GFX11-NEXT: v_trunc_f32_e32 v5, v5
-; GFX11-NEXT: v_fma_f32 v0, -v5, v4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v4, -v5, v4, v0
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
; GFX11-NEXT: global_store_b128 v8, v[0:3], s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -3506,33 +3688,36 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: v_trunc_f32_e32 v9, v9
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v9, 0x80000000, v9
-; GFX1150-NEXT: v_fma_f32 v3, v9, v7, v3
+; GFX1150-NEXT: v_fma_f32 v7, v9, v7, v3
; GFX1150-NEXT: v_div_scale_f32 v9, null, v6, v6, v2
-; GFX1150-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3
; GFX1150-NEXT: v_rcp_f32_e32 v10, v9
+; GFX1150-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2
; GFX1150-NEXT: s_denorm_mode 15
+; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v10, v11, v10
-; GFX1150-NEXT: v_mul_f32_e32 v11, v7, v10
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_mul_f32_e32 v11, v7, v10
; GFX1150-NEXT: v_fma_f32 v12, -v9, v11, v7
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v11, v12, v10
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v7, -v9, v11, v7
; GFX1150-NEXT: s_denorm_mode 12
-; GFX1150-NEXT: v_div_fmas_f32 v7, v7, v10, v11
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_div_fmas_f32 v7, v7, v10, v11
; GFX1150-NEXT: v_div_fixup_f32 v7, v7, v6, v2
-; GFX1150-NEXT: v_trunc_f32_e32 v7, v7
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_trunc_f32_e32 v7, v7
; GFX1150-NEXT: v_xor_b32_e32 v7, 0x80000000, v7
-; GFX1150-NEXT: v_fma_f32 v2, v7, v6, v2
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_fma_f32 v6, v7, v6, v2
; GFX1150-NEXT: v_div_scale_f32 v7, null, v5, v5, v1
-; GFX1150-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
; GFX1150-NEXT: v_rcp_f32_e32 v9, v7
+; GFX1150-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1
; GFX1150-NEXT: s_denorm_mode 15
; GFX1150-NEXT: v_fma_f32 v10, -v7, v9, 1.0
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3550,29 +3735,32 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: v_trunc_f32_e32 v6, v6
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_xor_b32_e32 v6, 0x80000000, v6
-; GFX1150-NEXT: v_fma_f32 v1, v6, v5, v1
+; GFX1150-NEXT: v_fma_f32 v5, v6, v5, v1
; GFX1150-NEXT: v_div_scale_f32 v6, null, v4, v4, v0
-; GFX1150-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1
; GFX1150-NEXT: v_rcp_f32_e32 v7, v6
+; GFX1150-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0
; GFX1150-NEXT: s_denorm_mode 15
+; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v9, -v6, v7, 1.0
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v7, v9, v7
-; GFX1150-NEXT: v_mul_f32_e32 v9, v5, v7
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_mul_f32_e32 v9, v5, v7
; GFX1150-NEXT: v_fma_f32 v10, -v6, v9, v5
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fmac_f32_e32 v9, v10, v7
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f32 v5, -v6, v9, v5
; GFX1150-NEXT: s_denorm_mode 12
-; GFX1150-NEXT: v_div_fmas_f32 v5, v5, v7, v9
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_div_fmas_f32 v5, v5, v7, v9
; GFX1150-NEXT: v_div_fixup_f32 v5, v5, v4, v0
-; GFX1150-NEXT: v_trunc_f32_e32 v5, v5
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_trunc_f32_e32 v5, v5
; GFX1150-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1150-NEXT: v_fmac_f32_e32 v0, v5, v4
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_fma_f32 v4, v5, v4, v0
+; GFX1150-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
; GFX1150-NEXT: global_store_b128 v8, v[0:3], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -3640,43 +3828,46 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: v_mov_b32_e32 v11, s8
; SI-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
; SI-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; SI-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; SI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
-; SI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
-; SI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; SI-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; SI-NEXT: s_brev_b32 s8, -2
+; SI-NEXT: v_bfi_b32 v7, s8, v7, v3
+; SI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1]
+; SI-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
+; SI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; SI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; SI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; SI-NEXT: v_div_scale_f64 v[10:11], s[0:1], v[0:1], v[4:5], v[0:1]
; SI-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; SI-NEXT: v_fma_f64 v[14:15], -v[6:7], v[12:13], v[10:11]
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
+; SI-NEXT: v_fma_f64 v[14:15], -v[2:3], v[12:13], v[10:11]
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
; SI-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v11
; SI-NEXT: s_xor_b64 vcc, s[0:1], vcc
; SI-NEXT: s_nop 1
-; SI-NEXT: v_div_fmas_f64 v[6:7], v[14:15], v[8:9], v[12:13]
-; SI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s8, v7
-; SI-NEXT: s_bfe_u32 s0, s8, 0xb0014
-; SI-NEXT: s_add_i32 s9, s0, 0xfffffc01
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s9
+; SI-NEXT: v_div_fmas_f64 v[2:3], v[14:15], v[8:9], v[12:13]
+; SI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s9, v3
+; SI-NEXT: s_bfe_u32 s0, s9, 0xb0014
+; SI-NEXT: s_add_i32 s10, s0, 0xfffffc01
+; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s10
; SI-NEXT: v_not_b32_e32 v8, s0
-; SI-NEXT: v_and_b32_e32 v8, v6, v8
+; SI-NEXT: v_and_b32_e32 v8, v2, v8
; SI-NEXT: v_not_b32_e32 v9, s1
-; SI-NEXT: v_and_b32_e32 v7, v7, v9
-; SI-NEXT: s_and_b32 s0, s8, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s9, 0
+; SI-NEXT: v_and_b32_e32 v3, v3, v9
+; SI-NEXT: s_and_b32 s0, s9, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s10, 0
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
; SI-NEXT: v_mov_b32_e32 v9, s0
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT: s_cmp_gt_i32 s9, 51
+; SI-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
+; SI-NEXT: s_cmp_gt_i32 s10, 51
; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v9, s8
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT: v_mov_b32_e32 v9, s9
+; SI-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
+; SI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; SI-NEXT: v_bfi_b32 v5, s8, v5, v1
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; CI-LABEL: frem_v2f64:
@@ -3696,6 +3887,7 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s11, s3
; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:64
+; CI-NEXT: s_brev_b32 s6, -2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_div_scale_f64 v[8:9], s[4:5], v[6:7], v[6:7], v[2:3]
; CI-NEXT: v_rcp_f64_e32 v[10:11], v[8:9]
@@ -3710,28 +3902,31 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
; CI-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
; CI-NEXT: v_trunc_f64_e32 v[8:9], v[8:9]
-; CI-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; CI-NEXT: v_div_scale_f64 v[6:7], s[4:5], v[4:5], v[4:5], v[0:1]
-; CI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
-; CI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; CI-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; CI-NEXT: v_bfi_b32 v7, s6, v7, v3
+; CI-NEXT: v_div_scale_f64 v[2:3], s[4:5], v[4:5], v[4:5], v[0:1]
+; CI-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
+; CI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; CI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; CI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; CI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; CI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; CI-NEXT: v_div_scale_f64 v[10:11], vcc, v[0:1], v[4:5], v[0:1]
; CI-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; CI-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
+; CI-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11]
; CI-NEXT: s_nop 1
-; CI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; CI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; CI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; CI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; CI-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13]
+; CI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
+; CI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v1
+; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: frem_v2f64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT: s_brev_b32 s2, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_add_u32 s0, s0, 64
@@ -3757,22 +3952,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_div_fmas_f64 v[10:11], v[10:11], v[12:13], v[16:17]
; VI-NEXT: v_div_fixup_f64 v[10:11], v[10:11], v[6:7], v[2:3]
; VI-NEXT: v_trunc_f64_e32 v[10:11], v[10:11]
-; VI-NEXT: v_fma_f64 v[2:3], -v[10:11], v[6:7], v[2:3]
-; VI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
-; VI-NEXT: v_rcp_f64_e32 v[10:11], v[6:7]
-; VI-NEXT: v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0
+; VI-NEXT: v_fma_f64 v[6:7], -v[10:11], v[6:7], v[2:3]
+; VI-NEXT: v_bfi_b32 v7, s2, v7, v3
+; VI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1]
+; VI-NEXT: v_rcp_f64_e32 v[10:11], v[2:3]
+; VI-NEXT: v_fma_f64 v[12:13], -v[2:3], v[10:11], 1.0
; VI-NEXT: v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
-; VI-NEXT: v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0
+; VI-NEXT: v_fma_f64 v[12:13], -v[2:3], v[10:11], 1.0
; VI-NEXT: v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
; VI-NEXT: v_div_scale_f64 v[12:13], vcc, v[0:1], v[4:5], v[0:1]
; VI-NEXT: v_mul_f64 v[14:15], v[12:13], v[10:11]
-; VI-NEXT: v_fma_f64 v[6:7], -v[6:7], v[14:15], v[12:13]
+; VI-NEXT: v_fma_f64 v[2:3], -v[2:3], v[14:15], v[12:13]
; VI-NEXT: s_nop 1
-; VI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[14:15]
-; VI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; VI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; VI-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[10:11], v[14:15]
+; VI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; VI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
+; VI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; VI-NEXT: v_bfi_b32 v5, s2, v5, v1
+; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: frem_v2f64:
@@ -3783,6 +3980,7 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
; GFX9-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:64
+; GFX9-NEXT: s_brev_b32 s2, -2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_div_scale_f64 v[8:9], s[0:1], v[6:7], v[6:7], v[2:3]
; GFX9-NEXT: v_rcp_f64_e32 v[10:11], v[8:9]
@@ -3797,22 +3995,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
; GFX9-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
; GFX9-NEXT: v_trunc_f64_e32 v[8:9], v[8:9]
-; GFX9-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; GFX9-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
-; GFX9-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
-; GFX9-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX9-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; GFX9-NEXT: v_bfi_b32 v7, s2, v7, v3
+; GFX9-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1]
+; GFX9-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
+; GFX9-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; GFX9-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; GFX9-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX9-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; GFX9-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; GFX9-NEXT: v_div_scale_f64 v[10:11], vcc, v[0:1], v[4:5], v[0:1]
; GFX9-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; GFX9-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
+; GFX9-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11]
; GFX9-NEXT: s_nop 1
-; GFX9-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; GFX9-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; GFX9-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; GFX9-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; GFX9-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5]
+; GFX9-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13]
+; GFX9-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; GFX9-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
+; GFX9-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; GFX9-NEXT: v_bfi_b32 v5, s2, v5, v1
+; GFX9-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: frem_v2f64:
@@ -3838,21 +4038,23 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
; GFX10-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
; GFX10-NEXT: v_trunc_f64_e32 v[8:9], v[8:9]
-; GFX10-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; GFX10-NEXT: v_div_scale_f64 v[6:7], s0, v[4:5], v[4:5], v[0:1]
-; GFX10-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
-; GFX10-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX10-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; GFX10-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3
+; GFX10-NEXT: v_div_scale_f64 v[2:3], s0, v[4:5], v[4:5], v[0:1]
+; GFX10-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
+; GFX10-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; GFX10-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
-; GFX10-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX10-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; GFX10-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; GFX10-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1]
; GFX10-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; GFX10-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
-; GFX10-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; GFX10-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; GFX10-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; GFX10-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; GFX10-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5]
+; GFX10-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11]
+; GFX10-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13]
+; GFX10-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; GFX10-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
+; GFX10-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; GFX10-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1
+; GFX10-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: frem_v2f64:
@@ -3884,27 +4086,30 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX11-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f64_e32 v[8:9], v[8:9]
-; GFX11-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; GFX11-NEXT: v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
+; GFX11-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3
+; GFX11-NEXT: v_div_scale_f64 v[2:3], null, v[4:5], v[4:5], v[0:1]
+; GFX11-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
; GFX11-NEXT: s_waitcnt_depctr 0xfff
-; GFX11-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
-; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
+; GFX11-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
+; GFX11-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; GFX11-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; GFX11-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; GFX11-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
+; GFX11-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11]
+; GFX11-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; GFX11-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; GFX11-NEXT: global_store_b128 v16, v[0:3], s[4:5]
+; GFX11-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; GFX11-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; GFX11-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1
+; GFX11-NEXT: global_store_b128 v16, v[4:7], s[4:5]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
@@ -3937,27 +4142,30 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
; GFX1150-NEXT: v_trunc_f64_e32 v[8:9], v[8:9]
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; GFX1150-NEXT: v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1]
-; GFX1150-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
-; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
-; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX1150-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3]
+; GFX1150-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3
+; GFX1150-NEXT: v_div_scale_f64 v[2:3], null, v[4:5], v[4:5], v[0:1]
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-NEXT: v_rcp_f64_e32 v[8:9], v[2:3]
+; GFX1150-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
+; GFX1150-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
; GFX1150-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1]
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
-; GFX1150-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
-; GFX1150-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
+; GFX1150-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11]
+; GFX1150-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13]
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1]
+; GFX1150-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT: v_trunc_f64_e32 v[6:7], v[6:7]
-; GFX1150-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
-; GFX1150-NEXT: global_store_b128 v16, v[0:3], s[4:5]
+; GFX1150-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1]
+; GFX1150-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1
+; GFX1150-NEXT: global_store_b128 v16, v[4:7], s[4:5]
; GFX1150-NEXT: s_nop 0
; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX1150-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index cadc23414dcac11..5fad26ea16d13f7 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -2584,7 +2584,8 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) {
; GFX1032-NEXT: v_div_fmas_f32 v1, v1, v2, v3
; GFX1032-NEXT: v_div_fixup_f32 v1, v1, s0, v0
; GFX1032-NEXT: v_trunc_f32_e32 v1, v1
-; GFX1032-NEXT: v_fma_f32 v0, -v1, s0, v0
+; GFX1032-NEXT: v_fma_f32 v1, -v1, s0, v0
+; GFX1032-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0
; GFX1032-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1
; GFX1032-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0
@@ -2616,7 +2617,8 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) {
; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v3
; GFX1064-NEXT: v_div_fixup_f32 v1, v1, s2, v0
; GFX1064-NEXT: v_trunc_f32_e32 v1, v1
-; GFX1064-NEXT: v_fma_f32 v0, -v1, s2, v0
+; GFX1064-NEXT: v_fma_f32 v1, -v1, s2, v0
+; GFX1064-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0
; GFX1064-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
; GFX1064-NEXT: s_lshr_b64 s[0:1], vcc, 1
; GFX1064-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
@@ -2773,7 +2775,8 @@ define amdgpu_kernel void @fcmp32(float %n, float %s) {
; GFX1032-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX1032-NEXT: v_div_fixup_f32 v1, v1, s0, v0
; GFX1032-NEXT: v_trunc_f32_e32 v1, v1
-; GFX1032-NEXT: v_fma_f32 v0, -v1, s0, v0
+; GFX1032-NEXT: v_fma_f32 v1, -v1, s0, v0
+; GFX1032-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0
; GFX1032-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1
; GFX1032-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0
@@ -2805,7 +2808,8 @@ define amdgpu_kernel void @fcmp32(float %n, float %s) {
; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX1064-NEXT: v_div_fixup_f32 v1, v1, s2, v0
; GFX1064-NEXT: v_trunc_f32_e32 v1, v1
-; GFX1064-NEXT: v_fma_f32 v0, -v1, s2, v0
+; GFX1064-NEXT: v_fma_f32 v1, -v1, s2, v0
+; GFX1064-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0
; GFX1064-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
; GFX1064-NEXT: s_lshr_b32 s0, vcc_lo, 1
; GFX1064-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
>From 7b6faba8094c50926472fc95a8d1a7e3bc5bad5b Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 27 Oct 2023 13:52:48 +0100
Subject: [PATCH 2/2] clang-format
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 2 +-
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 28f6ad62f2c37ed..9564c191b0d15c8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -2283,7 +2283,7 @@ SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
// TODO: For f32 use FMAD instead if !hasFastFMA32?
SDValue FMA = DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
- return NSZ ? FMA : DAG.getNode(ISD::FCOPYSIGN , SL, VT, FMA, X);
+ return NSZ ? FMA : DAG.getNode(ISD::FCOPYSIGN, SL, VT, FMA, X);
}
SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0775d8a3c2396f7..42d8721cd323edc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2343,7 +2343,7 @@ bool AMDGPULegalizerInfo::legalizeFceil(
static bool allowNoSignedZeros(const MachineFunction &MF, unsigned Flags) {
return (Flags & MachineInstr::FmNsz) ||
- MF.getTarget().Options.NoSignedZerosFPMath;
+ MF.getTarget().Options.NoSignedZerosFPMath;
}
// Legalize frem(x, y) -> copysign(x - y * trunc(x / y), x)
More information about the llvm-commits
mailing list