[llvm] [AMDGPU] Fix legalization of frem(-0.0, y) (PR #70448)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 27 05:16:04 PDT 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Jay Foad (jayfoad)
<details>
<summary>Changes</summary>
We legalized frem(x, y) -> x - y * trunc(x / y). When x is -0.0 this
evaluates to +0.0 but the result should be -0.0.
Fix this by legalizing to copysign(x - y * trunc(x / y), x).
---
Patch is 139.68 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/70448.diff
5 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (+6-2)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+13-1)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll (+164-21)
- (modified) llvm/test/CodeGen/AMDGPU/frem.ll (+544-336)
- (modified) llvm/test/CodeGen/AMDGPU/wave32.ll (+8-4)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index adf4e0139e03c1d..28f6ad62f2c37ed 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -2267,19 +2267,23 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
return DAG.getMergeValues(Res, DL);
}
-// (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x)
+// (frem x, y) -> (fcopysign (fma (fneg (ftrunc (fdiv x, y))), y, x), x)
+// The fcopysign is only required to get the correct result -0.0 when x is -0.0
+// (and y is non-zero). With NSZ it can be dropped.
SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
EVT VT = Op.getValueType();
auto Flags = Op->getFlags();
SDValue X = Op.getOperand(0);
SDValue Y = Op.getOperand(1);
+ bool NSZ = mayIgnoreSignedZero(Op);
SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags);
SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags);
SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
// TODO: For f32 use FMAD instead if !hasFastFMA32?
- return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
+ SDValue FMA = DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
+ return NSZ ? FMA : DAG.getNode(ISD::FCOPYSIGN , SL, VT, FMA, X);
}
SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 3d70ed150df12f8..0775d8a3c2396f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2341,6 +2341,14 @@ bool AMDGPULegalizerInfo::legalizeFceil(
return true;
}
+static bool allowNoSignedZeros(const MachineFunction &MF, unsigned Flags) {
+ return (Flags & MachineInstr::FmNsz) ||
+ MF.getTarget().Options.NoSignedZerosFPMath;
+}
+
+// Legalize frem(x, y) -> copysign(x - y * trunc(x / y), x)
+// The copysign is only required to get the correct result -0.0 when x is -0.0
+// (and y is non-zero). With NSZ it can be dropped.
bool AMDGPULegalizerInfo::legalizeFrem(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
@@ -2349,11 +2357,15 @@ bool AMDGPULegalizerInfo::legalizeFrem(
Register Src1Reg = MI.getOperand(2).getReg();
auto Flags = MI.getFlags();
LLT Ty = MRI.getType(DstReg);
+ bool NSZ = allowNoSignedZeros(B.getMF(), Flags);
auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
auto Neg = B.buildFNeg(Ty, Trunc, Flags);
- B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
+ DstOp FMADst = NSZ ? DstOp(DstReg) : DstOp(Ty);
+ auto FMA = B.buildFMA(FMADst, Neg, Src1Reg, Src0Reg, Flags);
+ if (!NSZ)
+ B.buildFCopysign(DstReg, FMA, Src0Reg);
MI.eraseFromParent();
return true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
index 0417b97a53c0ff3..da7fca95d5eef07 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
@@ -29,7 +29,10 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
@@ -46,12 +49,15 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
; VI-NEXT: v_cvt_f32_f16_e32 v2, s0
; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: v_mul_f32_e32 v0, v0, v2
; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
; VI-NEXT: v_div_fixup_f16 v0, v0, v1, s2
; VI-NEXT: v_trunc_f16_e32 v0, v0
-; VI-NEXT: v_fma_f16 v2, -v0, v1, s2
+; VI-NEXT: v_fma_f16 v0, -v0, v1, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_short v[0:1], v2
@@ -126,7 +132,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_rcp_f32_e32 v2, v1
; CI-NEXT: v_mul_f32_e32 v2, v0, v2
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
@@ -143,7 +152,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_rcp_f16_e32 v0, s0
; VI-NEXT: v_mul_f16_e32 v0, s2, v0
; VI-NEXT: v_trunc_f16_e32 v0, v0
-; VI-NEXT: v_fma_f16 v2, -v0, s0, v1
+; VI-NEXT: v_fma_f16 v0, -v0, s0, v1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_short v[0:1], v2
@@ -178,11 +190,14 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; CI-NEXT: v_trunc_f32_e32 v1, v1
; CI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -207,9 +222,12 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; VI-NEXT: v_trunc_f32_e32 v1, v1
-; VI-NEXT: v_fma_f32 v2, -v1, v0, s2
+; VI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -282,6 +300,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_mul_f32_e32 v0, s2, v0
; CI-NEXT: v_trunc_f32_e32 v0, v0
; CI-NEXT: v_fma_f32 v0, -v0, s0, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -297,7 +318,10 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_rcp_f32_e32 v0, s0
; VI-NEXT: v_mul_f32_e32 v0, s2, v0
; VI-NEXT: v_trunc_f32_e32 v0, v0
-; VI-NEXT: v_fma_f32 v2, -v0, s0, v1
+; VI-NEXT: v_fma_f32 v0, -v0, s0, v1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: v_or_b32_e32 v2, s0, v0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -325,6 +349,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3]
; CI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3]
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_brev_b32 s1, 1
+; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; CI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -336,6 +363,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[0:1], s[2:3]
; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3]
; CI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3]
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
+; CI-NEXT: v_or_b32_e32 v1, s1, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -351,6 +381,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3]
; VI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3]
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_brev_b32 s1, 1
+; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; VI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3]
; VI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
; VI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
@@ -364,6 +397,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3]
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_or_b32_e32 v1, s1, v1
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
%r0 = load double, ptr addrspace(1) %in1, align 8
@@ -453,6 +489,12 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
; CI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; CI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3]
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_brev_b32 s1, 1
+; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
+; CI-NEXT: v_or_b32_e32 v1, s1, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -476,8 +518,14 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5]
; VI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; VI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3]
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_brev_b32 s1, 1
+; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_or_b32_e32 v1, s1, v1
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
ptr addrspace(1) %in2) #1 {
@@ -515,7 +563,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f32_f16_e32 v1, s3
; CI-NEXT: v_cvt_f32_f16_e32 v2, s6
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -536,7 +587,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v1, -v3, v2, v1
+; CI-NEXT: v_fma_f32 v2, -v3, v2, v1
+; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; CI-NEXT: v_or_b32_e32 v1, v2, v1
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; CI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -559,6 +613,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_rcp_f32_e32 v2, v2
; VI-NEXT: s_lshr_b32 s1, s2, 16
; VI-NEXT: v_rcp_f32_e32 v3, v3
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v0, v0, v2
; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
; VI-NEXT: v_mov_b32_e32 v2, s3
@@ -566,13 +621,19 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_trunc_f16_e32 v0, v0
; VI-NEXT: v_fma_f16 v0, -v0, v1, s2
; VI-NEXT: v_cvt_f32_f16_e32 v1, s1
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: s_and_b32 s0, s1, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v1, v1, v3
; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s1
; VI-NEXT: v_trunc_f16_e32 v1, v1
; VI-NEXT: v_fma_f16 v1, -v1, v2, s1
+; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_or_b32_e32 v2, v0, v1
+; VI-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -614,7 +675,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0
; CI-NEXT: v_trunc_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v0, -v2, v1, v0
+; CI-NEXT: v_fma_f32 v1, -v2, v1, v0
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; CI-NEXT: v_or_b32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f32_f16_e32 v1, s8
; CI-NEXT: v_cvt_f32_f16_e32 v2, s10
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -633,7 +697,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6
; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1
; CI-NEXT: v_trunc_f32_e32 v3, v3
-; CI-NEXT: v_fma_f32 v1, -v3, v2, v1
+; CI-NEXT: v_fma_f32 v2, -v3, v2, v1
+; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; CI-NEXT: v_or_b32_e32 v1, v2, v1
; CI-NEXT: v_cvt_f32_f16_e32 v2, s3
; CI-NEXT: v_cvt_f32_f16_e32 v3, s1
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
@@ -651,7 +718,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7
; CI-NEXT: v_div_fixup_f32 v4, v4, v3, v2
; CI-NEXT: v_trunc_f32_e32 v4, v4
-; CI-NEXT: v_fma_f32 v2, -v4, v3, v2
+; CI-NEXT: v_fma_f32 v3, -v4, v3, v2
+; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CI-NEXT: v_and_b32_e32 v2, 0x80000000, v2
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_cvt_f32_f16_e32 v3, s9
; CI-NEXT: v_cvt_f32_f16_e32 v4, s11
; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -673,7 +743,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v5, v5, v4, v3
; CI-NEXT: v_trunc_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v3, -v5, v4, v3
+; CI-NEXT: v_fma_f32 v4, -v5, v4, v3
+; CI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v4
+; CI-NEXT: v_and_b32_e32 v3, 0x80000000, v3
+; CI-NEXT: v_or_b32_e32 v3, v4, v3
; CI-NEXT: v_cvt_f16_f32_e32 v3, v3
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; CI-NEXT: v_or_b32_e32 v1, v2, v1
@@ -708,30 +781,44 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_cvt_f32_f16_e32 v1, s6
; VI-NEXT: v_cvt_f32_f16_e32 v5, s9
; VI-NEXT: s_lshr_b32 s7, s3, 16
+; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; VI-NEXT: v_mul_f32_e32 v1, v1, v3
; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_rcp_f32_e32 v5, v5
+; VI-NEXT: s_and_b32 s0, s2, 0xffff8000
; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s6
; VI-NEXT: v_trunc_f16_e32 v1, v1
; VI-NEXT: v_fma_f16 v1, -v1, v2, s6
; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
+; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; VI-NEXT: s_and_b32 s0, s6, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v2, v2, v4
; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
; VI-NEXT: v_mov_b32_e32 v4, s9
+; VI-NEXT: v_or_b32_e32 v1, s0, v1
+; VI-NEXT: s_and_b32 s0, s3, 0xffff8000
; VI-NEXT: v_div_fixup_f16 v2, v2, v3, s3
; VI-NEXT: v_trunc_f16_e32 v2, v2
; VI-NEXT: v_fma_f16 v2, -v2, v3, s3
; VI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; VI-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; VI-NEXT: v_or_b32_e32 v2, s0, v2
+; VI-NEXT: s_and_b32 s0, s7, 0xffff8000
; VI-NEXT: v_mul_f32_e32 v3, v3, v5
; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_div_fixup_f16 v3, v3, v4, s7
; VI-NEXT: v_trunc_f16_e32 v3, v3
; VI-NEXT: v_fma_f16 v3, -v3, v4, s7
-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-NEXT: v_and_b32_e32 v3, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v3, s0, v3
+; VI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
@@ -766,10 +853,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_and_b32 s0, s2, 0x80000000
; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; CI-NEXT: v_trunc_f32_e32 v1, v1
; CI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3
; CI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3
; CI-NEXT: v_rcp_f32_e32 v4, v2
@@ -782,11 +872,14 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: v_fma_f32 v2, -v2, v5, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; CI-NEXT: s_and_b32 s0, s3, 0x80000000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: v_div_fixup_f32 v2, v2, v1, s3
; CI-NEXT: v_trunc_f32_e32 v2, v2
; CI-NEXT: v_fma_f32 v1, -v2, v1, s3
+; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; CI-NEXT: v_or_b32_e32 v1, s0, v1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
;
@@ -811,10 +904,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2
; VI-NEXT: v_trunc_f32_e32 v1, v1
; VI-NEXT: v_fma_f32 v0, -v1, v0, s2
+; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v0, s0, v0
; VI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3
; VI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3
; VI-NEXT: v_rcp_f32_e32 v4, v2
@@ -827,10 +923,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: v_fma_f32 v2, -v2, v5, v3
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/70448
More information about the llvm-commits
mailing list