[llvm] r213107 - R600/SI: Allow using f32 rcp / rsq when denormals not handled.
Matt Arsenault
Matthew.Arsenault at amd.com
Tue Jul 15 16:50:19 PDT 2014
Author: arsenm
Date: Tue Jul 15 18:50:10 2014
New Revision: 213107
URL: http://llvm.org/viewvc/llvm-project?rev=213107&view=rev
Log:
R600/SI: Allow using f32 rcp / rsq when denormals not handled.
These are precise enough to use for OpenCL unless denormals
are handled.
Modified:
llvm/trunk/lib/Target/R600/SIISelLowering.cpp
llvm/trunk/lib/Target/R600/SIISelLowering.h
llvm/trunk/lib/Target/R600/SIInstructions.td
llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.ll
llvm/trunk/test/CodeGen/R600/rsq.ll
Modified: llvm/trunk/lib/Target/R600/SIISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIISelLowering.cpp?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/R600/SIISelLowering.cpp Tue Jul 15 18:50:10 2014
@@ -936,16 +936,27 @@ SDValue SITargetLowering::LowerSELECT(SD
return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Res);
}
-static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) {
+// Catch division cases where we can use shortcuts with rcp and rsq
+// instructions.
+SDValue SITargetLowering::LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
EVT VT = Op.getValueType();
+ bool Unsafe = DAG.getTarget().Options.UnsafeFPMath;
if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) {
- if (CLHS->isExactlyValue(1.0)) {
+ if ((Unsafe || (VT == MVT::f32 && !Subtarget->hasFP32Denormals())) &&
+ CLHS->isExactlyValue(1.0)) {
+ // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
+ // the CI documentation has a worst case error of 1 ulp.
+ // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
+ // use it as long as we aren't trying to use denormals.
// 1.0 / sqrt(x) -> rsq(x)
+ //
+ // XXX - Is UnsafeFPMath sufficient to do this for f64? The maximum ULP
+ // error seems really high at 2^29 ULP.
if (RHS.getOpcode() == ISD::FSQRT)
return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0));
@@ -954,15 +965,25 @@ static SDValue performUnsafeFDIV(SDValue
}
}
- // Turn into multiply by the reciprocal
- // x / y -> x * (1.0 / y)
- SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
- return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
+ if (Unsafe) {
+ // Turn into multiply by the reciprocal.
+ // x / y -> x * (1.0 / y)
+ SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
+ return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
+ }
+
+ return SDValue();
}
SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
- if (DAG.getTarget().Options.UnsafeFPMath)
- return performUnsafeFDIV(Op, DAG);
+ SDValue FastLowered = LowerFastFDIV(Op, DAG);
+ if (FastLowered.getNode())
+ return FastLowered;
+
+ // This uses v_rcp_f32 which does not handle denormals. Let this hit a
+ // selection error for now rather than do something incorrect.
+ if (Subtarget->hasFP32Denormals())
+ return SDValue();
SDLoc SL(Op);
SDValue LHS = Op.getOperand(0);
Modified: llvm/trunk/lib/Target/R600/SIISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIISelLowering.h?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIISelLowering.h (original)
+++ llvm/trunk/lib/Target/R600/SIISelLowering.h Tue Jul 15 18:50:10 2014
@@ -27,6 +27,7 @@ class SITargetLowering : public AMDGPUTa
SelectionDAG &DAG) const;
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
Modified: llvm/trunk/lib/Target/R600/SIInstructions.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/SIInstructions.td?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/SIInstructions.td (original)
+++ llvm/trunk/lib/Target/R600/SIInstructions.td Tue Jul 15 18:50:10 2014
@@ -1800,10 +1800,9 @@ def : Pat <
// VOP1 Patterns
//===----------------------------------------------------------------------===//
+let Predicates = [UnsafeFPMath] in {
def : RcpPat<V_RCP_F64_e32, f64>;
defm : RsqPat<V_RSQ_F64_e32, f64>;
-
-let Predicates = [UnsafeFPMath] in {
defm : RsqPat<V_RSQ_F32_e32, f32>;
}
Modified: llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll (original)
+++ llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll Tue Jul 15 18:50:10 2014
@@ -20,7 +20,8 @@ define void @rcp_pat_f64(double addrspac
}
; FUNC-LABEL: @rsq_rcp_pat_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE-NOT: V_RSQ_F64_e32
define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind {
%sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone
%rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone
Modified: llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.ll?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.ll (original)
+++ llvm/trunk/test/CodeGen/R600/llvm.AMDGPU.rcp.ll Tue Jul 15 18:50:10 2014
@@ -1,5 +1,7 @@
-; RUN: llc -march=r600 -mcpu=SI -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+
+; XUN: llc -march=r600 -mcpu=SI -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE-SPDENORM -check-prefix=SI -check-prefix=FUNC %s
declare float @llvm.AMDGPU.rcp.f32(float) nounwind readnone
declare double @llvm.AMDGPU.rcp.f64(double) nounwind readnone
@@ -25,15 +27,8 @@ define void @rcp_f64(double addrspace(1)
}
; FUNC-LABEL: @rcp_pat_f32
-; SI-UNSAFE-NOT: V_MUL_F32
-; SI-UNSAFE: V_RCP_F32_e32
-; SI-UNSAFE-NOT: V_MUL_F32
-
-; Check for surrounding multiplies the correct divide has.
-; SI-SAFE: V_MUL_F32
; SI-SAFE: V_RCP_F32_e32
-; SI-SAFE: V_MUL_F32
-
+; XSI-SAFE-SPDENORM-NOT: V_RCP_F32_e32
define void @rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind {
%rcp = fdiv float 1.0, %src
store float %rcp, float addrspace(1)* %out, align 4
@@ -60,7 +55,8 @@ define void @rsq_rcp_pat_f32(float addrs
}
; FUNC-LABEL: @rsq_rcp_pat_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE-NOT: V_RSQ_F64_e32
define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind {
%sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone
%rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone
Modified: llvm/trunk/test/CodeGen/R600/rsq.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/R600/rsq.ll?rev=213107&r1=213106&r2=213107&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/R600/rsq.ll (original)
+++ llvm/trunk/test/CodeGen/R600/rsq.ll Tue Jul 15 18:50:10 2014
@@ -1,12 +1,11 @@
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
declare float @llvm.sqrt.f32(float) nounwind readnone
declare double @llvm.sqrt.f64(double) nounwind readnone
; SI-LABEL: @rsq_f32
-; SI-UNSAFE: V_RSQ_F32_e32
-; SI-SAFE: V_SQRT_F32
+; SI: V_RSQ_F32_e32
; SI: S_ENDPGM
define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
%val = load float addrspace(1)* %in, align 4
@@ -17,7 +16,8 @@ define void @rsq_f32(float addrspace(1)*
}
; SI-LABEL: @rsq_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE: V_SQRT_F64_e32
; SI: S_ENDPGM
define void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
%val = load double addrspace(1)* %in, align 4
More information about the llvm-commits
mailing list