[PATCH] R600/SI: Allow using f32 rcp / rsq when denormals not handled.

Tom Stellard tom at stellard.net
Tue Jul 15 16:06:17 PDT 2014 

On Tue, Jul 15, 2014 at 08:22:04PM +0000, Matt Arsenault wrote:
> These are precise enough to use for OpenCL unless denormals are handled.
> 

LGTM.

> http://reviews.llvm.org/D4526
> 
> Files:
>   lib/Target/R600/SIISelLowering.cpp
>   lib/Target/R600/SIISelLowering.h
>   test/CodeGen/R600/llvm.AMDGPU.rcp.ll
>   test/CodeGen/R600/rsq.ll

> Index: lib/Target/R600/SIISelLowering.cpp
> ===================================================================
> --- lib/Target/R600/SIISelLowering.cpp
> +++ lib/Target/R600/SIISelLowering.cpp
> @@ -934,16 +934,27 @@
>    return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Res);
>  }
>  
> -static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) {
> +// Catch division cases where we can use shortcuts with rcp and rsq
> +// instructions.
> +SDValue SITargetLowering::LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const {
>    SDLoc SL(Op);
>    SDValue LHS = Op.getOperand(0);
>    SDValue RHS = Op.getOperand(1);
>    EVT VT = Op.getValueType();
> +  bool Unsafe = DAG.getTarget().Options.UnsafeFPMath;
>  
>    if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) {
> -    if (CLHS->isExactlyValue(1.0)) {
> +    if ((Unsafe || (VT == MVT::f32 && !Subtarget->hasFP32Denormals())) &&
> +        CLHS->isExactlyValue(1.0)) {
> +      // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
> +      // the CI documentation has a worst case error of 1 ulp.
> +      // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
> +      // use it as long as we aren't trying to use denormals.
>  
>        // 1.0 / sqrt(x) -> rsq(x)
> +      //
> +      // XXX - Is UnsafeFPMath sufficient to do this for f64? The maximum ULP
> +      // error seems really high at 2^29 ULP.
>        if (RHS.getOpcode() == ISD::FSQRT)
>          return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0));
>  
> @@ -952,15 +963,25 @@
>      }
>    }
>  
> -  // Turn into multiply by the reciprocal
> -  // x / y -> x * (1.0 / y)
> -  SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
> -  return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
> +  if (Unsafe) {
> +    // Turn into multiply by the reciprocal.
> +    // x / y -> x * (1.0 / y)
> +    SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
> +    return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
> +  }
> +
> +  return SDValue();
>  }
>  
>  SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
> -  if (DAG.getTarget().Options.UnsafeFPMath)
> -    return performUnsafeFDIV(Op, DAG);
> +  SDValue FastLowered = LowerFastFDIV(Op, DAG);
> +  if (FastLowered.getNode())
> +    return FastLowered;
> +
> +  // This uses v_rcp_f32 which does not handle denormals. Let this hit a
> +  // selection error for now rather than do something incorrect.
> +  if (Subtarget->hasFP32Denormals())
> +    return SDValue();
>  
>    SDLoc SL(Op);
>    SDValue LHS = Op.getOperand(0);
> @@ -992,8 +1013,9 @@
>  }
>  
>  SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
> -  if (DAG.getTarget().Options.UnsafeFPMath)
> -    return performUnsafeFDIV(Op, DAG);
> +  SDValue FastLowered = LowerFastFDIV(Op, DAG);
> +  if (FastLowered.getNode())
> +    return FastLowered;
>  
>    SDLoc SL(Op);
>    SDValue X = Op.getOperand(0);
> Index: lib/Target/R600/SIISelLowering.h
> ===================================================================
> --- lib/Target/R600/SIISelLowering.h
> +++ lib/Target/R600/SIISelLowering.h
> @@ -27,6 +27,7 @@
>                                 SelectionDAG &DAG) const;
>    SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
>    SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
> +  SDValue LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const;
>    SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
>    SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
>    SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
> Index: test/CodeGen/R600/llvm.AMDGPU.rcp.ll
> ===================================================================
> --- test/CodeGen/R600/llvm.AMDGPU.rcp.ll
> +++ test/CodeGen/R600/llvm.AMDGPU.rcp.ll
> @@ -1,5 +1,7 @@
> -; RUN: llc -march=r600 -mcpu=SI -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
> -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
> +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
> +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
> +
> +; XUN: llc -march=r600 -mcpu=SI -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE-SPDENORM -check-prefix=SI -check-prefix=FUNC %s
>  
>  declare float @llvm.AMDGPU.rcp.f32(float) nounwind readnone
>  declare double @llvm.AMDGPU.rcp.f64(double) nounwind readnone
> @@ -25,15 +27,8 @@
>  }
>  
>  ; FUNC-LABEL: @rcp_pat_f32
> -; SI-UNSAFE-NOT: V_MUL_F32
> -; SI-UNSAFE: V_RCP_F32_e32
> -; SI-UNSAFE-NOT: V_MUL_F32
> -
> -; Check for surrounding multiplies the correct divide has.
> -; SI-SAFE: V_MUL_F32
>  ; SI-SAFE: V_RCP_F32_e32
> -; SI-SAFE: V_MUL_F32
> -
> +; XSI-SAFE-SPDENORM-NOT: V_RCP_F32_e32
>  define void @rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind {
>    %rcp = fdiv float 1.0, %src
>    store float %rcp, float addrspace(1)* %out, align 4
> Index: test/CodeGen/R600/rsq.ll
> ===================================================================
> --- test/CodeGen/R600/rsq.ll
> +++ test/CodeGen/R600/rsq.ll
> @@ -1,12 +1,11 @@
> -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
> -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
> +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
> +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
>  
>  declare float @llvm.sqrt.f32(float) nounwind readnone
>  declare double @llvm.sqrt.f64(double) nounwind readnone
>  
>  ; SI-LABEL: @rsq_f32
> -; SI-UNSAFE: V_RSQ_F32_e32
> -; SI-SAFE: V_SQRT_F32
> +; SI: V_RSQ_F32_e32
>  ; SI: S_ENDPGM
>  define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
>    %val = load float addrspace(1)* %in, align 4
> @@ -17,7 +16,8 @@
>  }
>  
>  ; SI-LABEL: @rsq_f64
> -; SI: V_RSQ_F64_e32
> +; SI-UNSAFE: V_RSQ_F64_e32
> +; SI-SAFE: V_SQRT_F64_e32
>  ; SI: S_ENDPGM
>  define void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
>    %val = load double addrspace(1)* %in, align 4

> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits

More information about the llvm-commits mailing list