[llvm] r284513 - revert r284495: [Target] remove TargetRecip class

Sanjay Patel via llvm-commits llvm-commits at lists.llvm.org
Tue Oct 18 11:36:49 PDT 2016


Author: spatel
Date: Tue Oct 18 13:36:49 2016
New Revision: 284513

URL: http://llvm.org/viewvc/llvm-project?rev=284513&view=rev
Log:
revert r284495: [Target] remove TargetRecip class

There's something wrong with the StringRef usage while parsing the attribute string.


Added:
    llvm/trunk/include/llvm/Target/TargetRecip.h
      - copied unchanged from r284494, llvm/trunk/include/llvm/Target/TargetRecip.h
    llvm/trunk/lib/Target/TargetRecip.cpp
      - copied unchanged from r284494, llvm/trunk/lib/Target/TargetRecip.cpp
Modified:
    llvm/trunk/include/llvm/Target/TargetLowering.h
    llvm/trunk/include/llvm/Target/TargetOptions.h
    llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
    llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
    llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
    llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h
    llvm/trunk/lib/Target/CMakeLists.txt
    llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp
    llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h
    llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
    llvm/trunk/lib/Target/X86/X86ISelLowering.h

Modified: llvm/trunk/include/llvm/Target/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/Target/TargetLowering.h Tue Oct 18 13:36:49 2016
@@ -61,6 +61,7 @@ namespace llvm {
   class MCSymbol;
   template<typename T> class SmallVectorImpl;
   class DataLayout;
+  struct TargetRecip;
   class TargetRegisterClass;
   class TargetLibraryInfo;
   class TargetLoweringObjectFile;
@@ -246,37 +247,6 @@ public:
     return false;
   }
 
-  /// Reciprocal estimate status values used by the functions below.
-  enum ReciprocalEstimate : int {
-    Unspecified = -1,
-    Disabled = 0,
-    Enabled = 1
-  };
-
-  /// Return a ReciprocalEstimate enum value for a square root of the given type
-  /// based on the function's attributes. If the operation is not overridden by
-  /// the function's attributes, "Unspecified" is returned and target defaults
-  /// are expected to be used for instruction selection.
-  int getSqrtEnabled(EVT VT, MachineFunction &MF) const;
-
-  /// Return a ReciprocalEstimate enum value for a division of the given type
-  /// based on the function's attributes. If the operation is not overridden by
-  /// the function's attributes, "Unspecified" is returned and target defaults
-  /// are expected to be used for instruction selection.
-  int getDivEnabled(EVT VT, MachineFunction &MF) const;
-
-  /// Return the refinement step count for a square root of the given type based
-  /// on the function's attributes. If the operation is not overridden by
-  /// the function's attributes, "Unspecified" is returned and target defaults
-  /// are expected to be used for instruction selection.
-  int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const;
-
-  /// Return the refinement step count for a division of the given type based
-  /// on the function's attributes. If the operation is not overridden by
-  /// the function's attributes, "Unspecified" is returned and target defaults
-  /// are expected to be used for instruction selection.
-  int getDivRefinementSteps(EVT VT, MachineFunction &MF) const;
-
   /// Returns true if target has indicated at least one type should be bypassed.
   bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
 
@@ -569,6 +539,12 @@ public:
     }
   }
 
+  /// Return the reciprocal estimate code generation preferences for this target
+  /// after potentially overriding settings using the function's attributes.
+  /// FIXME: Like all unsafe-math target settings, this should really be an
+  /// instruction-level attribute/metadata/FMF.
+  TargetRecip getTargetRecipForFunc(MachineFunction &MF) const;
+
   /// Vector types are broken down into some number of legal first class types.
   /// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
   /// promoted EVT::f64 values with the X86 FP stack.  Similarly, EVT::v2i64
@@ -2182,6 +2158,7 @@ protected:
   /// sequence of memory operands that is recognized by PrologEpilogInserter.
   MachineBasicBlock *emitPatchPoint(MachineInstr &MI,
                                     MachineBasicBlock *MBB) const;
+  TargetRecip ReciprocalEstimates;
 };
 
 /// This class defines information used to lower LLVM code to legal SelectionDAG
@@ -2994,35 +2971,31 @@ public:
   /// roots.
 
   /// Return a reciprocal square root estimate value for the input operand.
-  /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
-  /// 'Enabled' as set by a potential default override attribute.
-  /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
-  /// refinement iterations required to generate a sufficient (though not
-  /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
+  /// The RefinementSteps output is the number of Newton-Raphson refinement
+  /// iterations required to generate a sufficient (though not necessarily
+  /// IEEE-754 compliant) estimate for the value type.
   /// The boolean UseOneConstNR output is used to select a Newton-Raphson
   /// algorithm implementation that uses one constant or two constants.
   /// A target may choose to implement its own refinement within this function.
   /// If that's true, then return '0' as the number of RefinementSteps to avoid
   /// any further refinement of the estimate.
   /// An empty SDValue return means no estimate sequence can be created.
-  virtual SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG,
-                                   int Enabled, int &RefinementSteps,
+  virtual SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                                   unsigned &RefinementSteps,
                                    bool &UseOneConstNR) const {
     return SDValue();
   }
 
   /// Return a reciprocal estimate value for the input operand.
-  /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
-  /// 'Enabled' as set by a potential default override attribute.
-  /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
-  /// refinement iterations required to generate a sufficient (though not
-  /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
+  /// The RefinementSteps output is the number of Newton-Raphson refinement
+  /// iterations required to generate a sufficient (though not necessarily
+  /// IEEE-754 compliant) estimate for the value type.
   /// A target may choose to implement its own refinement within this function.
   /// If that's true, then return '0' as the number of RefinementSteps to avoid
   /// any further refinement of the estimate.
   /// An empty SDValue return means no estimate sequence can be created.
-  virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
-                                   int Enabled, int &RefinementSteps) const {
+  virtual SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                                   unsigned &RefinementSteps) const {
     return SDValue();
   }
 

Modified: llvm/trunk/include/llvm/Target/TargetOptions.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetOptions.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetOptions.h (original)
+++ llvm/trunk/include/llvm/Target/TargetOptions.h Tue Oct 18 13:36:49 2016
@@ -15,6 +15,7 @@
 #ifndef LLVM_TARGET_TARGETOPTIONS_H
 #define LLVM_TARGET_TARGETOPTIONS_H
 
+#include "llvm/Target/TargetRecip.h"
 #include "llvm/MC/MCTargetOptions.h"
 
 namespace llvm {

Modified: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp Tue Oct 18 13:36:49 2016
@@ -14870,21 +14870,11 @@ SDValue DAGCombiner::BuildReciprocalEsti
   if (Level >= AfterLegalizeDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
-  EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
-    return SDValue();
-
-  // If estimates are explicitly disabled for this function, we're done.
-  MachineFunction &MF = DAG.getMachineFunction();
-  int Enabled = TLI.getDivEnabled(VT, MF);
-  if (Enabled == TLI.ReciprocalEstimate::Disabled)
-    return SDValue();
+  // Expose the DAG combiner to the target combiner implementations.
+  TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this);
 
-  // Estimates may be explicitly enabled for this type with a custom number of
-  // refinement steps.
-  int Iterations = TLI.getDivRefinementSteps(VT, MF);
-  if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
+  unsigned Iterations = 0;
+  if (SDValue Est = TLI.getRecipEstimate(Op, DCI, Iterations)) {
     if (Iterations) {
       // Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
       // For the reciprocal, we need to find the zero of the function:
@@ -14899,7 +14889,7 @@ SDValue DAGCombiner::BuildReciprocalEsti
       AddToWorklist(Est.getNode());
 
       // Newton iterations: Est = Est + Est (1 - Arg * Est)
-      for (int i = 0; i < Iterations; ++i) {
+      for (unsigned i = 0; i < Iterations; ++i) {
         SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est, Flags);
         AddToWorklist(NewEst.getNode());
 
@@ -15021,24 +15011,11 @@ SDValue DAGCombiner::buildSqrtEstimateIm
   if (Level >= AfterLegalizeDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
-  EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
-    return SDValue();
-
-  // If estimates are explicitly disabled for this function, we're done.
-  MachineFunction &MF = DAG.getMachineFunction();
-  int Enabled = TLI.getSqrtEnabled(VT, MF);
-  if (Enabled == TLI.ReciprocalEstimate::Disabled)
-    return SDValue();
-
-  // Estimates may be explicitly enabled for this type with a custom number of
-  // refinement steps.
-  int Iterations = TLI.getSqrtRefinementSteps(VT, MF);
-
+  // Expose the DAG combiner to the target combiner implementations.
+  TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this);
+  unsigned Iterations = 0;
   bool UseOneConstNR = false;
-  if (SDValue Est =
-          TLI.getRsqrtEstimate(Op, DAG, Enabled, Iterations, UseOneConstNR)) {
+  if (SDValue Est = TLI.getRsqrtEstimate(Op, DCI, Iterations, UseOneConstNR)) {
     AddToWorklist(Est.getNode());
     if (Iterations) {
       Est = UseOneConstNR

Modified: llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp (original)
+++ llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp Tue Oct 18 13:36:49 2016
@@ -14,7 +14,6 @@
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -838,6 +837,7 @@ TargetLoweringBase::TargetLoweringBase(c
   InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
   InitCmpLibcallCCs(CmpLibcallCCs);
   InitLibcallCallingConvs(LibcallCallingConvs);
+  ReciprocalEstimates.set("all", false, 0);
 }
 
 void TargetLoweringBase::initActions() {
@@ -1485,6 +1485,22 @@ MVT::SimpleValueType TargetLoweringBase:
   return MVT::i32; // return the default value
 }
 
+TargetRecip
+TargetLoweringBase::getTargetRecipForFunc(MachineFunction &MF) const {
+  const Function *F = MF.getFunction();
+  StringRef RecipAttrName = "reciprocal-estimates";
+  if (!F->hasFnAttribute(RecipAttrName))
+    return ReciprocalEstimates;
+
+  // Make a copy of the target's default reciprocal codegen settings.
+  TargetRecip Recips = ReciprocalEstimates;
+
+  // Override any settings that are customized for this function.
+  StringRef RecipString = F->getFnAttribute(RecipAttrName).getValueAsString();
+  Recips.set(RecipString);
+  return Recips;
+}
+
 /// getVectorTypeBreakdown - Vector types are broken down into some number of
 /// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f32
 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
@@ -1875,187 +1891,3 @@ unsigned TargetLoweringBase::getMaximumJ
 void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {
   MaximumJumpTableSize = Val;
 }
-
-//===----------------------------------------------------------------------===//
-//  Reciprocal Estimates
-//===----------------------------------------------------------------------===//
-
-/// Get the reciprocal estimate attribute string for a function that will
-/// override the target defaults.
-static StringRef getRecipEstimateForFunc(MachineFunction &MF) {
-  const Function *F = MF.getFunction();
-  StringRef RecipAttrName = "reciprocal-estimates";
-  if (!F->hasFnAttribute(RecipAttrName))
-    return StringRef();
-
-  return F->getFnAttribute(RecipAttrName).getValueAsString();
-}
-
-/// Construct a string for the given reciprocal operation of the given type.
-/// This string should match the corresponding option to the front-end's
-/// "-mrecip" flag assuming those strings have been passed through in an
-/// attribute string. For example, "vec-divf" for a division of a vXf32.
-static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
-  std::string Name = VT.isVector() ? "vec-" : "";
-
-  Name += IsSqrt ? "sqrt" : "div";
-
-  // TODO: Handle "half" or other float types?
-  if (VT.getScalarType() == MVT::f64) {
-    Name += "d";
-  } else {
-    assert(VT.getScalarType() == MVT::f32 &&
-           "Unexpected FP type for reciprocal estimate");
-    Name += "f";
-  }
-
-  return Name;
-}
-
-/// Return the character position and value (a single numeric character) of a
-/// customized refinement operation in the input string if it exists. Return
-/// false if there is no customized refinement step count.
-static bool parseRefinementStep(StringRef In, size_t &Position,
-                                uint8_t &Value) {
-  const char RefStepToken = ':';
-  Position = In.find(RefStepToken);
-  if (Position == StringRef::npos)
-    return false;
-
-  StringRef RefStepString = In.substr(Position + 1);
-  // Allow exactly one numeric character for the additional refinement
-  // step parameter.
-  if (RefStepString.size() == 1) {
-    char RefStepChar = RefStepString[0];
-    if (RefStepChar >= '0' && RefStepChar <= '9') {
-      Value = RefStepChar - '0';
-      return true;
-    }
-  }
-  report_fatal_error("Invalid refinement step for -recip.");
-}
-
-/// For the input attribute string, return one of the ReciprocalEstimate enum
-/// status values (enabled, disabled, or not specified) for this operation on
-/// the specified data type.
-static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
-  if (Override.empty())
-    return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
-  SmallVector<StringRef, 4> OverrideVector;
-  SplitString(Override, OverrideVector, ",");
-  unsigned NumArgs = OverrideVector.size();
-
-  // Check if "all", "none", or "default" was specified.
-  if (NumArgs == 1) {
-    // Look for an optional setting of the number of refinement steps needed
-    // for this type of reciprocal operation.
-    size_t RefPos;
-    uint8_t RefSteps;
-    if (parseRefinementStep(Override, RefPos, RefSteps)) {
-      // Split the string for further processing.
-      Override = Override.substr(0, RefPos);
-    }
-
-    // All reciprocal types are enabled.
-    if (Override == "all")
-      return TargetLoweringBase::ReciprocalEstimate::Enabled;
-
-    // All reciprocal types are disabled.
-    if (Override == "none")
-      return TargetLoweringBase::ReciprocalEstimate::Disabled;
-
-    // Target defaults for enablement are used.
-    if (Override == "default")
-      return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-  }
-
-  // The attribute string may omit the size suffix ('f'/'d').
-  StringRef VTName = getReciprocalOpName(IsSqrt, VT);
-  StringRef VTNameNoSize = VTName.drop_back();
-  static const char DisabledPrefix = '!';
-
-  for (StringRef RecipType : OverrideVector) {
-    size_t RefPos;
-    uint8_t RefSteps;
-    if (parseRefinementStep(RecipType, RefPos, RefSteps))
-      RecipType = RecipType.substr(0, RefPos);
-
-    // Ignore the disablement token for string matching.
-    bool IsDisabled = RecipType[0] == DisabledPrefix;
-    if (IsDisabled)
-      RecipType = RecipType.substr(1);
-
-    if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
-      return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled
-                        : TargetLoweringBase::ReciprocalEstimate::Enabled;
-  }
-
-  return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-}
-
-/// For the input attribute string, return the customized refinement step count
-/// for this operation on the specified data type. If the step count does not
-/// exist, return the ReciprocalEstimate enum value for unspecified.
-static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
-  if (Override.empty())
-    return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
-  SmallVector<StringRef, 4> OverrideVector;
-  SplitString(Override, OverrideVector, ",");
-  unsigned NumArgs = OverrideVector.size();
-
-  // Check if "all", "default", or "none" was specified.
-  if (NumArgs == 1) {
-    // Look for an optional setting of the number of refinement steps needed
-    // for this type of reciprocal operation.
-    size_t RefPos;
-    uint8_t RefSteps;
-    if (!parseRefinementStep(Override, RefPos, RefSteps))
-      return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
-    // Split the string for further processing.
-    Override = Override.substr(0, RefPos);
-    assert(Override != "none" &&
-           "Disabled reciprocals, but specifed refinement steps?");
-
-    // If this is a general override, return the specified number of steps.
-    if (Override == "all" || Override == "default")
-      return RefSteps;
-  }
-
-  // The attribute string may omit the size suffix ('f'/'d').
-  StringRef VTName = getReciprocalOpName(IsSqrt, VT);
-  StringRef VTNameNoSize = VTName.drop_back();
-
-  for (StringRef RecipType : OverrideVector) {
-    size_t RefPos;
-    uint8_t RefSteps;
-    if (!parseRefinementStep(RecipType, RefPos, RefSteps))
-      continue;
-
-    RecipType = RecipType.substr(0, RefPos);
-    if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
-      return RefSteps;
-  }
-
-  return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-}
-
-int TargetLoweringBase::getSqrtEnabled(EVT VT, MachineFunction &MF) const {
-  return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getDivEnabled(EVT VT, MachineFunction &MF) const {
-  return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,
-                                               MachineFunction &MF) const {
-  return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getDivRefinementSteps(EVT VT,
-                                              MachineFunction &MF) const {
-  return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
-}

Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -2859,9 +2859,10 @@ const char* AMDGPUTargetLowering::getTar
 }
 
 SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand,
-                                               SelectionDAG &DAG, int Enabled,
-                                               int &RefinementSteps,
+                                               DAGCombinerInfo &DCI,
+                                               unsigned &RefinementSteps,
                                                bool &UseOneConstNR) const {
+  SelectionDAG &DAG = DCI.DAG;
   EVT VT = Operand.getValueType();
 
   if (VT == MVT::f32) {
@@ -2876,8 +2877,9 @@ SDValue AMDGPUTargetLowering::getRsqrtEs
 }
 
 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
-                                               SelectionDAG &DAG, int Enabled,
-                                               int &RefinementSteps) const {
+                                               DAGCombinerInfo &DCI,
+                                               unsigned &RefinementSteps) const {
+  SelectionDAG &DAG = DCI.DAG;
   EVT VT = Operand.getValueType();
 
   if (VT == MVT::f32) {

Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h Tue Oct 18 13:36:49 2016
@@ -164,11 +164,13 @@ public:
   bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override {
     return true;
   }
-  SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                           int &RefinementSteps,
+  SDValue getRsqrtEstimate(SDValue Operand,
+                           DAGCombinerInfo &DCI,
+                           unsigned &RefinementSteps,
                            bool &UseOneConstNR) const override;
-  SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                           int &RefinementSteps) const override;
+  SDValue getRecipEstimate(SDValue Operand,
+                           DAGCombinerInfo &DCI,
+                           unsigned &RefinementSteps) const override;
 
   virtual SDNode *PostISelFolding(MachineSDNode *N,
                                   SelectionDAG &DAG) const = 0;

Modified: llvm/trunk/lib/Target/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CMakeLists.txt?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CMakeLists.txt (original)
+++ llvm/trunk/lib/Target/CMakeLists.txt Tue Oct 18 13:36:49 2016
@@ -6,6 +6,7 @@ add_llvm_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetRecip.cpp
   TargetSubtargetInfo.cpp
 
   ADDITIONAL_HEADER_DIRS

Modified: llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -901,6 +901,23 @@ PPCTargetLowering::PPCTargetLowering(con
     setTargetDAGCombine(ISD::FSQRT);
   }
 
+  // For the estimates, convergence is quadratic, so we essentially double the
+  // number of digits correct after every iteration. For both FRE and FRSQRTE,
+  // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
+  // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
+  unsigned RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3,
+           RefinementSteps64 = RefinementSteps + 1;
+
+  ReciprocalEstimates.set("sqrtf", true, RefinementSteps);
+  ReciprocalEstimates.set("vec-sqrtf", true, RefinementSteps);
+  ReciprocalEstimates.set("divf", true, RefinementSteps);
+  ReciprocalEstimates.set("vec-divf", true, RefinementSteps);
+
+  ReciprocalEstimates.set("sqrtd", true, RefinementSteps64);
+  ReciprocalEstimates.set("vec-sqrtd", true, RefinementSteps64);
+  ReciprocalEstimates.set("divd", true, RefinementSteps64);
+  ReciprocalEstimates.set("vec-divd", true, RefinementSteps64);
+
   // Darwin long double math library functions have $LDBL128 appended.
   if (Subtarget.isDarwin()) {
     setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128");
@@ -9622,19 +9639,22 @@ PPCTargetLowering::EmitInstrWithCustomIn
 // Target Optimization Hooks
 //===----------------------------------------------------------------------===//
 
-static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
-  // For the estimates, convergence is quadratic, so we essentially double the
-  // number of digits correct after every iteration. For both FRE and FRSQRTE,
-  // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
-  // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
-  int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
+static std::string getRecipOp(const char *Base, EVT VT) {
+  std::string RecipOp(Base);
   if (VT.getScalarType() == MVT::f64)
-    RefinementSteps++;
-  return RefinementSteps;
+    RecipOp += "d";
+  else
+    RecipOp += "f";
+
+  if (VT.isVector())
+    RecipOp = "vec-" + RecipOp;
+
+  return RecipOp;
 }
 
-SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG,
-                                            int Enabled, int &RefinementSteps,
+SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand,
+                                            DAGCombinerInfo &DCI,
+                                            unsigned &RefinementSteps,
                                             bool &UseOneConstNR) const {
   EVT VT = Operand.getValueType();
   if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
@@ -9643,18 +9663,21 @@ SDValue PPCTargetLowering::getRsqrtEstim
       (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
       (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
       (VT == MVT::v4f64 && Subtarget.hasQPX())) {
-    if (RefinementSteps == ReciprocalEstimate::Unspecified)
-      RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
+    TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+    std::string RecipOp = getRecipOp("sqrt", VT);
+    if (!Recips.isEnabled(RecipOp))
+      return SDValue();
 
+    RefinementSteps = Recips.getRefinementSteps(RecipOp);
     UseOneConstNR = true;
-    return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
+    return DCI.DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
   }
   return SDValue();
 }
 
-SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
-                                            int Enabled,
-                                            int &RefinementSteps) const {
+SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand,
+                                            DAGCombinerInfo &DCI,
+                                            unsigned &RefinementSteps) const {
   EVT VT = Operand.getValueType();
   if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
       (VT == MVT::f64 && Subtarget.hasFRE()) ||
@@ -9662,9 +9685,13 @@ SDValue PPCTargetLowering::getRecipEstim
       (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
       (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
       (VT == MVT::v4f64 && Subtarget.hasQPX())) {
-    if (RefinementSteps == ReciprocalEstimate::Unspecified)
-      RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
-    return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
+    TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+    std::string RecipOp = getRecipOp("div", VT);
+    if (!Recips.isEnabled(RecipOp))
+      return SDValue();
+
+    RefinementSteps = Recips.getRefinementSteps(RecipOp);
+    return DCI.DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
   }
   return SDValue();
 }

Modified: llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h Tue Oct 18 13:36:49 2016
@@ -967,11 +967,11 @@ namespace llvm {
     SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const;
 
-    SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                             int &RefinementSteps,
+    SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                             unsigned &RefinementSteps,
                              bool &UseOneConstNR) const override;
-    SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                             int &RefinementSteps) const override;
+    SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                             unsigned &RefinementSteps) const override;
     unsigned combineRepeatedFPDivisors() const override;
 
     CCAssignFn *useFastISelCCs(unsigned Flag) const;

Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -53,6 +53,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetRecip.h"
 #include "X86IntrinsicsInfo.h"
 #include <bitset>
 #include <numeric>
@@ -84,6 +85,15 @@ X86TargetLowering::X86TargetLowering(con
   // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
+  // By default (and when -ffast-math is on), enable estimate codegen with 1
+  // refinement step for floats (not doubles) except scalar division. Scalar
+  // division estimates are disabled because they break too much real-world
+  // code. These defaults are intended to match GCC behavior.
+  ReciprocalEstimates.set("sqrtf", true, 1);
+  ReciprocalEstimates.set("divf", false, 1);
+  ReciprocalEstimates.set("vec-sqrtf", true, 1);
+  ReciprocalEstimates.set("vec-divf", true, 1);
+
   // For 64-bit, since we have so many registers, use the ILP scheduler.
   // For 32-bit, use the register pressure specific scheduling.
   // For Atom, always use ILP scheduling.
@@ -15241,10 +15251,11 @@ bool X86TargetLowering::isFsqrtCheap(SDV
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
 SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
-                                            SelectionDAG &DAG, int Enabled,
-                                            int &RefinementSteps,
+                                            DAGCombinerInfo &DCI,
+                                            unsigned &RefinementSteps,
                                             bool &UseOneConstNR) const {
   EVT VT = Op.getValueType();
+  const char *RecipOp;
 
   // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
   // TODO: Add support for AVX512 (v16f32).
@@ -15253,24 +15264,30 @@ SDValue X86TargetLowering::getRsqrtEstim
   // instructions: convert to single, rsqrtss, convert back to double, refine
   // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
-  if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
-      (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
-      (VT == MVT::v8f32 && Subtarget.hasAVX())) {
-    if (RefinementSteps == ReciprocalEstimate::Unspecified)
-      RefinementSteps = 1;
+  if (VT == MVT::f32 && Subtarget.hasSSE1())
+    RecipOp = "sqrtf";
+  else if ((VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
+           (VT == MVT::v8f32 && Subtarget.hasAVX()))
+    RecipOp = "vec-sqrtf";
+  else
+    return SDValue();
 
-    UseOneConstNR = false;
-    return DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
-  }
-  return SDValue();
+  TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+  if (!Recips.isEnabled(RecipOp))
+    return SDValue();
+
+  RefinementSteps = Recips.getRefinementSteps(RecipOp);
+  UseOneConstNR = false;
+  return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
 }
 
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
-SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
-                                            int Enabled,
-                                            int &RefinementSteps) const {
+SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
+                                            DAGCombinerInfo &DCI,
+                                            unsigned &RefinementSteps) const {
   EVT VT = Op.getValueType();
+  const char *RecipOp;
 
   // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
   // TODO: Add support for AVX512 (v16f32).
@@ -15279,22 +15296,20 @@ SDValue X86TargetLowering::getRecipEstim
   // 15 instructions: convert to single, rcpss, convert back to double, refine
   // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
+  if (VT == MVT::f32 && Subtarget.hasSSE1())
+    RecipOp = "divf";
+  else if ((VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
+           (VT == MVT::v8f32 && Subtarget.hasAVX()))
+    RecipOp = "vec-divf";
+  else
+    return SDValue();
 
-  if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
-      (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
-      (VT == MVT::v8f32 && Subtarget.hasAVX())) {
-    // Enable estimate codegen with 1 refinement step for vector division.
-    // Scalar division estimates are disabled because they break too much
-    // real-world code. These defaults are intended to match GCC behavior.
-    if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
-      return SDValue();
-
-    if (RefinementSteps == ReciprocalEstimate::Unspecified)
-      RefinementSteps = 1;
+  TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+  if (!Recips.isEnabled(RecipOp))
+    return SDValue();
 
-    return DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
-  }
-  return SDValue();
+  RefinementSteps = Recips.getRefinementSteps(RecipOp);
+  return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
 }
 
 /// If we have at least two divisions that use the same divisor, convert to

Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.h (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.h Tue Oct 18 13:36:49 2016
@@ -1266,13 +1266,13 @@ namespace llvm {
     bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override;
 
     /// Use rsqrt* to speed up sqrt calculations.
-    SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                             int &RefinementSteps,
+    SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                             unsigned &RefinementSteps,
                              bool &UseOneConstNR) const override;
 
     /// Use rcp* to speed up fdiv calculations.
-    SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
-                             int &RefinementSteps) const override;
+    SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                             unsigned &RefinementSteps) const override;
 
     /// Reassociate floating point divisions into multiply by reciprocal.
     unsigned combineRepeatedFPDivisors() const override;




More information about the llvm-commits mailing list