[llvm] r284513 - revert r284495: [Target] remove TargetRecip class
Sanjay Patel via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 18 11:36:49 PDT 2016
Author: spatel
Date: Tue Oct 18 13:36:49 2016
New Revision: 284513
URL: http://llvm.org/viewvc/llvm-project?rev=284513&view=rev
Log:
revert r284495: [Target] remove TargetRecip class
There's something wrong with the StringRef usage while parsing the attribute string.
Added:
llvm/trunk/include/llvm/Target/TargetRecip.h
- copied unchanged from r284494, llvm/trunk/include/llvm/Target/TargetRecip.h
llvm/trunk/lib/Target/TargetRecip.cpp
- copied unchanged from r284494, llvm/trunk/lib/Target/TargetRecip.cpp
Modified:
llvm/trunk/include/llvm/Target/TargetLowering.h
llvm/trunk/include/llvm/Target/TargetOptions.h
llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h
llvm/trunk/lib/Target/CMakeLists.txt
llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp
llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
llvm/trunk/lib/Target/X86/X86ISelLowering.h
Modified: llvm/trunk/include/llvm/Target/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/Target/TargetLowering.h Tue Oct 18 13:36:49 2016
@@ -61,6 +61,7 @@ namespace llvm {
class MCSymbol;
template<typename T> class SmallVectorImpl;
class DataLayout;
+ struct TargetRecip;
class TargetRegisterClass;
class TargetLibraryInfo;
class TargetLoweringObjectFile;
@@ -246,37 +247,6 @@ public:
return false;
}
- /// Reciprocal estimate status values used by the functions below.
- enum ReciprocalEstimate : int {
- Unspecified = -1,
- Disabled = 0,
- Enabled = 1
- };
-
- /// Return a ReciprocalEstimate enum value for a square root of the given type
- /// based on the function's attributes. If the operation is not overridden by
- /// the function's attributes, "Unspecified" is returned and target defaults
- /// are expected to be used for instruction selection.
- int getSqrtEnabled(EVT VT, MachineFunction &MF) const;
-
- /// Return a ReciprocalEstimate enum value for a division of the given type
- /// based on the function's attributes. If the operation is not overridden by
- /// the function's attributes, "Unspecified" is returned and target defaults
- /// are expected to be used for instruction selection.
- int getDivEnabled(EVT VT, MachineFunction &MF) const;
-
- /// Return the refinement step count for a square root of the given type based
- /// on the function's attributes. If the operation is not overridden by
- /// the function's attributes, "Unspecified" is returned and target defaults
- /// are expected to be used for instruction selection.
- int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const;
-
- /// Return the refinement step count for a division of the given type based
- /// on the function's attributes. If the operation is not overridden by
- /// the function's attributes, "Unspecified" is returned and target defaults
- /// are expected to be used for instruction selection.
- int getDivRefinementSteps(EVT VT, MachineFunction &MF) const;
-
/// Returns true if target has indicated at least one type should be bypassed.
bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
@@ -569,6 +539,12 @@ public:
}
}
+ /// Return the reciprocal estimate code generation preferences for this target
+ /// after potentially overriding settings using the function's attributes.
+ /// FIXME: Like all unsafe-math target settings, this should really be an
+ /// instruction-level attribute/metadata/FMF.
+ TargetRecip getTargetRecipForFunc(MachineFunction &MF) const;
+
/// Vector types are broken down into some number of legal first class types.
/// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
/// promoted EVT::f64 values with the X86 FP stack. Similarly, EVT::v2i64
@@ -2182,6 +2158,7 @@ protected:
/// sequence of memory operands that is recognized by PrologEpilogInserter.
MachineBasicBlock *emitPatchPoint(MachineInstr &MI,
MachineBasicBlock *MBB) const;
+ TargetRecip ReciprocalEstimates;
};
/// This class defines information used to lower LLVM code to legal SelectionDAG
@@ -2994,35 +2971,31 @@ public:
/// roots.
/// Return a reciprocal square root estimate value for the input operand.
- /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
- /// 'Enabled' as set by a potential default override attribute.
- /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
- /// refinement iterations required to generate a sufficient (though not
- /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
+ /// The RefinementSteps output is the number of Newton-Raphson refinement
+ /// iterations required to generate a sufficient (though not necessarily
+ /// IEEE-754 compliant) estimate for the value type.
/// The boolean UseOneConstNR output is used to select a Newton-Raphson
/// algorithm implementation that uses one constant or two constants.
/// A target may choose to implement its own refinement within this function.
/// If that's true, then return '0' as the number of RefinementSteps to avoid
/// any further refinement of the estimate.
/// An empty SDValue return means no estimate sequence can be created.
- virtual SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG,
- int Enabled, int &RefinementSteps,
+ virtual SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const {
return SDValue();
}
/// Return a reciprocal estimate value for the input operand.
- /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
- /// 'Enabled' as set by a potential default override attribute.
- /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
- /// refinement iterations required to generate a sufficient (though not
- /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
+ /// The RefinementSteps output is the number of Newton-Raphson refinement
+ /// iterations required to generate a sufficient (though not necessarily
+ /// IEEE-754 compliant) estimate for the value type.
/// A target may choose to implement its own refinement within this function.
/// If that's true, then return '0' as the number of RefinementSteps to avoid
/// any further refinement of the estimate.
/// An empty SDValue return means no estimate sequence can be created.
- virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
- int Enabled, int &RefinementSteps) const {
+ virtual SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const {
return SDValue();
}
Modified: llvm/trunk/include/llvm/Target/TargetOptions.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetOptions.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetOptions.h (original)
+++ llvm/trunk/include/llvm/Target/TargetOptions.h Tue Oct 18 13:36:49 2016
@@ -15,6 +15,7 @@
#ifndef LLVM_TARGET_TARGETOPTIONS_H
#define LLVM_TARGET_TARGETOPTIONS_H
+#include "llvm/Target/TargetRecip.h"
#include "llvm/MC/MCTargetOptions.h"
namespace llvm {
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp Tue Oct 18 13:36:49 2016
@@ -14870,21 +14870,11 @@ SDValue DAGCombiner::BuildReciprocalEsti
if (Level >= AfterLegalizeDAG)
return SDValue();
- // TODO: Handle half and/or extended types?
- EVT VT = Op.getValueType();
- if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
- return SDValue();
-
- // If estimates are explicitly disabled for this function, we're done.
- MachineFunction &MF = DAG.getMachineFunction();
- int Enabled = TLI.getDivEnabled(VT, MF);
- if (Enabled == TLI.ReciprocalEstimate::Disabled)
- return SDValue();
+ // Expose the DAG combiner to the target combiner implementations.
+ TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this);
- // Estimates may be explicitly enabled for this type with a custom number of
- // refinement steps.
- int Iterations = TLI.getDivRefinementSteps(VT, MF);
- if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
+ unsigned Iterations = 0;
+ if (SDValue Est = TLI.getRecipEstimate(Op, DCI, Iterations)) {
if (Iterations) {
// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
// For the reciprocal, we need to find the zero of the function:
@@ -14899,7 +14889,7 @@ SDValue DAGCombiner::BuildReciprocalEsti
AddToWorklist(Est.getNode());
// Newton iterations: Est = Est + Est (1 - Arg * Est)
- for (int i = 0; i < Iterations; ++i) {
+ for (unsigned i = 0; i < Iterations; ++i) {
SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est, Flags);
AddToWorklist(NewEst.getNode());
@@ -15021,24 +15011,11 @@ SDValue DAGCombiner::buildSqrtEstimateIm
if (Level >= AfterLegalizeDAG)
return SDValue();
- // TODO: Handle half and/or extended types?
- EVT VT = Op.getValueType();
- if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
- return SDValue();
-
- // If estimates are explicitly disabled for this function, we're done.
- MachineFunction &MF = DAG.getMachineFunction();
- int Enabled = TLI.getSqrtEnabled(VT, MF);
- if (Enabled == TLI.ReciprocalEstimate::Disabled)
- return SDValue();
-
- // Estimates may be explicitly enabled for this type with a custom number of
- // refinement steps.
- int Iterations = TLI.getSqrtRefinementSteps(VT, MF);
-
+ // Expose the DAG combiner to the target combiner implementations.
+ TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this);
+ unsigned Iterations = 0;
bool UseOneConstNR = false;
- if (SDValue Est =
- TLI.getRsqrtEstimate(Op, DAG, Enabled, Iterations, UseOneConstNR)) {
+ if (SDValue Est = TLI.getRsqrtEstimate(Op, DCI, Iterations, UseOneConstNR)) {
AddToWorklist(Est.getNode());
if (Iterations) {
Est = UseOneConstNR
Modified: llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp (original)
+++ llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp Tue Oct 18 13:36:49 2016
@@ -14,7 +14,6 @@
#include "llvm/Target/TargetLowering.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/Triple.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
@@ -838,6 +837,7 @@ TargetLoweringBase::TargetLoweringBase(c
InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
InitCmpLibcallCCs(CmpLibcallCCs);
InitLibcallCallingConvs(LibcallCallingConvs);
+ ReciprocalEstimates.set("all", false, 0);
}
void TargetLoweringBase::initActions() {
@@ -1485,6 +1485,22 @@ MVT::SimpleValueType TargetLoweringBase:
return MVT::i32; // return the default value
}
+TargetRecip
+TargetLoweringBase::getTargetRecipForFunc(MachineFunction &MF) const {
+ const Function *F = MF.getFunction();
+ StringRef RecipAttrName = "reciprocal-estimates";
+ if (!F->hasFnAttribute(RecipAttrName))
+ return ReciprocalEstimates;
+
+ // Make a copy of the target's default reciprocal codegen settings.
+ TargetRecip Recips = ReciprocalEstimates;
+
+ // Override any settings that are customized for this function.
+ StringRef RecipString = F->getFnAttribute(RecipAttrName).getValueAsString();
+ Recips.set(RecipString);
+ return Recips;
+}
+
/// getVectorTypeBreakdown - Vector types are broken down into some number of
/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
@@ -1875,187 +1891,3 @@ unsigned TargetLoweringBase::getMaximumJ
void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {
MaximumJumpTableSize = Val;
}
-
-//===----------------------------------------------------------------------===//
-// Reciprocal Estimates
-//===----------------------------------------------------------------------===//
-
-/// Get the reciprocal estimate attribute string for a function that will
-/// override the target defaults.
-static StringRef getRecipEstimateForFunc(MachineFunction &MF) {
- const Function *F = MF.getFunction();
- StringRef RecipAttrName = "reciprocal-estimates";
- if (!F->hasFnAttribute(RecipAttrName))
- return StringRef();
-
- return F->getFnAttribute(RecipAttrName).getValueAsString();
-}
-
-/// Construct a string for the given reciprocal operation of the given type.
-/// This string should match the corresponding option to the front-end's
-/// "-mrecip" flag assuming those strings have been passed through in an
-/// attribute string. For example, "vec-divf" for a division of a vXf32.
-static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
- std::string Name = VT.isVector() ? "vec-" : "";
-
- Name += IsSqrt ? "sqrt" : "div";
-
- // TODO: Handle "half" or other float types?
- if (VT.getScalarType() == MVT::f64) {
- Name += "d";
- } else {
- assert(VT.getScalarType() == MVT::f32 &&
- "Unexpected FP type for reciprocal estimate");
- Name += "f";
- }
-
- return Name;
-}
-
-/// Return the character position and value (a single numeric character) of a
-/// customized refinement operation in the input string if it exists. Return
-/// false if there is no customized refinement step count.
-static bool parseRefinementStep(StringRef In, size_t &Position,
- uint8_t &Value) {
- const char RefStepToken = ':';
- Position = In.find(RefStepToken);
- if (Position == StringRef::npos)
- return false;
-
- StringRef RefStepString = In.substr(Position + 1);
- // Allow exactly one numeric character for the additional refinement
- // step parameter.
- if (RefStepString.size() == 1) {
- char RefStepChar = RefStepString[0];
- if (RefStepChar >= '0' && RefStepChar <= '9') {
- Value = RefStepChar - '0';
- return true;
- }
- }
- report_fatal_error("Invalid refinement step for -recip.");
-}
-
-/// For the input attribute string, return one of the ReciprocalEstimate enum
-/// status values (enabled, disabled, or not specified) for this operation on
-/// the specified data type.
-static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
- if (Override.empty())
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
- SmallVector<StringRef, 4> OverrideVector;
- SplitString(Override, OverrideVector, ",");
- unsigned NumArgs = OverrideVector.size();
-
- // Check if "all", "none", or "default" was specified.
- if (NumArgs == 1) {
- // Look for an optional setting of the number of refinement steps needed
- // for this type of reciprocal operation.
- size_t RefPos;
- uint8_t RefSteps;
- if (parseRefinementStep(Override, RefPos, RefSteps)) {
- // Split the string for further processing.
- Override = Override.substr(0, RefPos);
- }
-
- // All reciprocal types are enabled.
- if (Override == "all")
- return TargetLoweringBase::ReciprocalEstimate::Enabled;
-
- // All reciprocal types are disabled.
- if (Override == "none")
- return TargetLoweringBase::ReciprocalEstimate::Disabled;
-
- // Target defaults for enablement are used.
- if (Override == "default")
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
- }
-
- // The attribute string may omit the size suffix ('f'/'d').
- StringRef VTName = getReciprocalOpName(IsSqrt, VT);
- StringRef VTNameNoSize = VTName.drop_back();
- static const char DisabledPrefix = '!';
-
- for (StringRef RecipType : OverrideVector) {
- size_t RefPos;
- uint8_t RefSteps;
- if (parseRefinementStep(RecipType, RefPos, RefSteps))
- RecipType = RecipType.substr(0, RefPos);
-
- // Ignore the disablement token for string matching.
- bool IsDisabled = RecipType[0] == DisabledPrefix;
- if (IsDisabled)
- RecipType = RecipType.substr(1);
-
- if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
- return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled
- : TargetLoweringBase::ReciprocalEstimate::Enabled;
- }
-
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-}
-
-/// For the input attribute string, return the customized refinement step count
-/// for this operation on the specified data type. If the step count does not
-/// exist, return the ReciprocalEstimate enum value for unspecified.
-static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
- if (Override.empty())
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
- SmallVector<StringRef, 4> OverrideVector;
- SplitString(Override, OverrideVector, ",");
- unsigned NumArgs = OverrideVector.size();
-
- // Check if "all", "default", or "none" was specified.
- if (NumArgs == 1) {
- // Look for an optional setting of the number of refinement steps needed
- // for this type of reciprocal operation.
- size_t RefPos;
- uint8_t RefSteps;
- if (!parseRefinementStep(Override, RefPos, RefSteps))
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-
- // Split the string for further processing.
- Override = Override.substr(0, RefPos);
- assert(Override != "none" &&
- "Disabled reciprocals, but specifed refinement steps?");
-
- // If this is a general override, return the specified number of steps.
- if (Override == "all" || Override == "default")
- return RefSteps;
- }
-
- // The attribute string may omit the size suffix ('f'/'d').
- StringRef VTName = getReciprocalOpName(IsSqrt, VT);
- StringRef VTNameNoSize = VTName.drop_back();
-
- for (StringRef RecipType : OverrideVector) {
- size_t RefPos;
- uint8_t RefSteps;
- if (!parseRefinementStep(RecipType, RefPos, RefSteps))
- continue;
-
- RecipType = RecipType.substr(0, RefPos);
- if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
- return RefSteps;
- }
-
- return TargetLoweringBase::ReciprocalEstimate::Unspecified;
-}
-
-int TargetLoweringBase::getSqrtEnabled(EVT VT, MachineFunction &MF) const {
- return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getDivEnabled(EVT VT, MachineFunction &MF) const {
- return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,
- MachineFunction &MF) const {
- return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
-}
-
-int TargetLoweringBase::getDivRefinementSteps(EVT VT,
- MachineFunction &MF) const {
- return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
-}
Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -2859,9 +2859,10 @@ const char* AMDGPUTargetLowering::getTar
}
SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand,
- SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const {
+ SelectionDAG &DAG = DCI.DAG;
EVT VT = Operand.getValueType();
if (VT == MVT::f32) {
@@ -2876,8 +2877,9 @@ SDValue AMDGPUTargetLowering::getRsqrtEs
}
SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
- SelectionDAG &DAG, int Enabled,
- int &RefinementSteps) const {
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const {
+ SelectionDAG &DAG = DCI.DAG;
EVT VT = Operand.getValueType();
if (VT == MVT::f32) {
Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h Tue Oct 18 13:36:49 2016
@@ -164,11 +164,13 @@ public:
bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override {
return true;
}
- SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+ SDValue getRsqrtEstimate(SDValue Operand,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const override;
- SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps) const override;
+ SDValue getRecipEstimate(SDValue Operand,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const override;
virtual SDNode *PostISelFolding(MachineSDNode *N,
SelectionDAG &DAG) const = 0;
Modified: llvm/trunk/lib/Target/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/CMakeLists.txt?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/CMakeLists.txt (original)
+++ llvm/trunk/lib/Target/CMakeLists.txt Tue Oct 18 13:36:49 2016
@@ -6,6 +6,7 @@ add_llvm_library(LLVMTarget
TargetLoweringObjectFile.cpp
TargetMachine.cpp
TargetMachineC.cpp
+ TargetRecip.cpp
TargetSubtargetInfo.cpp
ADDITIONAL_HEADER_DIRS
Modified: llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -901,6 +901,23 @@ PPCTargetLowering::PPCTargetLowering(con
setTargetDAGCombine(ISD::FSQRT);
}
+ // For the estimates, convergence is quadratic, so we essentially double the
+ // number of digits correct after every iteration. For both FRE and FRSQRTE,
+ // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
+ // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
+ unsigned RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3,
+ RefinementSteps64 = RefinementSteps + 1;
+
+ ReciprocalEstimates.set("sqrtf", true, RefinementSteps);
+ ReciprocalEstimates.set("vec-sqrtf", true, RefinementSteps);
+ ReciprocalEstimates.set("divf", true, RefinementSteps);
+ ReciprocalEstimates.set("vec-divf", true, RefinementSteps);
+
+ ReciprocalEstimates.set("sqrtd", true, RefinementSteps64);
+ ReciprocalEstimates.set("vec-sqrtd", true, RefinementSteps64);
+ ReciprocalEstimates.set("divd", true, RefinementSteps64);
+ ReciprocalEstimates.set("vec-divd", true, RefinementSteps64);
+
// Darwin long double math library functions have $LDBL128 appended.
if (Subtarget.isDarwin()) {
setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128");
@@ -9622,19 +9639,22 @@ PPCTargetLowering::EmitInstrWithCustomIn
// Target Optimization Hooks
//===----------------------------------------------------------------------===//
-static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
- // For the estimates, convergence is quadratic, so we essentially double the
- // number of digits correct after every iteration. For both FRE and FRSQRTE,
- // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
- // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
- int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
+static std::string getRecipOp(const char *Base, EVT VT) {
+ std::string RecipOp(Base);
if (VT.getScalarType() == MVT::f64)
- RefinementSteps++;
- return RefinementSteps;
+ RecipOp += "d";
+ else
+ RecipOp += "f";
+
+ if (VT.isVector())
+ RecipOp = "vec-" + RecipOp;
+
+ return RecipOp;
}
-SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG,
- int Enabled, int &RefinementSteps,
+SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const {
EVT VT = Operand.getValueType();
if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
@@ -9643,18 +9663,21 @@ SDValue PPCTargetLowering::getRsqrtEstim
(VT == MVT::v2f64 && Subtarget.hasVSX()) ||
(VT == MVT::v4f32 && Subtarget.hasQPX()) ||
(VT == MVT::v4f64 && Subtarget.hasQPX())) {
- if (RefinementSteps == ReciprocalEstimate::Unspecified)
- RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
+ TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+ std::string RecipOp = getRecipOp("sqrt", VT);
+ if (!Recips.isEnabled(RecipOp))
+ return SDValue();
+ RefinementSteps = Recips.getRefinementSteps(RecipOp);
UseOneConstNR = true;
- return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
+ return DCI.DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
}
return SDValue();
}
-SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
- int Enabled,
- int &RefinementSteps) const {
+SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const {
EVT VT = Operand.getValueType();
if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
(VT == MVT::f64 && Subtarget.hasFRE()) ||
@@ -9662,9 +9685,13 @@ SDValue PPCTargetLowering::getRecipEstim
(VT == MVT::v2f64 && Subtarget.hasVSX()) ||
(VT == MVT::v4f32 && Subtarget.hasQPX()) ||
(VT == MVT::v4f64 && Subtarget.hasQPX())) {
- if (RefinementSteps == ReciprocalEstimate::Unspecified)
- RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
- return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
+ TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+ std::string RecipOp = getRecipOp("div", VT);
+ if (!Recips.isEnabled(RecipOp))
+ return SDValue();
+
+ RefinementSteps = Recips.getRefinementSteps(RecipOp);
+ return DCI.DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
}
return SDValue();
}
Modified: llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCISelLowering.h Tue Oct 18 13:36:49 2016
@@ -967,11 +967,11 @@ namespace llvm {
SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const;
- SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+ SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const override;
- SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps) const override;
+ SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const override;
unsigned combineRepeatedFPDivisors() const override;
CCAssignFn *useFastISelCCs(unsigned Flag) const;
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Tue Oct 18 13:36:49 2016
@@ -53,6 +53,7 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetRecip.h"
#include "X86IntrinsicsInfo.h"
#include <bitset>
#include <numeric>
@@ -84,6 +85,15 @@ X86TargetLowering::X86TargetLowering(con
// X86-SSE is even stranger. It uses -1 or 0 for vector masks.
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
+ // By default (and when -ffast-math is on), enable estimate codegen with 1
+ // refinement step for floats (not doubles) except scalar division. Scalar
+ // division estimates are disabled because they break too much real-world
+ // code. These defaults are intended to match GCC behavior.
+ ReciprocalEstimates.set("sqrtf", true, 1);
+ ReciprocalEstimates.set("divf", false, 1);
+ ReciprocalEstimates.set("vec-sqrtf", true, 1);
+ ReciprocalEstimates.set("vec-divf", true, 1);
+
// For 64-bit, since we have so many registers, use the ILP scheduler.
// For 32-bit, use the register pressure specific scheduling.
// For Atom, always use ILP scheduling.
@@ -15241,10 +15251,11 @@ bool X86TargetLowering::isFsqrtCheap(SDV
/// The minimum architected relative accuracy is 2^-12. We need one
/// Newton-Raphson step to have a good float result (24 bits of precision).
SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
- SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const {
EVT VT = Op.getValueType();
+ const char *RecipOp;
// SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
// TODO: Add support for AVX512 (v16f32).
@@ -15253,24 +15264,30 @@ SDValue X86TargetLowering::getRsqrtEstim
// instructions: convert to single, rsqrtss, convert back to double, refine
// (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
// along with FMA, this could be a throughput win.
- if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
- (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
- (VT == MVT::v8f32 && Subtarget.hasAVX())) {
- if (RefinementSteps == ReciprocalEstimate::Unspecified)
- RefinementSteps = 1;
+ if (VT == MVT::f32 && Subtarget.hasSSE1())
+ RecipOp = "sqrtf";
+ else if ((VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
+ (VT == MVT::v8f32 && Subtarget.hasAVX()))
+ RecipOp = "vec-sqrtf";
+ else
+ return SDValue();
- UseOneConstNR = false;
- return DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
- }
- return SDValue();
+ TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+ if (!Recips.isEnabled(RecipOp))
+ return SDValue();
+
+ RefinementSteps = Recips.getRefinementSteps(RecipOp);
+ UseOneConstNR = false;
+ return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
}
/// The minimum architected relative accuracy is 2^-12. We need one
/// Newton-Raphson step to have a good float result (24 bits of precision).
-SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
- int Enabled,
- int &RefinementSteps) const {
+SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
+ DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const {
EVT VT = Op.getValueType();
+ const char *RecipOp;
// SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
// TODO: Add support for AVX512 (v16f32).
@@ -15279,22 +15296,20 @@ SDValue X86TargetLowering::getRecipEstim
// 15 instructions: convert to single, rcpss, convert back to double, refine
// (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
// along with FMA, this could be a throughput win.
+ if (VT == MVT::f32 && Subtarget.hasSSE1())
+ RecipOp = "divf";
+ else if ((VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
+ (VT == MVT::v8f32 && Subtarget.hasAVX()))
+ RecipOp = "vec-divf";
+ else
+ return SDValue();
- if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
- (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
- (VT == MVT::v8f32 && Subtarget.hasAVX())) {
- // Enable estimate codegen with 1 refinement step for vector division.
- // Scalar division estimates are disabled because they break too much
- // real-world code. These defaults are intended to match GCC behavior.
- if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
- return SDValue();
-
- if (RefinementSteps == ReciprocalEstimate::Unspecified)
- RefinementSteps = 1;
+ TargetRecip Recips = getTargetRecipForFunc(DCI.DAG.getMachineFunction());
+ if (!Recips.isEnabled(RecipOp))
+ return SDValue();
- return DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
- }
- return SDValue();
+ RefinementSteps = Recips.getRefinementSteps(RecipOp);
+ return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
}
/// If we have at least two divisions that use the same divisor, convert to
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.h?rev=284513&r1=284512&r2=284513&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.h (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.h Tue Oct 18 13:36:49 2016
@@ -1266,13 +1266,13 @@ namespace llvm {
bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override;
/// Use rsqrt* to speed up sqrt calculations.
- SDValue getRsqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+ SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps,
bool &UseOneConstNR) const override;
/// Use rcp* to speed up fdiv calculations.
- SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps) const override;
+ SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+ unsigned &RefinementSteps) const override;
/// Reassociate floating point divisions into multiply by reciprocal.
unsigned combineRepeatedFPDivisors() const override;
More information about the llvm-commits
mailing list