[llvm] r345039 - [LegalizeDAG] Share Vector/Scalar CTTZ Expansion
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 23 08:37:19 PDT 2018
Author: rksimon
Date: Tue Oct 23 08:37:19 2018
New Revision: 345039
URL: http://llvm.org/viewvc/llvm-project?rev=345039&view=rev
Log:
[LegalizeDAG] Share Vector/Scalar CTTZ Expansion
As suggested on D53258, this patch demonstrates sharing common CTTZ expansion code between VectorLegalizer and SelectionDAGLegalize by putting it in TargetLowering.
I intend to move CTLZ and (scalar) CTPOP over as well and then update D53258 accordingly.
Differential Revision: https://reviews.llvm.org/D53474
Modified:
llvm/trunk/include/llvm/CodeGen/TargetLowering.h
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Modified: llvm/trunk/include/llvm/CodeGen/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/TargetLowering.h?rev=345039&r1=345038&r2=345039&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/CodeGen/TargetLowering.h Tue Oct 23 08:37:19 2018
@@ -3647,6 +3647,13 @@ public:
/// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand CTTZ/CTTZ_ZERO_UNDEF nodes. Expands vector/scalar CTTZ nodes,
+ /// vector nodes can only succeed if all operations are legal/custom.
+ /// \param N Node to expand
+ /// \param Result output after conversion
+ /// \returns True, if the expansion was successful, false otherwise
+ bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+
/// Turn load of vector type into a load of the individual elements.
/// \param LD load to expand
/// \returns MERGE_VALUEs of the scalar loads with their chains.
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=345039&r1=345038&r2=345039&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Tue Oct 23 08:37:19 2018
@@ -2789,35 +2789,6 @@ SDValue SelectionDAGLegalize::ExpandBitC
Op = DAG.getNOT(dl, Op, VT);
return DAG.getNode(ISD::CTPOP, dl, VT, Op);
}
- case ISD::CTTZ_ZERO_UNDEF:
- // This trivially expands to CTTZ.
- return DAG.getNode(ISD::CTTZ, dl, VT, Op);
- case ISD::CTTZ: {
- if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
- EVT SetCCVT = getSetCCResultType(VT);
- SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
- SDValue Zero = DAG.getConstant(0, dl, VT);
- SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
- return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
- DAG.getConstant(Len, dl, VT), CTTZ);
- }
-
- // for now, we use: { return popcount(~x & (x - 1)); }
- // unless the target has ctlz but not ctpop, in which case we use:
- // { return 32 - nlz(~x & (x-1)); }
- // Ref: "Hacker's Delight" by Henry Warren
- SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
- DAG.getNOT(dl, Op, VT),
- DAG.getNode(ISD::SUB, dl, VT, Op,
- DAG.getConstant(1, dl, VT)));
- // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
- if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
- TLI.isOperationLegal(ISD::CTLZ, VT))
- return DAG.getNode(ISD::SUB, dl, VT,
- DAG.getConstant(Len, dl, VT),
- DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
- return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
- }
}
}
@@ -2831,11 +2802,14 @@ bool SelectionDAGLegalize::ExpandNode(SD
case ISD::CTPOP:
case ISD::CTLZ:
case ISD::CTLZ_ZERO_UNDEF:
- case ISD::CTTZ:
- case ISD::CTTZ_ZERO_UNDEF:
Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
Results.push_back(Tmp1);
break;
+ case ISD::CTTZ:
+ case ISD::CTTZ_ZERO_UNDEF:
+ if (TLI.expandCTTZ(Node, Tmp1, DAG))
+ Results.push_back(Tmp1);
+ break;
case ISD::BITREVERSE:
Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
break;
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp?rev=345039&r1=345038&r2=345039&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp Tue Oct 23 08:37:19 2018
@@ -1105,23 +1105,11 @@ SDValue VectorLegalizer::ExpandCTLZ(SDVa
SDValue VectorLegalizer::ExpandCTTZ(SDValue Op) {
EVT VT = Op.getValueType();
- unsigned NumBitsPerElt = VT.getScalarSizeInBits();
- // If the non-ZERO_UNDEF version is supported we can use that instead.
- if (TLI.isOperationLegalOrCustom(ISD::CTTZ, VT)) {
- SDLoc DL(Op);
- return DAG.getNode(ISD::CTTZ, DL, VT, Op.getOperand(0));
- }
-
- // If we have the appropriate vector bit operations, it is better to use them
- // than unrolling and expanding each component.
- if (isPowerOf2_32(NumBitsPerElt) &&
- (TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) ||
- TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) &&
- TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
- TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
- TLI.isOperationLegalOrCustomOrPromote(ISD::XOR, VT))
- return Op;
+ // Attempt to expand using TargetLowering.
+ SDValue Result;
+ if (TLI.expandCTTZ(Op.getNode(), Result, DAG))
+ return Result;
// Otherwise go ahead and unroll.
return DAG.UnrollVectorOp(Op.getNode());
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp?rev=345039&r1=345038&r2=345039&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp Tue Oct 23 08:37:19 2018
@@ -4142,6 +4142,61 @@ SDValue TargetLowering::expandFMINNUM_FM
return SDValue();
}
+bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
+ SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ EVT VT = Node->getValueType(0);
+ SDValue Op = Node->getOperand(0);
+ unsigned NumBitsPerElt = VT.getScalarSizeInBits();
+
+ // If the non-ZERO_UNDEF version is supported we can use that instead.
+ if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
+ isOperationLegalOrCustom(ISD::CTTZ, VT)) {
+ Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
+ return true;
+ }
+
+ // If the ZERO_UNDEF version is supported use that and handle the zero case.
+ if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
+ EVT SetCCVT =
+ getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
+ SDValue Zero = DAG.getConstant(0, dl, VT);
+ SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
+ Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
+ DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
+ return true;
+ }
+
+ // Only expand vector types if we have the appropriate vector bit operations.
+ if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
+ (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
+ !isOperationLegalOrCustom(ISD::CTLZ, VT)) ||
+ !isOperationLegalOrCustom(ISD::SUB, VT) ||
+ !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
+ !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
+ return false;
+
+ // for now, we use: { return popcount(~x & (x - 1)); }
+ // unless the target has ctlz but not ctpop, in which case we use:
+ // { return 32 - nlz(~x & (x-1)); }
+ // Ref: "Hacker's Delight" by Henry Warren
+ SDValue Tmp = DAG.getNode(
+ ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
+ DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
+
+ // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
+ if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
+ Result =
+ DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
+ DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
+ return true;
+ }
+
+ Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
+ return true;
+}
+
SDValue TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
SelectionDAG &DAG) const {
SDLoc SL(LD);
More information about the llvm-commits
mailing list