[llvm-commits] [llvm] r90108 - in /llvm/trunk: include/llvm/CodeGen/SelectionDAG.h lib/CodeGen/SelectionDAG/LegalizeTypes.cpp lib/CodeGen/SelectionDAG/LegalizeTypes.h lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp lib/CodeGen/SelectionDAG/SelectionDAG.cpp lib/Target/X86/X86ISelLowering.cpp
Chris Lattner
clattner at apple.com
Sun Nov 29 20:21:11 PST 2009
On Nov 29, 2009, at 6:42 PM, Mon P Wang wrote:
> Author: wangmp
> Date: Sun Nov 29 20:42:02 2009
> New Revision: 90108
>
> URL: http://llvm.org/viewvc/llvm-project?rev=90108&view=rev
> Log:
> Added support to allow clients to custom widen. For X86, custom widen vectors for
> divide/remainder since these operations can trap by unroll them and adding undefs
> for the resulting vector.
Hi Mon Ping,
For a vec3 divide, wouldn't it be more efficient to shuffle one of the three valid elements of the divisor into the fourth element, then do a vec4 divide? I'd think that one vec4 divide + shuffle would be faster than a bunch of extracts, scalar divs then reconstructions.
-Chris
>
> Modified:
> llvm/trunk/include/llvm/CodeGen/SelectionDAG.h
> llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
> llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h
> llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
> llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
> llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
> llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
>
> Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAG.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/include/llvm/CodeGen/SelectionDAG.h (original)
> +++ llvm/trunk/include/llvm/CodeGen/SelectionDAG.h Sun Nov 29 20:42:02 2009
> @@ -882,6 +882,14 @@
> /// element of the result of the vector shuffle.
> SDValue getShuffleScalarElt(const ShuffleVectorSDNode *N, unsigned Idx);
>
> + /// UnrollVectorOp - Utility function used by legalize and lowering to
> + /// "unroll" a vector operation by splitting out the scalars and operating
> + /// on each element individually. If the ResNE is 0, fully unroll the vector
> + /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
> + /// If the ResNE is greater than the width of the vector op, unroll the
> + /// vector op and fill the end of the resulting vector with UNDEFS.
> + SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
> +
> private:
> bool RemoveNodeFromCSEMaps(SDNode *N);
> void AddModifiedNodeToCSEMaps(SDNode *N, DAGUpdateListener *UpdateListener);
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp Sun Nov 29 20:42:02 2009
> @@ -907,6 +907,29 @@
> return true;
> }
>
> +
> +/// CustomWidenLowerNode - Widen the node's results with custom code provided
> +/// by the target and return "true", or do nothing and return "false".
> +bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) {
> + // See if the target wants to custom lower this node.
> + if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom)
> + return false;
> +
> + SmallVector<SDValue, 8> Results;
> + TLI.ReplaceNodeResults(N, Results, DAG);
> +
> + if (Results.empty())
> + // The target didn't want to custom widen lower its result after all.
> + return false;
> +
> + // Update the widening map.
> + assert(Results.size() == N->getNumValues() &&
> + "Custom lowering returned the wrong number of results!");
> + for (unsigned i = 0, e = Results.size(); i != e; ++i)
> + SetWidenedVector(SDValue(N, i), Results[i]);
> + return true;
> +}
> +
> /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
> /// which is split into two not necessarily identical pieces.
> void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) {
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h Sun Nov 29 20:42:02 2009
> @@ -188,6 +188,7 @@
> SDValue BitConvertVectorToIntegerVector(SDValue Op);
> SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
> bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
> + bool CustomWidenLowerNode(SDNode *N, EVT VT);
> SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index);
> SDValue JoinIntegers(SDValue Lo, SDValue Hi);
> SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp Sun Nov 29 20:42:02 2009
> @@ -54,9 +54,6 @@
> SDValue LegalizeOp(SDValue Op);
> // Assuming the node is legal, "legalize" the results
> SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
> - // Implements unrolling a generic vector operation, i.e. turning it into
> - // scalar operations.
> - SDValue UnrollVectorOp(SDValue Op);
> // Implements unrolling a VSETCC.
> SDValue UnrollVSETCC(SDValue Op);
> // Implements expansion for FNEG; falls back to UnrollVectorOp if FSUB
> @@ -211,7 +208,7 @@
> else if (Node->getOpcode() == ISD::VSETCC)
> Result = UnrollVSETCC(Op);
> else
> - Result = UnrollVectorOp(Op);
> + Result = DAG.UnrollVectorOp(Op.getNode());
> break;
> }
>
> @@ -256,7 +253,7 @@
> return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
> Zero, Op.getOperand(0));
> }
> - return UnrollVectorOp(Op);
> + return DAG.UnrollVectorOp(Op.getNode());
> }
>
> SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
> @@ -282,56 +279,6 @@
> return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElems);
> }
>
> -/// UnrollVectorOp - We know that the given vector has a legal type, however
> -/// the operation it performs is not legal, and the target has requested that
> -/// the operation be expanded. "Unroll" the vector, splitting out the scalars
> -/// and operating on each element individually.
> -SDValue VectorLegalizer::UnrollVectorOp(SDValue Op) {
> - EVT VT = Op.getValueType();
> - assert(Op.getNode()->getNumValues() == 1 &&
> - "Can't unroll a vector with multiple results!");
> - unsigned NE = VT.getVectorNumElements();
> - EVT EltVT = VT.getVectorElementType();
> - DebugLoc dl = Op.getDebugLoc();
> -
> - SmallVector<SDValue, 8> Scalars;
> - SmallVector<SDValue, 4> Operands(Op.getNumOperands());
> - for (unsigned i = 0; i != NE; ++i) {
> - for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
> - SDValue Operand = Op.getOperand(j);
> - EVT OperandVT = Operand.getValueType();
> - if (OperandVT.isVector()) {
> - // A vector operand; extract a single element.
> - EVT OperandEltVT = OperandVT.getVectorElementType();
> - Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
> - OperandEltVT,
> - Operand,
> - DAG.getConstant(i, MVT::i32));
> - } else {
> - // A scalar operand; just use it as is.
> - Operands[j] = Operand;
> - }
> - }
> -
> - switch (Op.getOpcode()) {
> - default:
> - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT,
> - &Operands[0], Operands.size()));
> - break;
> - case ISD::SHL:
> - case ISD::SRA:
> - case ISD::SRL:
> - case ISD::ROTL:
> - case ISD::ROTR:
> - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0],
> - DAG.getShiftAmountOperand(Operands[1])));
> - break;
> - }
> - }
> -
> - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size());
> -}
> -
> }
>
> bool SelectionDAG::LegalizeVectors() {
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp Sun Nov 29 20:42:02 2009
> @@ -1118,8 +1118,12 @@
> DEBUG(errs() << "Widen node result " << ResNo << ": ";
> N->dump(&DAG);
> errs() << "\n");
> - SDValue Res = SDValue();
>
> + // See if the target wants to custom widen this node.
> + if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
> + return;
> +
> + SDValue Res = SDValue();
> switch (N->getOpcode()) {
> default:
> #ifndef NDEBUG
>
> Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (original)
> +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp Sun Nov 29 20:42:02 2009
> @@ -5807,6 +5807,66 @@
> N->dump(G);
> }
>
> +SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
> + assert(N->getNumValues() == 1 &&
> + "Can't unroll a vector with multiple results!");
> +
> + EVT VT = N->getValueType(0);
> + unsigned NE = VT.getVectorNumElements();
> + EVT EltVT = VT.getVectorElementType();
> + DebugLoc dl = N->getDebugLoc();
> +
> + SmallVector<SDValue, 8> Scalars;
> + SmallVector<SDValue, 4> Operands(N->getNumOperands());
> +
> + // If ResNE is 0, fully unroll the vector op.
> + if (ResNE == 0)
> + ResNE = NE;
> + else if (NE > ResNE)
> + NE = ResNE;
> +
> + unsigned i;
> + for (i= 0; i != NE; ++i) {
> + for (unsigned j = 0; j != N->getNumOperands(); ++j) {
> + SDValue Operand = N->getOperand(j);
> + EVT OperandVT = Operand.getValueType();
> + if (OperandVT.isVector()) {
> + // A vector operand; extract a single element.
> + EVT OperandEltVT = OperandVT.getVectorElementType();
> + Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
> + OperandEltVT,
> + Operand,
> + getConstant(i, MVT::i32));
> + } else {
> + // A scalar operand; just use it as is.
> + Operands[j] = Operand;
> + }
> + }
> +
> + switch (N->getOpcode()) {
> + default:
> + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
> + &Operands[0], Operands.size()));
> + break;
> + case ISD::SHL:
> + case ISD::SRA:
> + case ISD::SRL:
> + case ISD::ROTL:
> + case ISD::ROTR:
> + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
> + getShiftAmountOperand(Operands[1])));
> + break;
> + }
> + }
> +
> + for (; i < ResNE; ++i)
> + Scalars.push_back(getUNDEF(EltVT));
> +
> + return getNode(ISD::BUILD_VECTOR, dl,
> + EVT::getVectorVT(*getContext(), EltVT, ResNE),
> + &Scalars[0], Scalars.size());
> +}
> +
> void SelectionDAG::dump() const {
> errs() << "SelectionDAG has " << AllNodes.size() << " nodes:";
>
> @@ -5962,3 +6022,4 @@
> return false;
> return true;
> }
> +
>
> Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=90108&r1=90107&r2=90108&view=diff
>
> ==============================================================================
> --- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
> +++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Sun Nov 29 20:42:02 2009
> @@ -975,6 +975,19 @@
>
> computeRegisterProperties();
>
> + // Divide and reminder operations have no vector equivalent and can
> + // trap. Do a custom widening for these operations in which we never
> + // generate more divides/remainder than the original vector width.
> + for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
> + VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
> + if (!isTypeLegal((MVT::SimpleValueType)VT)) {
> + setOperationAction(ISD::SDIV, (MVT::SimpleValueType) VT, Custom);
> + setOperationAction(ISD::UDIV, (MVT::SimpleValueType) VT, Custom);
> + setOperationAction(ISD::SREM, (MVT::SimpleValueType) VT, Custom);
> + setOperationAction(ISD::UREM, (MVT::SimpleValueType) VT, Custom);
> + }
> + }
> +
> // FIXME: These should be based on subtarget info. Plus, the values should
> // be smaller when we are in optimizing for size mode.
> maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
> @@ -7170,6 +7183,14 @@
> Results.push_back(edx.getValue(1));
> return;
> }
> + case ISD::SDIV:
> + case ISD::UDIV:
> + case ISD::SREM:
> + case ISD::UREM: {
> + EVT WidenVT = getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
> + Results.push_back(DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()));
> + return;
> + }
> case ISD::ATOMIC_CMP_SWAP: {
> EVT T = N->getValueType(0);
> assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list