[llvm] r354226 - [SelectionDAG] Extract [US]MULO expansion into TL method; NFC
Nikita Popov via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 17 09:40:47 PST 2019
Author: nikic
Date: Sun Feb 17 09:40:47 2019
New Revision: 354226
URL: http://llvm.org/viewvc/llvm-project?rev=354226&view=rev
Log:
[SelectionDAG] Extract [US]MULO expansion into TL method; NFC
In preparation for supporting vector expansion.
Add an isPostTypeLegalization flag to makeLibCall(), because this
expansion relies on the legalized form using MERGE_VALUES. Drop
the corresponding variant of ExpandLibCall, which is no longer used.
Differential Revision: https://reviews.llvm.org/D58006
Modified:
llvm/trunk/include/llvm/CodeGen/TargetLowering.h
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Modified: llvm/trunk/include/llvm/CodeGen/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/TargetLowering.h?rev=354226&r1=354225&r2=354226&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/CodeGen/TargetLowering.h Sun Feb 17 09:40:47 2019
@@ -2858,11 +2858,10 @@ public:
/// Returns a pair of (return value, chain).
/// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC.
- std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
- EVT RetVT, ArrayRef<SDValue> Ops,
- bool isSigned, const SDLoc &dl,
- bool doesNotReturn = false,
- bool isReturnValueUsed = true) const;
+ std::pair<SDValue, SDValue> makeLibCall(
+ SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops,
+ bool isSigned, const SDLoc &dl, bool doesNotReturn = false,
+ bool isReturnValueUsed = true, bool isPostTypeLegalization = false) const;
/// Check whether parameters to a call that are passed in callee saved
/// registers are the same as from the calling function. This needs to be
@@ -3872,6 +3871,10 @@ public:
/// integers as its arguments.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
+ /// Method for building the DAG expansion of ISD::[US]MULO, returning the two
+ /// result values as a pair.
+ std::pair<SDValue, SDValue> expandMULO(SDNode *Node, SelectionDAG &DAG) const;
+
//===--------------------------------------------------------------------===//
// Instruction Emitting Hooks
//
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=354226&r1=354225&r2=354226&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Sun Feb 17 09:40:47 2019
@@ -136,8 +136,6 @@ private:
bool &NeedInvert, const SDLoc &dl);
SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
- SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
- unsigned NumOps, bool isSigned, const SDLoc &dl);
std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
SDNode *Node, bool isSigned);
@@ -2049,41 +2047,6 @@ SDValue SelectionDAGLegalize::ExpandLibC
return CallInfo.first;
}
-/// Generate a libcall taking the given operands as arguments
-/// and returning a result of type RetVT.
-SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
- const SDValue *Ops, unsigned NumOps,
- bool isSigned, const SDLoc &dl) {
- TargetLowering::ArgListTy Args;
- Args.reserve(NumOps);
-
- TargetLowering::ArgListEntry Entry;
- for (unsigned i = 0; i != NumOps; ++i) {
- Entry.Node = Ops[i];
- Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
- Entry.IsSExt = isSigned;
- Entry.IsZExt = !isSigned;
- Args.push_back(Entry);
- }
- SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
- TLI.getPointerTy(DAG.getDataLayout()));
-
- Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
-
- TargetLowering::CallLoweringInfo CLI(DAG);
- CLI.setDebugLoc(dl)
- .setChain(DAG.getEntryNode())
- .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
- std::move(Args))
- .setSExtResult(isSigned)
- .setZExtResult(!isSigned)
- .setIsPostTypeLegalization(true);
-
- std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
-
- return CallInfo.first;
-}
-
// Expand a node into a call to a libcall. Similar to
// ExpandLibCall except that the first operand is the in-chain.
std::pair<SDValue, SDValue>
@@ -3358,116 +3321,9 @@ bool SelectionDAGLegalize::ExpandNode(SD
}
case ISD::UMULO:
case ISD::SMULO: {
- EVT VT = Node->getValueType(0);
- EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
- SDValue LHS = Node->getOperand(0);
- SDValue RHS = Node->getOperand(1);
- SDValue BottomHalf;
- SDValue TopHalf;
- static const unsigned Ops[2][3] =
- { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
- { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
- bool isSigned = Node->getOpcode() == ISD::SMULO;
- if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
- BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
- TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
- } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
- BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
- RHS);
- TopHalf = BottomHalf.getValue(1);
- } else if (TLI.isTypeLegal(WideVT)) {
- LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
- RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
- Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
- BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
- DAG.getIntPtrConstant(0, dl));
- TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
- DAG.getIntPtrConstant(1, dl));
- } else {
- // We can fall back to a libcall with an illegal type for the MUL if we
- // have a libcall big enough.
- // Also, we can fall back to a division in some cases, but that's a big
- // performance hit in the general case.
- RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
- if (WideVT == MVT::i16)
- LC = RTLIB::MUL_I16;
- else if (WideVT == MVT::i32)
- LC = RTLIB::MUL_I32;
- else if (WideVT == MVT::i64)
- LC = RTLIB::MUL_I64;
- else if (WideVT == MVT::i128)
- LC = RTLIB::MUL_I128;
- assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
-
- SDValue HiLHS;
- SDValue HiRHS;
- if (isSigned) {
- // The high part is obtained by SRA'ing all but one of the bits of low
- // part.
- unsigned LoSize = VT.getSizeInBits();
- HiLHS =
- DAG.getNode(ISD::SRA, dl, VT, LHS,
- DAG.getConstant(LoSize - 1, dl,
- TLI.getPointerTy(DAG.getDataLayout())));
- HiRHS =
- DAG.getNode(ISD::SRA, dl, VT, RHS,
- DAG.getConstant(LoSize - 1, dl,
- TLI.getPointerTy(DAG.getDataLayout())));
- } else {
- HiLHS = DAG.getConstant(0, dl, VT);
- HiRHS = DAG.getConstant(0, dl, VT);
- }
-
- // Here we're passing the 2 arguments explicitly as 4 arguments that are
- // pre-lowered to the correct types. This all depends upon WideVT not
- // being a legal type for the architecture and thus has to be split to
- // two arguments.
- SDValue Ret;
- if (DAG.getDataLayout().isLittleEndian()) {
- // Halves of WideVT are packed into registers in different order
- // depending on platform endianness. This is usually handled by
- // the C calling convention, but we can't defer to it in
- // the legalizer.
- SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
- Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
- } else {
- SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
- Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
- }
- assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
- "Ret value is a collection of constituent nodes holding result.");
- if (DAG.getDataLayout().isLittleEndian()) {
- // Same as above.
- BottomHalf = Ret.getOperand(0);
- TopHalf = Ret.getOperand(1);
- } else {
- BottomHalf = Ret.getOperand(1);
- TopHalf = Ret.getOperand(0);
- }
- }
-
- if (isSigned) {
- Tmp1 = DAG.getConstant(
- VT.getSizeInBits() - 1, dl,
- TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
- Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
- TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
- ISD::SETNE);
- } else {
- TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
- DAG.getConstant(0, dl, VT), ISD::SETNE);
- }
-
- // Truncate the result if SetCC returns a larger type than needed.
- EVT RType = Node->getValueType(1);
- if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
- TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
-
- assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
- "Unexpected result type for S/UMULO legalization");
-
- Results.push_back(BottomHalf);
- Results.push_back(TopHalf);
+ auto Pair = TLI.expandMULO(Node, DAG);
+ Results.push_back(Pair.first);
+ Results.push_back(Pair.second);
break;
}
case ISD::BUILD_PAIR: {
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp?rev=354226&r1=354225&r2=354226&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp Sun Feb 17 09:40:47 2019
@@ -120,7 +120,8 @@ std::pair<SDValue, SDValue>
TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
ArrayRef<SDValue> Ops, bool isSigned,
const SDLoc &dl, bool doesNotReturn,
- bool isReturnValueUsed) const {
+ bool isReturnValueUsed,
+ bool isPostTypeLegalization) const {
TargetLowering::ArgListTy Args;
Args.reserve(Ops.size());
@@ -146,6 +147,7 @@ TargetLowering::makeLibCall(SelectionDAG
.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
.setNoReturn(doesNotReturn)
.setDiscardResult(!isReturnValueUsed)
+ .setIsPostTypeLegalization(isPostTypeLegalization)
.setSExtResult(signExtend)
.setZExtResult(!signExtend);
return LowerCallTo(CLI);
@@ -5519,3 +5521,121 @@ TargetLowering::expandFixedPointMul(SDNo
return DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
DAG.getConstant(Scale, dl, ShiftTy));
}
+
+std::pair<SDValue, SDValue> TargetLowering::expandMULO(
+ SDNode *Node, SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ EVT VT = Node->getValueType(0);
+ EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
+ SDValue LHS = Node->getOperand(0);
+ SDValue RHS = Node->getOperand(1);
+ SDValue BottomHalf;
+ SDValue TopHalf;
+ static const unsigned Ops[2][3] =
+ { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
+ { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
+ bool isSigned = Node->getOpcode() == ISD::SMULO;
+ if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
+ BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+ TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
+ } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
+ BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
+ RHS);
+ TopHalf = BottomHalf.getValue(1);
+ } else if (isTypeLegal(WideVT)) {
+ LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
+ RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
+ SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
+ BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Mul,
+ DAG.getIntPtrConstant(0, dl));
+ TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Mul,
+ DAG.getIntPtrConstant(1, dl));
+ } else {
+ // We can fall back to a libcall with an illegal type for the MUL if we
+ // have a libcall big enough.
+ // Also, we can fall back to a division in some cases, but that's a big
+ // performance hit in the general case.
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+ if (WideVT == MVT::i16)
+ LC = RTLIB::MUL_I16;
+ else if (WideVT == MVT::i32)
+ LC = RTLIB::MUL_I32;
+ else if (WideVT == MVT::i64)
+ LC = RTLIB::MUL_I64;
+ else if (WideVT == MVT::i128)
+ LC = RTLIB::MUL_I128;
+ assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
+
+ SDValue HiLHS;
+ SDValue HiRHS;
+ if (isSigned) {
+ // The high part is obtained by SRA'ing all but one of the bits of low
+ // part.
+ unsigned LoSize = VT.getSizeInBits();
+ HiLHS =
+ DAG.getNode(ISD::SRA, dl, VT, LHS,
+ DAG.getConstant(LoSize - 1, dl,
+ getPointerTy(DAG.getDataLayout())));
+ HiRHS =
+ DAG.getNode(ISD::SRA, dl, VT, RHS,
+ DAG.getConstant(LoSize - 1, dl,
+ getPointerTy(DAG.getDataLayout())));
+ } else {
+ HiLHS = DAG.getConstant(0, dl, VT);
+ HiRHS = DAG.getConstant(0, dl, VT);
+ }
+
+ // Here we're passing the 2 arguments explicitly as 4 arguments that are
+ // pre-lowered to the correct types. This all depends upon WideVT not
+ // being a legal type for the architecture and thus has to be split to
+ // two arguments.
+ SDValue Ret;
+ if (DAG.getDataLayout().isLittleEndian()) {
+ // Halves of WideVT are packed into registers in different order
+ // depending on platform endianness. This is usually handled by
+ // the C calling convention, but we can't defer to it in
+ // the legalizer.
+ SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
+ Ret = makeLibCall(DAG, LC, WideVT, Args, isSigned, dl,
+ /* doesNotReturn */ false, /* isReturnValueUsed */ true,
+ /* isPostTypeLegalization */ true).first;
+ } else {
+ SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
+ Ret = makeLibCall(DAG, LC, WideVT, Args, isSigned, dl,
+ /* doesNotReturn */ false, /* isReturnValueUsed */ true,
+ /* isPostTypeLegalization */ true).first;
+ }
+ assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
+ "Ret value is a collection of constituent nodes holding result.");
+ if (DAG.getDataLayout().isLittleEndian()) {
+ // Same as above.
+ BottomHalf = Ret.getOperand(0);
+ TopHalf = Ret.getOperand(1);
+ } else {
+ BottomHalf = Ret.getOperand(1);
+ TopHalf = Ret.getOperand(0);
+ }
+ }
+
+ EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ if (isSigned) {
+ SDValue ShiftAmt = DAG.getConstant(
+ VT.getSizeInBits() - 1, dl,
+ getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
+ SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
+ TopHalf = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
+ } else {
+ TopHalf = DAG.getSetCC(dl, SetCCVT, TopHalf,
+ DAG.getConstant(0, dl, VT), ISD::SETNE);
+ }
+
+ // Truncate the result if SetCC returns a larger type than needed.
+ EVT RType = Node->getValueType(1);
+ if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
+ TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
+
+ assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
+ "Unexpected result type for S/UMULO legalization");
+
+ return std::make_pair(BottomHalf, TopHalf);
+}
More information about the llvm-commits
mailing list