[llvm] r304404 - Do not legalize large setcc with setcce, introduce setcccarry and do it with usubo/setcccarry.
Amaury Sechet via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 1 04:14:17 PDT 2017
Author: deadalnix
Date: Thu Jun 1 06:14:17 2017
New Revision: 304404
URL: http://llvm.org/viewvc/llvm-project?rev=304404&view=rev
Log:
Do not legalize large setcc with setcce, introduce setcccarry and do it with usubo/setcccarry.
Summary:
This is a continuation of the work started in D29872 . Passing the carry down as a value rather than as a glue allows for further optimizations. Introducing setcccarry makes the use of addc/subc unecessary and we can start the removal process.
This patch only introduce the optimization strictly required to get the same level of optimization as was available before nothing more.
Reviewers: jyknight, nemanjai, mkuper, spatel, RKSimon, zvi, bkramer
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D33374
Modified:
llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h
llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
llvm/trunk/lib/Target/X86/X86ISelLowering.h
Modified: llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h (original)
+++ llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h Thu Jun 1 06:14:17 2017
@@ -410,12 +410,22 @@ namespace ISD {
/// then the result type must also be a vector type.
SETCC,
- /// Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but
+ /// Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, and
/// op #2 is a *carry value*. This operator checks the result of
/// "LHS - RHS - Carry", and can be used to compare two wide integers:
/// (setcce lhshi rhshi (subc lhslo rhslo) cc). Only valid for integers.
+ /// FIXME: This node is deprecated in favor of SETCCCARRY.
+ /// It is kept around for now to provide a smooth transition path
+ /// toward the use of SETCCCARRY and will eventually be removed.
SETCCE,
+ /// Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but
+ /// op #2 is a boolean indicating if there is an incoming carry. This
+ /// operator checks the result of "LHS - RHS - Carry", and can be used to
+ /// compare two wide integers: (setcce lhshi rhshi (subc lhslo rhslo) cc).
+ /// Only valid for integers.
+ SETCCCARRY,
+
/// SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded
/// integer shift operations. The operation ordering is:
/// [Lo,Hi] = op [LoLHS,HiLHS], Amt
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp Thu Jun 1 06:14:17 2017
@@ -280,6 +280,7 @@ namespace {
SDValue visitSELECT_CC(SDNode *N);
SDValue visitSETCC(SDNode *N);
SDValue visitSETCCE(SDNode *N);
+ SDValue visitSETCCCARRY(SDNode *N);
SDValue visitSIGN_EXTEND(SDNode *N);
SDValue visitZERO_EXTEND(SDNode *N);
SDValue visitANY_EXTEND(SDNode *N);
@@ -1457,6 +1458,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
case ISD::SETCCE: return visitSETCCE(N);
+ case ISD::SETCCCARRY: return visitSETCCCARRY(N);
case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
@@ -6836,6 +6838,19 @@ SDValue DAGCombiner::visitSETCCE(SDNode
return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
return SDValue();
+}
+
+SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
+ SDValue LHS = N->getOperand(0);
+ SDValue RHS = N->getOperand(1);
+ SDValue Carry = N->getOperand(2);
+ SDValue Cond = N->getOperand(3);
+
+ // If Carry is false, fold to a regular SETCC.
+ if (isNullConstant(Carry))
+ return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
+
+ return SDValue();
}
/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp Thu Jun 1 06:14:17 2017
@@ -2875,6 +2875,7 @@ bool DAGTypeLegalizer::ExpandIntegerOper
case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break;
+ case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
@@ -3009,14 +3010,16 @@ void DAGTypeLegalizer::IntegerExpandSetC
return;
}
- // Lower with SETCCE if the target supports it.
+ // Lower with SETCCE or SETCCCARRY if the target supports it.
+ EVT HiVT = LHSHi.getValueType();
+ EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
+ bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
+
// FIXME: Make all targets support this, then remove the other lowering.
- if (TLI.getOperationAction(
- ISD::SETCCE,
- TLI.getTypeToExpandTo(*DAG.getContext(), LHSLo.getValueType())) ==
- TargetLowering::Custom) {
- // SETCCE can detect < and >= directly. For > and <=, flip operands and
- // condition code.
+ if (HasSETCCCARRY ||
+ TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom) {
+ // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip
+ // operands and condition code.
bool FlipOperands = false;
switch (CCCode) {
case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
@@ -3030,27 +3033,28 @@ void DAGTypeLegalizer::IntegerExpandSetC
std::swap(LHSHi, RHSHi);
}
// Perform a wide subtraction, feeding the carry from the low part into
- // SETCCE. The SETCCE operation is essentially looking at the high part of
- // the result of LHS - RHS. It is negative iff LHS < RHS. It is zero or
- // positive iff LHS >= RHS.
- SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
- SDValue LowCmp = DAG.getNode(ISD::SUBC, dl, VTList, LHSLo, RHSLo);
- SDValue Res =
- DAG.getNode(ISD::SETCCE, dl, getSetCCResultType(LHSLo.getValueType()),
- LHSHi, RHSHi, LowCmp.getValue(1), DAG.getCondCode(CCCode));
+ // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially
+ // looking at the high part of the result of LHS - RHS. It is negative
+ // iff LHS < RHS. It is zero or positive iff LHS >= RHS.
+ EVT LoVT = LHSLo.getValueType();
+ SDVTList VTList = DAG.getVTList(
+ LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue);
+ SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl,
+ VTList, LHSLo, RHSLo);
+ SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl,
+ getSetCCResultType(HiVT), LHSHi, RHSHi,
+ LowCmp.getValue(1), DAG.getCondCode(CCCode));
NewLHS = Res;
NewRHS = SDValue();
return;
}
- NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
- LHSHi, RHSHi, ISD::SETEQ, false,
- DagCombineInfo, dl);
+ NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
+ false, DagCombineInfo, dl);
if (!NewLHS.getNode())
- NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
- LHSHi, RHSHi, ISD::SETEQ);
- NewLHS = DAG.getSelect(dl, LoCmp.getValueType(),
- NewLHS, LoCmp, HiCmp);
+ NewLHS =
+ DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
+ NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
NewRHS = SDValue();
}
@@ -3103,8 +3107,8 @@ SDValue DAGTypeLegalizer::ExpandIntOp_SE
}
// Otherwise, update N to have the operands specified.
- return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
- DAG.getCondCode(CCCode)), 0);
+ return SDValue(
+ DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
}
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
@@ -3125,6 +3129,24 @@ SDValue DAGTypeLegalizer::ExpandIntOp_SE
LowCmp.getValue(1), Cond);
}
+SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
+ SDValue LHS = N->getOperand(0);
+ SDValue RHS = N->getOperand(1);
+ SDValue Carry = N->getOperand(2);
+ SDValue Cond = N->getOperand(3);
+ SDLoc dl = SDLoc(N);
+
+ SDValue LHSLo, LHSHi, RHSLo, RHSHi;
+ GetExpandedInteger(LHS, LHSLo, LHSHi);
+ GetExpandedInteger(RHS, RHSLo, RHSHi);
+
+ // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
+ SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
+ SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
+ return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
+ LowCmp.getValue(1), Cond);
+}
+
SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
// The value being shifted is legal, but the shift amount is too big.
// It follows that either the result of the shift is undefined, or the
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h Thu Jun 1 06:14:17 2017
@@ -381,6 +381,7 @@ private:
SDValue ExpandIntOp_SELECT_CC(SDNode *N);
SDValue ExpandIntOp_SETCC(SDNode *N);
SDValue ExpandIntOp_SETCCE(SDNode *N);
+ SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
SDValue ExpandIntOp_Shift(SDNode *N);
SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp Thu Jun 1 06:14:17 2017
@@ -214,6 +214,7 @@ std::string SDNode::getOperationName(con
case ISD::FPOWI: return "fpowi";
case ISD::SETCC: return "setcc";
case ISD::SETCCE: return "setcce";
+ case ISD::SETCCCARRY: return "setcccarry";
case ISD::SELECT: return "select";
case ISD::VSELECT: return "vselect";
case ISD::SELECT_CC: return "select_cc";
Modified: llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp (original)
+++ llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp Thu Jun 1 06:14:17 2017
@@ -927,6 +927,7 @@ void TargetLoweringBase::initActions() {
// ADDCARRY operations default to expand
setOperationAction(ISD::ADDCARRY, VT, Expand);
setOperationAction(ISD::SUBCARRY, VT, Expand);
+ setOperationAction(ISD::SETCCCARRY, VT, Expand);
// These default to Expand so they will be expanded to CTLZ/CTTZ by default.
setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Thu Jun 1 06:14:17 2017
@@ -1578,6 +1578,7 @@ X86TargetLowering::X86TargetLowering(con
// Support carry in as value rather than glue.
setOperationAction(ISD::ADDCARRY, VT, Custom);
setOperationAction(ISD::SUBCARRY, VT, Custom);
+ setOperationAction(ISD::SETCCCARRY, VT, Custom);
}
if (!Subtarget.is64Bit()) {
@@ -17413,6 +17414,30 @@ SDValue X86TargetLowering::LowerSETCCE(S
return SetCC;
}
+SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
+ SDValue LHS = Op.getOperand(0);
+ SDValue RHS = Op.getOperand(1);
+ SDValue Carry = Op.getOperand(2);
+ SDValue Cond = Op.getOperand(3);
+ SDLoc DL(Op);
+
+ assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
+ X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
+
+ // Recreate the carry if needed.
+ EVT CarryVT = Carry.getValueType();
+ APInt NegOne = APInt::getAllOnesValue(CarryVT.getScalarSizeInBits());
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
+ Carry, DAG.getConstant(NegOne, DL, CarryVT));
+
+ SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
+ SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
+ SDValue SetCC = getSETCC(CC, Cmp.getValue(1), DL, DAG);
+ if (Op.getSimpleValueType() == MVT::i1)
+ return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
+ return SetCC;
+}
+
/// Return true if opcode is a X86 logical comparison.
static bool isX86LogicalCmp(SDValue Op) {
unsigned Opc = Op.getOpcode();
@@ -23782,6 +23807,7 @@ SDValue X86TargetLowering::LowerOperatio
case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
case ISD::SETCC: return LowerSETCC(Op, DAG);
case ISD::SETCCE: return LowerSETCCE(Op, DAG);
+ case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
case ISD::SELECT: return LowerSELECT(Op, DAG);
case ISD::BRCOND: return LowerBRCOND(Op, DAG);
case ISD::JumpTable: return LowerJumpTable(Op, DAG);
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.h?rev=304404&r1=304403&r2=304404&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.h (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.h Thu Jun 1 06:14:17 2017
@@ -1164,6 +1164,7 @@ namespace llvm {
SelectionDAG &DAG) const;
SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSETCCE(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
More information about the llvm-commits
mailing list