[llvm] r349025 - [TargetLowering] Add ISD::ROTL/ROTR vector expansion
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 13 03:20:48 PST 2018
Author: rksimon
Date: Thu Dec 13 03:20:48 2018
New Revision: 349025
URL: http://llvm.org/viewvc/llvm-project?rev=349025&view=rev
Log:
[TargetLowering] Add ISD::ROTL/ROTR vector expansion
Move existing rotation expansion code into TargetLowering and set it up for vectors as well.
Ideally this would share more of the funnel shift expansion, but we handle the shift amount modulo quite differently at the moment.
Begun removing x86 vector rotate custom lowering to use the expansion.
Modified:
llvm/trunk/include/llvm/CodeGen/TargetLowering.h
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
Modified: llvm/trunk/include/llvm/CodeGen/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/TargetLowering.h?rev=349025&r1=349024&r2=349025&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/CodeGen/TargetLowering.h Thu Dec 13 03:20:48 2018
@@ -3727,6 +3727,12 @@ public:
/// \returns True, if the expansion was successful, false otherwise
bool expandFunnelShift(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+ /// Expand rotations.
+ /// \param N Node to expand
+ /// \param Result output after conversion
+ /// \returns True, if the expansion was successful, false otherwise
+ bool expandROT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+
/// Expand float(f32) to SINT(i64) conversion
/// \param N Node to expand
/// \param Result output after conversion
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=349025&r1=349024&r2=349025&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Thu Dec 13 03:20:48 2018
@@ -3275,6 +3275,11 @@ bool SelectionDAGLegalize::ExpandNode(SD
if (TLI.expandFunnelShift(Node, Tmp1, DAG))
Results.push_back(Tmp1);
break;
+ case ISD::ROTL:
+ case ISD::ROTR:
+ if (TLI.expandROT(Node, Tmp1, DAG))
+ Results.push_back(Tmp1);
+ break;
case ISD::SADDSAT:
case ISD::UADDSAT:
case ISD::SSUBSAT:
@@ -3725,46 +3730,6 @@ bool SelectionDAGLegalize::ExpandNode(SD
ReplaceNode(SDValue(Node, 0), Result);
break;
}
- case ISD::ROTL:
- case ISD::ROTR: {
- bool IsLeft = Node->getOpcode() == ISD::ROTL;
- SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
- EVT ResVT = Node->getValueType(0);
- EVT OpVT = Op0.getValueType();
- assert(OpVT == ResVT &&
- "The result and the operand types of rotate should match");
- EVT ShVT = Op1.getValueType();
- SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
-
- // If a rotate in the other direction is legal, use it.
- unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
- if (TLI.isOperationLegal(RevRot, ResVT)) {
- SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
- Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
- break;
- }
-
- // Otherwise,
- // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
- // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
- //
- assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&
- "Expecting the type bitwidth to be a power of 2");
- unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
- unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
- SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
- Width, DAG.getConstant(1, dl, ShVT));
- SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
- SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
- SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
-
- SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
- DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
- DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
- Results.push_back(Or);
- break;
- }
-
case ISD::GLOBAL_OFFSET_TABLE:
case ISD::GlobalAddress:
case ISD::GlobalTLSAddress:
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp?rev=349025&r1=349024&r2=349025&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp Thu Dec 13 03:20:48 2018
@@ -133,6 +133,7 @@ class VectorLegalizer {
SDValue ExpandCTLZ(SDValue Op);
SDValue ExpandCTTZ(SDValue Op);
SDValue ExpandFunnelShift(SDValue Op);
+ SDValue ExpandROT(SDValue Op);
SDValue ExpandFMINNUM_FMAXNUM(SDValue Op);
SDValue ExpandStrictFPOp(SDValue Op);
@@ -759,6 +760,9 @@ SDValue VectorLegalizer::Expand(SDValue
case ISD::FSHL:
case ISD::FSHR:
return ExpandFunnelShift(Op);
+ case ISD::ROTL:
+ case ISD::ROTR:
+ return ExpandROT(Op);
case ISD::FMINNUM:
case ISD::FMAXNUM:
return ExpandFMINNUM_FMAXNUM(Op);
@@ -1165,6 +1169,14 @@ SDValue VectorLegalizer::ExpandFunnelShi
return Result;
return DAG.UnrollVectorOp(Op.getNode());
+}
+
+SDValue VectorLegalizer::ExpandROT(SDValue Op) {
+ SDValue Result;
+ if (TLI.expandROT(Op.getNode(), Result, DAG))
+ return Result;
+
+ return DAG.UnrollVectorOp(Op.getNode());
}
SDValue VectorLegalizer::ExpandFMINNUM_FMAXNUM(SDValue Op) {
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp?rev=349025&r1=349024&r2=349025&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/TargetLowering.cpp Thu Dec 13 03:20:48 2018
@@ -4213,6 +4213,51 @@ bool TargetLowering::expandFunnelShift(S
return true;
}
+// TODO: Merge with expandFunnelShift.
+bool TargetLowering::expandROT(SDNode *Node, SDValue &Result,
+ SelectionDAG &DAG) const {
+ EVT VT = Node->getValueType(0);
+ unsigned EltSizeInBits = VT.getScalarSizeInBits();
+ bool IsLeft = Node->getOpcode() == ISD::ROTL;
+ SDValue Op0 = Node->getOperand(0);
+ SDValue Op1 = Node->getOperand(1);
+ SDLoc DL(SDValue(Node, 0));
+
+ EVT ShVT = Op1.getValueType();
+ SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
+
+ // If a rotate in the other direction is legal, use it.
+ unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
+ if (isOperationLegal(RevRot, VT)) {
+ SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
+ Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
+ return true;
+ }
+
+ if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
+ !isOperationLegalOrCustom(ISD::SRL, VT) ||
+ !isOperationLegalOrCustom(ISD::SUB, VT) ||
+ !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
+ !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
+ return false;
+
+ // Otherwise,
+ // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
+ // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
+ //
+ assert(isPowerOf2_32(EltSizeInBits) && EltSizeInBits > 1 &&
+ "Expecting the type bitwidth to be a power of 2");
+ unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
+ unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
+ SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
+ SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
+ SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
+ SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
+ Result = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ShOpc, DL, VT, Op0, And0),
+ DAG.getNode(HsOpc, DL, VT, Op0, And1));
+ return true;
+}
+
bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
SelectionDAG &DAG) const {
SDValue Src = Node->getOperand(0);
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=349025&r1=349024&r2=349025&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Thu Dec 13 03:20:48 2018
@@ -24815,19 +24815,13 @@ static SDValue LowerRotate(SDValue Op, c
"Only vXi32/vXi16/vXi8 vector rotates supported");
// Rotate by an uniform constant - expand back to shifts.
- // TODO - legalizers should be able to handle this.
- if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt)) {
+ if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt))
if (auto *RotateConst = BVAmt->getConstantSplatNode()) {
uint64_t RotateAmt = RotateConst->getAPIntValue().urem(EltSizeInBits);
if (RotateAmt == 0)
return R;
-
- SDValue AmtR = DAG.getConstant(EltSizeInBits - RotateAmt, DL, VT);
- SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
- SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
- return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
+ return SDValue();
}
- }
// TODO: ISD::ROT* uses modulo rotate amounts, we need to handle this.
More information about the llvm-commits
mailing list