[llvm] 94a901a - [X86] Move LowerFunnelShift below LowerShift. NFC.
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 11 10:47:09 PST 2021
Author: Simon Pilgrim
Date: 2021-11-11T18:45:51Z
New Revision: 94a901a50ad0d927a7c32e638053958891754955
URL: https://github.com/llvm/llvm-project/commit/94a901a50ad0d927a7c32e638053958891754955
DIFF: https://github.com/llvm/llvm-project/commit/94a901a50ad0d927a7c32e638053958891754955.diff
LOG: [X86] Move LowerFunnelShift below LowerShift. NFC.
Makes it easier to reuse the various vector shift helpers defined above LowerShift
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 373a6619a0d12..b796f12baa345 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -20089,92 +20089,6 @@ static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
}
-static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
- MVT VT = Op.getSimpleValueType();
- assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
- "Unexpected funnel shift opcode!");
-
- SDLoc DL(Op);
- SDValue Op0 = Op.getOperand(0);
- SDValue Op1 = Op.getOperand(1);
- SDValue Amt = Op.getOperand(2);
-
- bool IsFSHR = Op.getOpcode() == ISD::FSHR;
-
- if (VT.isVector()) {
- assert(Subtarget.hasVBMI2() && "Expected VBMI2");
-
- if (IsFSHR)
- std::swap(Op0, Op1);
-
- // With AVX512, but not VLX we need to widen to get a 512-bit result type.
- if (!Subtarget.hasVLX() && !VT.is512BitVector()) {
- Op0 = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
- Op1 = widenSubVector(Op1, false, Subtarget, DAG, DL, 512);
- }
-
- SDValue Funnel;
- APInt APIntShiftAmt;
- MVT ResultVT = Op0.getSimpleValueType();
- if (X86::isConstantSplat(Amt, APIntShiftAmt)) {
- uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits());
- Funnel =
- DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, ResultVT, Op0,
- Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
- } else {
- if (!Subtarget.hasVLX() && !VT.is512BitVector())
- Amt = widenSubVector(Amt, false, Subtarget, DAG, DL, 512);
- Funnel = DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL,
- ResultVT, Op0, Op1, Amt);
- }
- if (!Subtarget.hasVLX() && !VT.is512BitVector())
- Funnel = extractSubVector(Funnel, 0, DAG, DL, VT.getSizeInBits());
- return Funnel;
- }
- assert(
- (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
- "Unexpected funnel shift type!");
-
- // Expand slow SHLD/SHRD cases if we are not optimizing for size.
- bool OptForSize = DAG.shouldOptForSize();
- bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow();
-
- // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
- // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
- if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&
- !isa<ConstantSDNode>(Amt)) {
- unsigned EltSizeInBits = VT.getScalarSizeInBits();
- SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
- SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
- Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
- Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
- Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
- SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
- Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
- if (IsFSHR) {
- Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
- } else {
- Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
- Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
- }
- return DAG.getZExtOrTrunc(Res, DL, VT);
- }
-
- if (VT == MVT::i8 || ExpandFunnel)
- return SDValue();
-
- // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
- if (VT == MVT::i16) {
- Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
- DAG.getConstant(15, DL, Amt.getValueType()));
- unsigned FSHOp = (IsFSHR ? X86ISD::FSHR : X86ISD::FSHL);
- return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
- }
-
- return Op;
-}
-
// Try to use a packed vector operation to handle i64 on 32-bit targets when
// AVX512DQ is enabled.
static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
@@ -29660,6 +29574,92 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
return SDValue();
}
+static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ MVT VT = Op.getSimpleValueType();
+ assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
+ "Unexpected funnel shift opcode!");
+
+ SDLoc DL(Op);
+ SDValue Op0 = Op.getOperand(0);
+ SDValue Op1 = Op.getOperand(1);
+ SDValue Amt = Op.getOperand(2);
+
+ bool IsFSHR = Op.getOpcode() == ISD::FSHR;
+
+ if (VT.isVector()) {
+ assert(Subtarget.hasVBMI2() && "Expected VBMI2");
+
+ if (IsFSHR)
+ std::swap(Op0, Op1);
+
+ // With AVX512, but not VLX we need to widen to get a 512-bit result type.
+ if (!Subtarget.hasVLX() && !VT.is512BitVector()) {
+ Op0 = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
+ Op1 = widenSubVector(Op1, false, Subtarget, DAG, DL, 512);
+ }
+
+ SDValue Funnel;
+ APInt APIntShiftAmt;
+ MVT ResultVT = Op0.getSimpleValueType();
+ if (X86::isConstantSplat(Amt, APIntShiftAmt)) {
+ uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits());
+ Funnel =
+ DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, ResultVT, Op0,
+ Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
+ } else {
+ if (!Subtarget.hasVLX() && !VT.is512BitVector())
+ Amt = widenSubVector(Amt, false, Subtarget, DAG, DL, 512);
+ Funnel = DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL,
+ ResultVT, Op0, Op1, Amt);
+ }
+ if (!Subtarget.hasVLX() && !VT.is512BitVector())
+ Funnel = extractSubVector(Funnel, 0, DAG, DL, VT.getSizeInBits());
+ return Funnel;
+ }
+ assert(
+ (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
+ "Unexpected funnel shift type!");
+
+ // Expand slow SHLD/SHRD cases if we are not optimizing for size.
+ bool OptForSize = DAG.shouldOptForSize();
+ bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow();
+
+ // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
+ // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
+ if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&
+ !isa<ConstantSDNode>(Amt)) {
+ unsigned EltSizeInBits = VT.getScalarSizeInBits();
+ SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
+ SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
+ Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
+ Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
+ Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
+ SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
+ Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
+ if (IsFSHR) {
+ Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
+ } else {
+ Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
+ Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
+ }
+ return DAG.getZExtOrTrunc(Res, DL, VT);
+ }
+
+ if (VT == MVT::i8 || ExpandFunnel)
+ return SDValue();
+
+ // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
+ if (VT == MVT::i16) {
+ Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
+ DAG.getConstant(15, DL, Amt.getValueType()));
+ unsigned FSHOp = (IsFSHR ? X86ISD::FSHR : X86ISD::FSHL);
+ return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
+ }
+
+ return Op;
+}
+
static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
MVT VT = Op.getSimpleValueType();
More information about the llvm-commits
mailing list