[llvm] [X86] Lower vector integer division and remainder through float division (PR #205263)

[Phoebe Wang via llvm-commits]

================
@@ -50564,6 +50572,200 @@ static SDValue combineMulToPMADD52(SDNode *N, const SDLoc &DL,
   return SDValue();
 }
 
+// x86 has no vector integer divide instructions. Lower vector
+// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
+// scalar hardware divides.
+static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
+                                TargetLowering::DAGCombinerInfo &DCI,
+                                const X86Subtarget &Subtarget) {
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  // Run before the legalizer expands the division.
+  if (!VT.isVector() || !Subtarget.hasSSE2() || !DCI.isBeforeLegalizeOps())
+    return SDValue();
+
+  SDValue Dividend = N->getOperand(0);
+  SDValue Divisor = N->getOperand(1);
+  unsigned Opc = N->getOpcode();
+
+  // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+  if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
+      Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
+    Opc = ISD::getUnmaskedBinOpOpcode(Opc);
+  bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
+  bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+
+  // If the result is only read back as scalar extracts, scalarization computes
+  // just the demanded lanes.
+  if (all_of(N->users(), [](const SDNode *U) {
+        return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
+      }))
+    return SDValue();
+
+  // Magic multiply lowers constant divisors cheaper than a divide.
+  if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
+    return SDValue();
+
+  unsigned EltBits = VT.getScalarSizeInBits();
+  auto FitsFP = [&](SDValue V, const fltSemantics &Sem) {
+    unsigned Precision = APFloat::semanticsPrecision(Sem);
+    return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
+                    : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
+  };
+  auto BothFitFP = [&](const fltSemantics &Sem) {
+    return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
+  };
+  // i64 needs the qq converts which is AVX512DQ only
+  bool NarrowI64 = EltBits == 64 && Subtarget.hasDQI() &&
+                   Subtarget.useAVX512Regs() &&
+                   BothFitFP(APFloat::IEEEdouble());
+
+  // i8/i16/i32 and narrow value i64: the operands fit the float mantissa
+  // exactly so one float divide recovers the exact quotient.
+  if (EltBits <= 32 || NarrowI64) {
+    // f32 recovers the quotient exactly when both operands fit in 24 bits
+    MVT FPSclVT = MVT::f64;
+    if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
+      FPSclVT = MVT::f32;
+    EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+
+    // Nothing will split an illegal FP type after type legalization.
+    if (!DCI.isBeforeLegalize() &&
+        !DAG.getTargetLoweringInfo().isTypeLegal(FPVT))
+      return SDValue();
+
+    bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
+        Attribute::StrictFP);
+    if (IsStrict) {
+      // No SAE below 512-bit AVX512
+      if (!Subtarget.useAVX512Regs())
+        return SDValue();
+      // More lanes than one zmm divide can hold so split the divide.
+      if (FPVT.getSizeInBits() > 512)
+        return splitVectorIntBinary(SDValue(N, 0), DAG, DL);
+    } else if (!IsSigned && VT.getScalarSizeInBits() == 32 &&
+               !Subtarget.hasAVX2()) {
+      // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
+      // for latency and code size before AVX2.
+      return SDValue();
+    }
+
+    unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+    SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
+    SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
+    SDValue Q;
+    if (IsStrict) {
+      // The converts are exact so only the divide and the truncate can
+      // raise flags.
+      unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
+      MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
+      // Narrow i64 quotients can pass 2^31 so the f64 tier lands on i64.
+      MVT WideIScl = EltBits == 64 && FPSclVT == MVT::f64 ? MVT::i64 : MVT::i32;
+      MVT WideI = MVT::getVectorVT(WideIScl, WideElts);
+      SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT,
+                                         DL, MVT::i32); // {rn-sae}
+      SDValue Quot =
+          DAG.getNode(X86ISD::FDIV_RND, DL, WideFP,
+                      widenSubVector(X, false, Subtarget, DAG, DL, 512),
+                      widenSubVector(Y, false, Subtarget, DAG, DL, 512), RN);
+      unsigned FromFP = IsSigned ? X86ISD::CVTTP2SI_SAE : X86ISD::CVTTP2UI_SAE;
+      Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq/qq {sae}
+      MVT NarrowI = MVT::getVectorVT(WideIScl, VT.getVectorNumElements());
+      Q = extractSubVector(Q, 0, DAG, DL, NarrowI.getSizeInBits());
+      Q = IsSigned ? DAG.getSExtOrTrunc(Q, DL, VT)
+                   : DAG.getZExtOrTrunc(Q, DL, VT);
+    } else {
+      unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
+      Q = DAG.getNode(FromFP, DL, VT, DAG.getNode(ISD::FDIV, DL, FPVT, X, Y));
+    }
+    if (!IsRem)
+      return Q;
+    // rem = dividend - quotient * divisor
+    return DAG.getNode(ISD::SUB, DL, VT, Dividend,
+                       DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
+  }
+
+  // i64: the quotient doesn't fit f64 exactly, so build it from two
+  // rounded-down reciprocal multiplies, one of the dividend and one of its
+  // remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
+  if ((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
+      Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
+    bool Widen = VT != MVT::v8i64;
+    // v2/v4 keep their integer ops at the original width and run only the
+    // rounded FP ops in the low half of a zmm, which needs VLX.
+    if (Widen && !Subtarget.hasVLX())
----------------
phoebewang wrote:

We don't need VLX if widen/extract to/from 512-bit.

https://github.com/llvm/llvm-project/pull/205263