[llvm] [SDAG][X86] Hoist vector integer div/rem via FP into TargetLowering (PR #221877)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 22:04:26 PDT 2026
https://github.com/as4230 created https://github.com/llvm/llvm-project/pull/221877
x86 lowers vector integer division through a float divide rather than scalarizing into per-lane divides. A lot of that lowering is not x86 specific, so this moves it into TargetLowering. Requested in #215043.
Moved to generic layer includes the exactness check and FP type choice, the profitability bails, widening a non power of two lane count, halving an FP type too wide for the target, and the four MASKED_ div/rem opcodes, which DAGCombiner did not visit before.
A target opts in with getIntDivRemFPExpansion which lists the FP scalar types it can convert to and from. emitIntDivRemViaFP is virtual with a working default of convert, FDIV, convert back. x86 overrides it because strict FP needs the SAE forms, and an i64 value too wide for the f64 mantissa needs its dividend and divisor converted with different rounding modes, which generic code cannot supply.
combineIntDivRem becomes expandIntDivRemViaFP and combineIntDivRemViaExactFPDiv splits between the generic default and the x86 override.
>From c425154f89c29958e999b269709fa092225a11c7 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 7 Sep 2026 06:11:30 +0000
Subject: [PATCH] Hoist the int div via FP expansion into TargetLowering
---
llvm/include/llvm/CodeGen/TargetLowering.h | 21 ++
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 23 ++
.../CodeGen/SelectionDAG/TargetLowering.cpp | 157 +++++++++++
llvm/lib/Target/X86/X86ISelLowering.cpp | 254 ++++++------------
llvm/lib/Target/X86/X86ISelLowering.h | 14 +
5 files changed, 293 insertions(+), 176 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 69f5e0e4e3011..b5ae66a77c4d5 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5557,6 +5557,27 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
SelectionDAG &DAG,
SmallVectorImpl<SDNode *> &Created) const;
+ /// Describe how this target wants a vector integer divide or remainder of
+ /// \p VT expanded through a floating point divide. \p Candidates takes the
+ /// usable FP scalar types narrowest first, and stays empty to decline.
+ /// \p RequiredBits is the one width the emitted form encodes at or zero.
+ virtual void getIntDivRemFPExpansion(SmallVectorImpl<MVT> &Candidates,
+ unsigned &RequiredBits, EVT VT,
+ bool IsSigned, bool IsStrict) const {}
+
+ /// Expand the vector integer divide or remainder \p N through a floating
+ /// point divide, when getIntDivRemFPExpansion says the target wants one.
+ SDValue expandIntDivRemViaFP(SDNode *N, SelectionDAG &DAG,
+ CombineLevel Level) const;
+
+ /// Emit the expansion body in \p FPVT. \p OperandsExact says whether both
+ /// operands were proven to fit that mantissa. The default converts, divides
+ /// with ISD::FDIV and converts back. An empty return declines.
+ virtual SDValue emitIntDivRemViaFP(SDNode *N, EVT FPVT, bool IsSigned,
+ bool IsRem, bool IsStrict,
+ bool OperandsExact,
+ SelectionDAG &DAG) const;
+
/// Targets may override this function to provide custom SDIV lowering for
/// power-of-2 denominators. If the target returns an empty SDValue, LLVM
/// assumes SDIV is expensive and replaces it with a series of other integer
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 733d0eb9baa40..46ee2bbd60772 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -444,6 +444,7 @@ namespace {
SDValue visitMULFIX(SDNode *N);
SDValue useDivRem(SDNode *N);
SDValue visitSDIV(SDNode *N);
+ SDValue visitMaskedDivRem(SDNode *N);
SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
SDValue visitUDIV(SDNode *N);
SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
@@ -1994,6 +1995,10 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::UDIV: return visitUDIV(N);
case ISD::SREM:
case ISD::UREM: return visitREM(N);
+ case ISD::MASKED_SDIV:
+ case ISD::MASKED_UDIV:
+ case ISD::MASKED_SREM:
+ case ISD::MASKED_UREM: return visitMaskedDivRem(N);
case ISD::MULHU: return visitMULHU(N);
case ISD::MULHS: return visitMULHS(N);
case ISD::AVGFLOORS:
@@ -5277,6 +5282,12 @@ static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
return SDValue();
}
+// handles ISD::MASKED_SDIV, MASKED_UDIV, MASKED_SREM and MASKED_UREM
+SDValue DAGCombiner::visitMaskedDivRem(SDNode *N) {
+ // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+ return TLI.expandIntDivRemViaFP(N, DAG, Level);
+}
+
SDValue DAGCombiner::visitSDIV(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
@@ -5341,6 +5352,10 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
if (SDValue DivRem = useDivRem(N))
return DivRem;
+ // A target without vector integer division may want a float divide.
+ if (SDValue V = TLI.expandIntDivRemViaFP(N, DAG, Level))
+ return V;
+
return SDValue();
}
@@ -5497,6 +5512,10 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) {
if (SimplifyDemandedBits(SDValue(N, 0)))
return SDValue(N, 0);
+ // A target without vector integer division may want a float divide.
+ if (SDValue V = TLI.expandIntDivRemViaFP(N, DAG, Level))
+ return V;
+
return SDValue();
}
@@ -5655,6 +5674,10 @@ SDValue DAGCombiner::visitREM(SDNode *N) {
return DAG.getNode(ISD::SREM, DL, VT, A, DAG.getConstant(Op1Cst, DL, VT));
}
+ // A target without vector integer division may want a float divide.
+ if (SDValue V = TLI.expandIntDivRemViaFP(N, DAG, Level))
+ return V;
+
return SDValue();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 16f91bb50e717..7ee35b8dfe95d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -7217,6 +7217,163 @@ turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
}
+/// Return true when \p FPSclVT holds every value \p V can take, so a round
+/// trip through it rounds nothing away.
+static bool isIntExactlyRepresentableInFP(SDValue V, MVT FPSclVT, bool IsSigned,
+ SelectionDAG &DAG) {
+ unsigned Precision = APFloat::semanticsPrecision(FPSclVT.getFltSemantics());
+ unsigned EltBits = V.getValueType().getScalarSizeInBits();
+ return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
+ : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
+}
+
+SDValue TargetLowering::emitIntDivRemViaFP(SDNode *N, EVT FPVT, bool IsSigned,
+ bool IsRem, bool IsStrict,
+ bool OperandsExact,
+ SelectionDAG &DAG) const {
+ // Only an exact divide is recoverable without a target refinement sequence.
+ if (!OperandsExact)
+ return SDValue();
+
+ SDLoc DL(N);
+ EVT IntVT = N->getValueType(0);
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
+
+ unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+ unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
+ SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
+ SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
+ SDValue Q =
+ DAG.getNode(FromFP, DL, IntVT, DAG.getNode(ISD::FDIV, DL, FPVT, X, Y));
+ if (!IsRem)
+ return Q;
+ // rem = dividend - quotient * divisor
+ return DAG.getNode(ISD::SUB, DL, IntVT, Dividend,
+ DAG.getNode(ISD::MUL, DL, IntVT, Q, Divisor));
+}
+
+SDValue TargetLowering::expandIntDivRemViaFP(SDNode *N, SelectionDAG &DAG,
+ CombineLevel Level) const {
+ EVT VT = N->getValueType(0);
+ // Everything below works on a constant lane count.
+ if (!VT.isFixedLengthVector())
+ return SDValue();
+
+ SDLoc DL(N);
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
+ unsigned Opc = N->getOpcode();
+
+ // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+ if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
+ Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
+ Opc = ISD::getUnmaskedBinOpOpcode(Opc);
+ bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
+ bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+
+ bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::StrictFP);
+ SmallVector<MVT, 2> Candidates;
+ unsigned RequiredBits = 0;
+ getIntDivRemFPExpansion(Candidates, RequiredBits, VT, IsSigned, IsStrict);
+ if (Candidates.empty())
+ return SDValue();
+
+ // If the result is only read back as scalar extracts, scalarization computes
+ // just the demanded lanes. Keep the vector operation when every lane is
+ // extracted, which occurs when non-power-of-two vectors are returned.
+ APInt ExtractedElts = APInt::getZero(VT.getVectorNumElements());
+ bool OnlyExtracts = true;
+ for (const SDNode *U : N->users()) {
+ if (U->getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
+ OnlyExtracts = false;
+ break;
+ }
+ auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
+ if (!Idx)
+ continue;
+ const APInt &IdxVal = Idx->getAPIntValue();
+ if (IdxVal.uge(VT.getVectorNumElements()))
+ continue;
+ ExtractedElts.setBit(IdxVal.getZExtValue());
+ }
+ if (OnlyExtracts && !ExtractedElts.isAllOnes())
+ return SDValue();
+
+ // Magic multiply lowers constant divisors cheaper than a divide.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
+ return SDValue();
+
+ auto BothFitFP = [&](MVT FPSclVT) {
+ return isIntExactlyRepresentableInFP(Dividend, FPSclVT, IsSigned, DAG) &&
+ isIntExactlyRepresentableInFP(Divisor, FPSclVT, IsSigned, DAG);
+ };
+
+ // Take the narrowest type that holds both operands. When none does, pass the
+ // widest anyway, since a target may still recover the quotient from it.
+ MVT FPSclVT = Candidates.back();
+ bool OperandsExact = false;
+ for (MVT C : Candidates) {
+ if (!BothFitFP(C))
+ continue;
+ FPSclVT = C;
+ OperandsExact = true;
+ break;
+ }
+
+ EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+
+ // Nothing will split an illegal FP type after type legalization, and a form
+ // that only encodes at one width cannot hold an FP type wider than it.
+ bool FPVTUsable = RequiredBits
+ ? FPVT.getSizeInBits() <= RequiredBits
+ : Level == BeforeLegalizeTypes || isTypeLegal(FPVT);
+
+ // Halve the divide while the integer halves stay legal.
+ if (!FPVTUsable) {
+ if (VT.is256BitVector() || VT.is512BitVector()) {
+ EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
+ if (isTypeLegal(HalfVT)) {
+ SmallVector<SDValue, 3> LoOps, HiOps;
+ for (const SDValue &Op : N->ops()) {
+ auto [Lo, Hi] = DAG.SplitVector(Op, DL);
+ LoOps.push_back(Lo);
+ HiOps.push_back(Hi);
+ }
+ SDValue Lo = DAG.getNode(N->getOpcode(), DL, HalfVT, LoOps);
+ SDValue Hi = DAG.getNode(N->getOpcode(), DL, HalfVT, HiOps);
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
+ }
+ }
+ return SDValue();
+ }
+
+ SDValue Res = emitIntDivRemViaFP(N, FPVT, IsSigned, IsRem, IsStrict,
+ OperandsExact, DAG);
+
+ // Widen a non-power-of-two lane count to get a machine type, but only
+ // while it still fits one divide. Two chains lose to a chain plus a scalar.
+ unsigned NumElts = VT.getVectorNumElements();
+ if (!Res && RequiredBits && !isPowerOf2_32(NumElts) &&
+ NextPowerOf2(NumElts) * FPSclVT.getSizeInBits() <= RequiredBits) {
+ Dividend = DAG.WidenVector(Dividend, DL);
+ Divisor = DAG.WidenVector(Divisor, DL);
+ SDNode *WideN =
+ DAG.getNode(Opc, DL, Dividend.getValueType(), Dividend, Divisor)
+ .getNode();
+ EVT WideFPVT = Dividend.getValueType().changeVectorElementType(
+ *DAG.getContext(), FPSclVT);
+ Res = emitIntDivRemViaFP(WideN, WideFPVT, IsSigned, IsRem, IsStrict,
+ OperandsExact, DAG);
+ }
+
+ // Narrow a widened result back to VT.
+ if (Res && Res.getValueType() != VT)
+ Res = DAG.getExtractSubvector(DL, VT, Res, 0);
+ return Res;
+}
+
/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
/// where the divisor and comparison target are constants,
/// return a DAG expression that will generate the same comparison result
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d41405645cc34..82a5f623f7743 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50792,79 +50792,6 @@ static SDValue combineMulToPMADD52(SDNode *N, const SDLoc &DL,
return SDValue();
}
-// The operands fit the float mantissa exactly so one float divide recovers
-// the exact quotient.
-static SDValue
-combineIntDivRemViaExactFPDiv(SDNode *N, MVT FPSclVT, bool IsSigned, bool IsRem,
- bool IsStrict, SelectionDAG &DAG,
- TargetLowering::DAGCombinerInfo &DCI,
- const X86Subtarget &Subtarget, const SDLoc &DL) {
- EVT VT = N->getValueType(0);
- SDValue Dividend = N->getOperand(0);
- SDValue Divisor = N->getOperand(1);
- EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
-
- // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
- // for latency and code size before AVX2.
- if (!IsStrict && !IsSigned && VT.getScalarSizeInBits() == 32 &&
- !Subtarget.hasAVX2())
- return SDValue();
-
- // Nothing will split an illegal FP type after type legalization and the
- // strict SAE divide is 512-bit only.
- bool FPVTUsable = IsStrict
- ? FPVT.getSizeInBits() <= 512
- : DCI.isBeforeLegalize() ||
- DAG.getTargetLoweringInfo().isTypeLegal(FPVT);
-
- // Halve the divide while the integer halves stay legal.
- if (!FPVTUsable) {
- if (VT.is256BitVector() || VT.is512BitVector()) {
- EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
- if (DAG.getTargetLoweringInfo().isTypeLegal(HalfVT))
- return splitVectorIntBinary(SDValue(N, 0), DAG, DL);
- }
- return SDValue();
- }
-
- unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
- SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
- SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
- SDValue Q;
- if (IsStrict) {
- // The converts are exact so only the divide and the truncate can
- // raise flags.
- unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
- MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
- // Only an f64 quotient of i64 operands needs the qq convert to come back
- // whole. Everything else fits i32 lanes.
- MVT WideIScl = MVT::i32;
- if (VT.getScalarSizeInBits() == 64 && FPSclVT == MVT::f64)
- WideIScl = MVT::i64;
- MVT WideI = MVT::getVectorVT(WideIScl, WideElts);
- SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT, DL,
- MVT::i32); // {rn-sae}
- SDValue Quot =
- DAG.getNode(X86ISD::FDIV_RND, DL, WideFP,
- widenSubVector(X, false, Subtarget, DAG, DL, 512),
- widenSubVector(Y, false, Subtarget, DAG, DL, 512), RN);
- unsigned FromFP = IsSigned ? X86ISD::CVTTP2SI_SAE : X86ISD::CVTTP2UI_SAE;
- Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq/qq {sae}
- MVT NarrowI = MVT::getVectorVT(WideIScl, VT.getVectorNumElements());
- Q = extractSubVector(Q, 0, DAG, DL, NarrowI.getSizeInBits());
- Q = IsSigned ? DAG.getSExtOrTrunc(Q, DL, VT)
- : DAG.getZExtOrTrunc(Q, DL, VT);
- } else {
- unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
- Q = DAG.getNode(FromFP, DL, VT, DAG.getNode(ISD::FDIV, DL, FPVT, X, Y));
- }
- if (!IsRem)
- return Q;
- // rem = dividend - quotient * divisor
- return DAG.getNode(ISD::SUB, DL, VT, Dividend,
- DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
-}
-
// i64: the quotient doesn't fit f64 exactly, so build it from two
// rounded-down reciprocal multiplies, one of the dividend and one of its
// remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
@@ -50948,114 +50875,97 @@ static SDValue combineInt64DivRemViaFPReciprocal(SDNode *N, bool IsSigned,
Mag);
}
-// x86 has no vector integer divide instructions. Lower vector
-// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
-// scalar hardware divides.
-static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
- TargetLowering::DAGCombinerInfo &DCI,
- const X86Subtarget &Subtarget) {
- EVT VT = N->getValueType(0);
- SDLoc DL(N);
-
- // Vector division never survives op legalization to reach later rounds.
- if (!VT.isVector() || !Subtarget.hasSSE2())
- return SDValue();
+void X86TargetLowering::getIntDivRemFPExpansion(
+ SmallVectorImpl<MVT> &Candidates, unsigned &RequiredBits, EVT VT,
+ bool IsSigned, bool IsStrict) const {
+ if (!Subtarget.hasSSE2())
+ return;
- SDValue Dividend = N->getOperand(0);
- SDValue Divisor = N->getOperand(1);
- unsigned Opc = N->getOpcode();
+ // Nothing wider than i64 converts to or from FP.
+ unsigned EltBits = VT.getScalarSizeInBits();
+ if (EltBits > 64)
+ return;
- // Disabled lanes are poison and fdiv never traps, so ignore the mask.
- if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
- Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
- Opc = ISD::getUnmaskedBinOpOpcode(Opc);
- bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
- bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
-
- // If the result is only read back as scalar extracts, scalarization computes
- // just the demanded lanes. Keep the vector operation when every lane is
- // extracted, which occurs when non-power-of-two vectors are returned.
- APInt ExtractedElts = APInt::getZero(VT.getVectorNumElements());
- bool OnlyExtracts = true;
- for (const SDNode *U : N->users()) {
- if (U->getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
- OnlyExtracts = false;
- break;
- }
- auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
- if (!Idx)
- continue;
- const APInt &IdxVal = Idx->getAPIntValue();
- if (IdxVal.uge(VT.getVectorNumElements()))
- continue;
- ExtractedElts.setBit(IdxVal.getZExtValue());
+ // i64 needs the qq converts which is AVX512DQ only.
+ if (EltBits == 64) {
+ if (!Subtarget.hasDQI() || !Subtarget.useAVX512Regs())
+ return;
+ RequiredBits = 512;
}
- if (OnlyExtracts && !ExtractedElts.isAllOnes())
- return SDValue();
-
- // Magic multiply lowers constant divisors cheaper than a divide.
- if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
- return SDValue();
- unsigned EltBits = VT.getScalarSizeInBits();
- auto FitsFP = [&](SDValue V, const fltSemantics &Sem) {
- unsigned Precision = APFloat::semanticsPrecision(Sem);
- return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
- : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
- };
- auto BothFitFP = [&](const fltSemantics &Sem) {
- return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
- };
+ // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
+ // for latency and code size before AVX2.
+ if (!IsStrict && !IsSigned && EltBits == 32 && !Subtarget.hasAVX2())
+ return;
- // i64 needs the qq converts which is AVX512DQ only.
- bool NarrowI64 = EltBits == 64 && Subtarget.hasDQI() &&
- Subtarget.useAVX512Regs() &&
- BothFitFP(APFloat::IEEEdouble());
+ // The SAE forms are 512-bit only. Inputs widen into a zmm, which requires
+ // 512-bit types to be legal.
+ if (IsStrict) {
+ if (!Subtarget.useAVX512Regs())
+ return;
+ RequiredBits = 512;
+ }
- // i8/i16/i32 and narrow value i64 take one exact float divide.
- bool UseExactFPDiv = EltBits <= 32 || NarrowI64;
- if (!UseExactFPDiv && EltBits != 64)
- return SDValue();
+ Candidates.assign({MVT::f32, MVT::f64});
+}
- // f32 recovers the quotient exactly when both operands fit in 24 bits
- MVT FPSclVT = MVT::f64;
- if (UseExactFPDiv && (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle())))
- FPSclVT = MVT::f32;
+SDValue X86TargetLowering::emitIntDivRemViaFP(SDNode *N, EVT FPVT,
+ bool IsSigned, bool IsRem,
+ bool IsStrict, bool OperandsExact,
+ SelectionDAG &DAG) const {
+ SDLoc DL(N);
+ EVT IntVT = N->getValueType(0);
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
- bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
- Attribute::StrictFP);
+ if (OperandsExact && !IsStrict)
+ return TargetLowering::emitIntDivRemViaFP(N, FPVT, IsSigned, IsRem,
+ IsStrict, OperandsExact, DAG);
- // The SAE forms are 512-bit only. Inputs widen into a zmm below, which
- // requires 512-bit types to be legal.
- if (IsStrict && !Subtarget.useAVX512Regs())
+ // Widening the rounded FP steps into a zmm needs a machine type.
+ if (!isPowerOf2_32(IntVT.getVectorNumElements()))
return SDValue();
- if (!UseExactFPDiv && (!Subtarget.hasDQI() || !Subtarget.useAVX512Regs()))
- return SDValue();
+ // No exact single divide, so recover the quotient from reciprocal multiplies.
+ if (!OperandsExact)
+ return combineInt64DivRemViaFPReciprocal(N, IsSigned, IsRem, DAG, Subtarget,
+ DL);
- // Widen a non-power-of-two lane count to get a machine type, but only
- // while it still fits one divide. Two chains lose to a chain plus a scalar.
- unsigned NumElts = VT.getVectorNumElements();
- bool Needs512 = IsStrict || !UseExactFPDiv;
- if (Needs512 && !isPowerOf2_32(NumElts)) {
- if (NextPowerOf2(NumElts) * FPSclVT.getSizeInBits() > 512)
- return SDValue();
- Dividend = DAG.WidenVector(Dividend, DL);
- Divisor = DAG.WidenVector(Divisor, DL);
- N = DAG.getNode(Opc, DL, Dividend.getValueType(), Dividend, Divisor)
- .getNode();
- }
+ // The converts are exact so only the divide and the truncate can
+ // raise flags.
+ MVT FPSclVT = FPVT.getScalarType().getSimpleVT();
+ unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+ SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
+ SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
- SDValue Res =
- UseExactFPDiv
- ? combineIntDivRemViaExactFPDiv(N, FPSclVT, IsSigned, IsRem, IsStrict,
- DAG, DCI, Subtarget, DL)
- : combineInt64DivRemViaFPReciprocal(N, IsSigned, IsRem, DAG,
- Subtarget, DL);
- // Narrow a widened result back to VT.
- if (Res && Res.getValueType() != VT)
- Res = DAG.getExtractSubvector(DL, VT, Res, 0);
- return Res;
+ unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
+ MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
+ // Only an f64 quotient of i64 operands needs the qq convert to come back
+ // whole. Everything else fits i32 lanes.
+ MVT WideIScl = MVT::i32;
+ if (IntVT.getScalarSizeInBits() == 64 && FPSclVT == MVT::f64)
+ WideIScl = MVT::i64;
+ MVT WideI = MVT::getVectorVT(WideIScl, WideElts);
+
+ SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT, DL,
+ MVT::i32); // {rn-sae}
+ SDValue Quot =
+ DAG.getNode(X86ISD::FDIV_RND, DL, WideFP,
+ widenSubVector(X, false, Subtarget, DAG, DL, 512),
+ widenSubVector(Y, false, Subtarget, DAG, DL, 512), RN);
+
+ unsigned FromFP = IsSigned ? X86ISD::CVTTP2SI_SAE : X86ISD::CVTTP2UI_SAE;
+ SDValue Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq/qq {sae}
+ MVT NarrowI = MVT::getVectorVT(WideIScl, IntVT.getVectorNumElements());
+ Q = extractSubVector(Q, 0, DAG, DL, NarrowI.getSizeInBits());
+ Q = IsSigned ? DAG.getSExtOrTrunc(Q, DL, IntVT)
+ : DAG.getZExtOrTrunc(Q, DL, IntVT);
+
+ if (!IsRem)
+ return Q;
+ // rem = dividend - quotient * divisor
+ return DAG.getNode(ISD::SUB, DL, IntVT, Dividend,
+ DAG.getNode(ISD::MUL, DL, IntVT, Q, Divisor));
}
static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
@@ -63866,14 +63776,6 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::ADOX: return combineADOX(N, DAG);
case X86ISD::ADC: return combineADC(N, DAG, DCI);
case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
- case ISD::UDIV:
- case ISD::SDIV:
- case ISD::UREM:
- case ISD::SREM:
- case ISD::MASKED_UDIV:
- case ISD::MASKED_SDIV:
- case ISD::MASKED_UREM:
- case ISD::MASKED_SREM: return combineIntDivRem(N, DAG, DCI, Subtarget);
case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget);
case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget);
case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index b220c1bbe7bc7..dbaa3ede07c92 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -176,6 +176,20 @@ namespace llvm {
///
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
+ /// x86 has no vector integer divide, so route i8/i16/i32 and narrow i64
+ /// through an f32 or f64 divide. Under strict FP the SAE forms are needed
+ /// and those encode only at 512 bits.
+ void getIntDivRemFPExpansion(SmallVectorImpl<MVT> &Candidates,
+ unsigned &RequiredBits, EVT VT, bool IsSigned,
+ bool IsStrict) const override;
+
+ /// Emit the divide, which is an SAE sequence under strict FP, a rounded
+ /// reciprocal chain when an i64 operand misses the f64 mantissa, and the
+ /// base implementation otherwise.
+ SDValue emitIntDivRemViaFP(SDNode *N, EVT FPVT, bool IsSigned, bool IsRem,
+ bool IsStrict, bool OperandsExact,
+ SelectionDAG &DAG) const override;
+
/// Replace the results of node with an illegal result
/// type with new values built out of custom code.
///
More information about the llvm-commits
mailing list