[llvm] 343b566 - [TargetLowering] Move the MULH/MUL_LOHI legality checks to the beginning of BuildSDIV/UDIV. NFCI (#187780)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 20 14:52:33 PDT 2026
Author: Craig Topper
Date: 2026-03-20T14:52:28-07:00
New Revision: 343b566b5746c2bb06d758d8a70e14c376a33de4
URL: https://github.com/llvm/llvm-project/commit/343b566b5746c2bb06d758d8a70e14c376a33de4
DIFF: https://github.com/llvm/llvm-project/commit/343b566b5746c2bb06d758d8a70e14c376a33de4.diff
LOG: [TargetLowering] Move the MULH/MUL_LOHI legality checks to the beginning of BuildSDIV/UDIV. NFCI (#187780)
This groups the type and operation legality checks to the beginning. The
rest of the code can focus on the transformation.
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index e2a47e293ca38..c9bddf636c751 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6628,6 +6628,27 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+ bool HasMULHS = isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization);
+ bool HasSMUL_LOHI =
+ isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization);
+
+ if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
+ // If type twice as wide legal, widen and use a mul plus a shift.
+ unsigned Size = VT.getScalarSizeInBits();
+ EVT WideVT = VT.changeElementType(
+ *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), Size * 2));
+ // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
+ // custom lowered. This is very expensive so avoid it at all costs for
+ // constant divisors.
+ if ((!IsAfterLegalTypes && isOperationExpand(ISD::SDIV, VT) &&
+ isOperationCustom(ISD::SDIVREM, VT.getScalarType())) ||
+ isOperationLegalOrCustom(ISD::MUL, WideVT))
+ MulVT = WideVT;
+ }
+
+ if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
+ return SDValue();
+
SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
auto BuildSDIVPattern = [&](ConstantSDNode *C) {
@@ -6693,44 +6714,21 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
}
// Multiply the numerator (operand 0) by the magic value.
- // FIXME: We should support doing a MUL in a wider type.
auto GetMULHS = [&](SDValue X, SDValue Y) {
- // If the type isn't legal, use a wider mul of the type calculated
- // earlier.
- if (!isTypeLegal(VT)) {
- X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
- Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
- Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
- Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
- DAG.getShiftAmountConstant(EltBits, MulVT, dl));
- return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
- }
-
- if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
+ if (HasMULHS)
return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
- if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
+ if (HasSMUL_LOHI) {
SDValue LoHi =
DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
- return SDValue(LoHi.getNode(), 1);
- }
- // If type twice as wide legal, widen and use a mul plus a shift.
- unsigned Size = VT.getScalarSizeInBits();
- EVT WideVT = VT.changeElementType(
- *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), Size * 2));
- // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
- // custom lowered. This is very expensive so avoid it at all costs for
- // constant divisors.
- if ((!IsAfterLegalTypes && isOperationExpand(ISD::SDIV, VT) &&
- isOperationCustom(ISD::SDIVREM, VT.getScalarType())) ||
- isOperationLegalOrCustom(ISD::MUL, WideVT)) {
- X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X);
- Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y);
- Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
- Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
- DAG.getShiftAmountConstant(EltBits, WideVT, dl));
- return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
+ return LoHi.getValue(1);
}
- return SDValue();
+
+ X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
+ Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
+ Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
+ Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
+ DAG.getShiftAmountConstant(EltBits, MulVT, dl));
+ return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
};
SDValue Q = GetMULHS(N0, MagicFactor);
@@ -6797,6 +6795,27 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+ bool HasMULHU = isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization);
+ bool HasUMUL_LOHI =
+ isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization);
+
+ if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
+ // If type twice as wide legal, widen and use a mul plus a shift.
+ unsigned Size = VT.getScalarSizeInBits();
+ EVT WideVT = VT.changeElementType(
+ *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), Size * 2));
+ // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
+ // custom lowered. This is very expensive so avoid it at all costs for
+ // constant divisors.
+ if ((!IsAfterLegalTypes && isOperationExpand(ISD::UDIV, VT) &&
+ isOperationCustom(ISD::UDIVREM, VT.getScalarType())) ||
+ isOperationLegalOrCustom(ISD::MUL, WideVT))
+ MulVT = WideVT;
+ }
+
+ if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
+ return SDValue();
+
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
@@ -6937,44 +6956,21 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
Created.push_back(Q.getNode());
}
- // FIXME: We should support doing a MUL in a wider type.
auto GetMULHU = [&](SDValue X, SDValue Y) {
- // If the type isn't legal, use a wider mul of the type calculated
- // earlier.
- if (!isTypeLegal(VT)) {
- X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
- Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
- Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
- Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
- DAG.getShiftAmountConstant(EltBits, MulVT, dl));
- return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
- }
-
- if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
+ if (HasMULHU)
return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
- if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
+ if (HasUMUL_LOHI) {
SDValue LoHi =
DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
- return SDValue(LoHi.getNode(), 1);
+ return LoHi.getValue(1);
}
- // If type twice as wide legal, widen and use a mul plus a shift.
- unsigned Size = VT.getScalarSizeInBits();
- EVT WideVT = VT.changeElementType(
- *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), Size * 2));
- // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
- // custom lowered. This is very expensive so avoid it at all costs for
- // constant divisors.
- if ((!IsAfterLegalTypes && isOperationExpand(ISD::UDIV, VT) &&
- isOperationCustom(ISD::UDIVREM, VT.getScalarType())) ||
- isOperationLegalOrCustom(ISD::MUL, WideVT)) {
- X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X);
- Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y);
- Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
- Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
- DAG.getShiftAmountConstant(EltBits, WideVT, dl));
- return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
- }
- return SDValue(); // No mulhu or equivalent
+
+ X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
+ Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
+ Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
+ Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
+ DAG.getShiftAmountConstant(EltBits, MulVT, dl));
+ return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
};
// Multiply the numerator (operand 0) by the magic value.
More information about the llvm-commits
mailing list