[llvm] [AArch64] missed fold vector fptoui(fdiv(uitofp(x), uitofp(y))) or fptosi(fdiv(sitofp(x), sitofp(y))) into udiv(x, y) or sdiv(x, y) for sve- #215146 (PR #215526)
Durgesh Nandan Mohanty via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 04:38:40 PDT 2026
https://github.com/dnmohanty created https://github.com/llvm/llvm-project/pull/215526
This PR implements a DAG combine in AArch64ISelLowering.cpp to fold vector floating-point division and conversions into native SVE integer division (udiv/sdiv).
DAG Combine Logic: Added performFPToIntToDivCombine supporting ISD::FP_TO_UINT and ISD::FP_TO_SINT with strict type checks ensuring safe conversions (i32vectors viaf64). Tests: Added llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll` to verify optimal single-instruction codegen.
Commit History: Structured into two clean commits as requested:
1.Pre-commit baseline test (NFC)
2.Fixup commit with compiler changes and updated test checks
Fixes https://github.com/llvm/llvm-project/issues/214927
>From a4cc7a81c230ba028ba661cc70d306b6443a57d8 Mon Sep 17 00:00:00 2001
From: Durgesh Nandan Mohanty <durgeshnandanmohanty at gmail.com>
Date: Tue, 11 Aug 2026 11:36:18 +0000
Subject: [PATCH] Pre-commit baseline test for SVE fdiv fold (NFC)
---
.../Target/AArch64/AArch64ISelLowering.cpp | 317 ++++++++----------
.../test/CodeGen/AArch64/sve-fdiv-int-fold.ll | 66 ++++
2 files changed, 209 insertions(+), 174 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6003db1c72449..2e6bc7102a184 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -364,13 +364,6 @@ static bool isMergePassthruOpcode(unsigned Opc) {
case AArch64ISD::FSQRT_MERGE_PASSTHRU:
case AArch64ISD::FRECPX_MERGE_PASSTHRU:
case AArch64ISD::FABS_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FCEIL_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FFLOOR_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FNEARBYINT_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FROUND_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FROUNDEVEN_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FTRUNC_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FSQRT_MERGE_PASSTHRU:
return true;
}
}
@@ -1637,14 +1630,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
}
- if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
- for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
- setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
-
- for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
- setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
- }
-
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
if (Subtarget->isSVEorStreamingSVEAvailable() &&
@@ -1841,30 +1826,30 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
setOperationAction(ISD::SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction({ISD::FADD, ISD::STRICT_FADD}, VT, Custom);
+ setOperationAction(ISD::FADD, VT, Custom);
setOperationAction(ISD::FCANONICALIZE, VT, Custom);
setOperationAction(ISD::FCOPYSIGN, VT, Custom);
- setOperationAction({ISD::FDIV, ISD::STRICT_FDIV}, VT, Custom);
- setOperationAction({ISD::FMA, ISD::STRICT_FMA}, VT, Custom);
- setOperationAction({ISD::FMAXIMUM, ISD::STRICT_FMAXIMUM}, VT, Custom);
- setOperationAction({ISD::FMAXNUM, ISD::STRICT_FMAXNUM}, VT, Custom);
+ setOperationAction(ISD::FDIV, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Custom);
+ setOperationAction(ISD::FMAXIMUM, VT, Custom);
+ setOperationAction(ISD::FMAXNUM, VT, Custom);
setOperationAction(ISD::FMAXNUM_IEEE, VT, Custom);
- setOperationAction({ISD::FMINIMUM, ISD::STRICT_FMINIMUM}, VT, Custom);
- setOperationAction({ISD::FMINNUM, ISD::STRICT_FMINNUM}, VT, Custom);
+ setOperationAction(ISD::FMINIMUM, VT, Custom);
+ setOperationAction(ISD::FMINNUM, VT, Custom);
setOperationAction(ISD::FMINNUM_IEEE, VT, Custom);
- setOperationAction({ISD::FMUL, ISD::STRICT_FMUL}, VT, Custom);
+ setOperationAction(ISD::FMUL, VT, Custom);
setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction({ISD::FSUB, ISD::STRICT_FSUB}, VT, Custom);
- setOperationAction({ISD::FCEIL, ISD::STRICT_FCEIL}, VT, Custom);
- setOperationAction({ISD::FFLOOR, ISD::STRICT_FFLOOR}, VT, Custom);
- setOperationAction({ISD::FNEARBYINT, ISD::STRICT_FNEARBYINT}, VT, Custom);
+ setOperationAction(ISD::FSUB, VT, Custom);
+ setOperationAction(ISD::FCEIL, VT, Custom);
+ setOperationAction(ISD::FFLOOR, VT, Custom);
+ setOperationAction(ISD::FNEARBYINT, VT, Custom);
setOperationAction(ISD::FRINT, VT, Custom);
setOperationAction(ISD::LRINT, VT, Custom);
setOperationAction(ISD::LLRINT, VT, Custom);
- setOperationAction({ISD::FROUND, ISD::STRICT_FROUND}, VT, Custom);
- setOperationAction({ISD::FROUNDEVEN, ISD::STRICT_FROUNDEVEN}, VT, Custom);
- setOperationAction({ISD::FTRUNC, ISD::STRICT_FTRUNC}, VT, Custom);
- setOperationAction({ISD::FSQRT, ISD::STRICT_FSQRT}, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUNDEVEN, VT, Custom);
+ setOperationAction(ISD::FTRUNC, VT, Custom);
+ setOperationAction(ISD::FSQRT, VT, Custom);
setOperationAction(ISD::FABS, VT, Custom);
setOperationAction(ISD::FP_EXTEND, VT, Custom);
setOperationAction(ISD::FP_ROUND, VT, Custom);
@@ -1880,45 +1865,27 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Expand);
- setOperationAction({ISD::FREM, ISD::STRICT_FREM}, VT, Expand);
- setOperationAction({ISD::FPOW, ISD::STRICT_FPOW}, VT, Expand);
- setOperationAction({ISD::FPOWI, ISD::STRICT_FPOWI}, VT, Expand);
+ setOperationAction(ISD::FREM, VT, Expand);
+ setOperationAction(ISD::FPOW, VT, Expand);
+ setOperationAction(ISD::FPOWI, VT, Expand);
setOperationAction(ISD::FCBRT, VT, Expand);
- setOperationAction({ISD::FCOS, ISD::STRICT_FCOS}, VT, Expand);
- setOperationAction({ISD::FSIN, ISD::STRICT_FSIN}, VT, Expand);
+ setOperationAction(ISD::FCOS, VT, Expand);
+ setOperationAction(ISD::FSIN, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
- setOperationAction({ISD::FTAN, ISD::STRICT_FTAN}, VT, Expand);
- setOperationAction({ISD::FACOS, ISD::STRICT_FACOS}, VT, Expand);
- setOperationAction({ISD::FASIN, ISD::STRICT_FASIN}, VT, Expand);
- setOperationAction({ISD::FATAN, ISD::STRICT_FATAN}, VT, Expand);
- setOperationAction({ISD::FATAN2, ISD::STRICT_FATAN2}, VT, Expand);
- setOperationAction({ISD::FCOSH, ISD::STRICT_FCOSH}, VT, Expand);
- setOperationAction({ISD::FSINH, ISD::STRICT_FSINH}, VT, Expand);
- setOperationAction({ISD::FTANH, ISD::STRICT_FTANH}, VT, Expand);
- setOperationAction({ISD::FEXP, ISD::STRICT_FEXP}, VT, Expand);
- setOperationAction({ISD::FEXP2, ISD::STRICT_FEXP2}, VT, Expand);
+ setOperationAction(ISD::FTAN, VT, Expand);
+ setOperationAction(ISD::FACOS, VT, Expand);
+ setOperationAction(ISD::FASIN, VT, Expand);
+ setOperationAction(ISD::FATAN, VT, Expand);
+ setOperationAction(ISD::FATAN2, VT, Expand);
+ setOperationAction(ISD::FCOSH, VT, Expand);
+ setOperationAction(ISD::FSINH, VT, Expand);
+ setOperationAction(ISD::FTANH, VT, Expand);
+ setOperationAction(ISD::FEXP, VT, Expand);
+ setOperationAction(ISD::FEXP2, VT, Expand);
setOperationAction(ISD::FEXP10, VT, Expand);
- setOperationAction({ISD::FLOG, ISD::STRICT_FLOG}, VT, Expand);
- setOperationAction({ISD::FLOG2, ISD::STRICT_FLOG2}, VT, Expand);
- setOperationAction({ISD::FLOG10, ISD::STRICT_FLOG10}, VT, Expand);
-
- // TODO: These require custom lowering.
- setOperationAction(ISD::STRICT_FLDEXP, VT, Expand);
- setOperationAction(ISD::STRICT_FRINT, VT, Expand);
- setOperationAction(ISD::STRICT_PSEUDO_FMIN, VT, Expand);
- setOperationAction(ISD::STRICT_PSEUDO_FMAX, VT, Expand);
- setOperationAction(ISD::STRICT_LROUND, VT, Expand);
- setOperationAction(ISD::STRICT_LLROUND, VT, Expand);
- setOperationAction(ISD::STRICT_LRINT, VT, Expand);
- setOperationAction(ISD::STRICT_LLRINT, VT, Expand);
- setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Expand);
- setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Expand);
- setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Expand);
- setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Expand);
- setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand);
- setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand);
- setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
- setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
+ setOperationAction(ISD::FLOG, VT, Expand);
+ setOperationAction(ISD::FLOG2, VT, Expand);
+ setOperationAction(ISD::FLOG10, VT, Expand);
setCondCodeAction(ISD::SETO, VT, Expand);
setCondCodeAction(ISD::SETOLT, VT, Expand);
@@ -2492,6 +2459,19 @@ bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
VT != MVT::v4i1 && VT != MVT::v2i1;
}
+bool AArch64TargetLowering::shouldExpandVectorMatch(EVT VT,
+ unsigned SearchSize) const {
+ // MATCH is SVE2 and only available in non-streaming mode.
+ if (!Subtarget->hasSVE2() || !Subtarget->isSVEAvailable())
+ return true;
+ // Furthermore, we can only use it for 8-bit or 16-bit elements.
+ if (VT == MVT::nxv8i16 || VT == MVT::v8i16)
+ return SearchSize != 8;
+ if (VT == MVT::nxv16i8 || VT == MVT::v16i8 || VT == MVT::v8i8)
+ return SearchSize != 8 && SearchSize != 16;
+ return true;
+}
+
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
@@ -6479,29 +6459,21 @@ static SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
static SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
SDLoc DL(Op);
- auto Op1 = Op.getOperand(0);
- auto Op2 = Op.getOperand(1);
- auto Mask = Op.getOperand(2);
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
+
+ auto Op1 = Op.getOperand(1);
+ auto Op2 = Op.getOperand(2);
+ auto Mask = Op.getOperand(3);
EVT Op1VT = Op1.getValueType();
EVT Op2VT = Op2.getValueType();
EVT ResVT = Op.getValueType();
- unsigned SearchSize = Op2VT.getVectorNumElements();
assert((Op1VT.getVectorElementType() == MVT::i8 ||
Op1VT.getVectorElementType() == MVT::i16) &&
"Expected 8-bit or 16-bit characters.");
- if ((Op1VT == MVT::nxv8i16 || Op1VT == MVT::v8i16) && SearchSize != 8)
- return SDValue();
-
- if ((Op1VT == MVT::nxv16i8 || Op1VT == MVT::v16i8 || Op1VT == MVT::v8i8) &&
- SearchSize != 8 && SearchSize != 16)
- return SDValue();
-
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
-
// Scalable vector type used to wrap operands.
// A single container is enough for both operands because ultimately the
// operands will have to be wrapped to the same type (nxv16i8 or nxv8i16).
@@ -7228,6 +7200,9 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
ADDLV, DAG.getConstant(0, DL, MVT::i64));
return EXTRACT_VEC_ELT;
}
+ case Intrinsic::experimental_vector_match: {
+ return LowerVectorMatch(Op, DAG);
+ }
case Intrinsic::aarch64_cls:
case Intrinsic::aarch64_cls64: {
SDValue Res = DAG.getNode(ISD::CTLS, DL, Op.getOperand(1).getValueType(),
@@ -8620,63 +8595,32 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerXALUO(Op, DAG);
case ISD::FADD:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
- case ISD::STRICT_FADD:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FADD_PRED);
case ISD::FSUB:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
- case ISD::STRICT_FSUB:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FSUB_PRED);
case ISD::FMUL:
return LowerFMUL(Op, DAG);
- case ISD::STRICT_FMUL:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMUL_PRED);
case ISD::FMA:
return LowerFMA(Op, DAG);
- case ISD::STRICT_FMA:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMA_PRED);
case ISD::FDIV:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
- case ISD::STRICT_FDIV:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FDIV_PRED);
case ISD::FNEG:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEG_MERGE_PASSTHRU);
case ISD::FCEIL:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FCEIL_MERGE_PASSTHRU);
- case ISD::STRICT_FCEIL:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FCEIL_MERGE_PASSTHRU);
case ISD::FFLOOR:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FFLOOR_MERGE_PASSTHRU);
- case ISD::STRICT_FFLOOR:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FFLOOR_MERGE_PASSTHRU);
case ISD::FNEARBYINT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEARBYINT_MERGE_PASSTHRU);
- case ISD::STRICT_FNEARBYINT:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FNEARBYINT_MERGE_PASSTHRU);
case ISD::FRINT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FRINT_MERGE_PASSTHRU);
case ISD::FROUND:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUND_MERGE_PASSTHRU);
- case ISD::STRICT_FROUND:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FROUND_MERGE_PASSTHRU);
case ISD::FROUNDEVEN:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU);
- case ISD::STRICT_FROUNDEVEN:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FROUNDEVEN_MERGE_PASSTHRU);
case ISD::FTRUNC:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FTRUNC_MERGE_PASSTHRU);
- case ISD::STRICT_FTRUNC:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FTRUNC_MERGE_PASSTHRU);
case ISD::FSQRT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSQRT_MERGE_PASSTHRU);
- case ISD::STRICT_FSQRT:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FSQRT_MERGE_PASSTHRU);
case ISD::FABS:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FABS_MERGE_PASSTHRU);
case ISD::FP_ROUND:
@@ -8851,22 +8795,14 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerToScalableOp(Op, DAG);
case ISD::FMAXIMUM:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAX_PRED);
- case ISD::STRICT_FMAXIMUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMAX_PRED);
case ISD::FMAXNUM:
case ISD::FMAXNUM_IEEE:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAXNM_PRED);
- case ISD::STRICT_FMAXNUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMAXNM_PRED);
case ISD::FMINIMUM:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMIN_PRED);
- case ISD::STRICT_FMINIMUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMIN_PRED);
case ISD::FMINNUM:
case ISD::FMINNUM_IEEE:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);
- case ISD::STRICT_FMINNUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMINNM_PRED);
case ISD::VSELECT:
return LowerFixedLengthVectorSelectToSVE(Op, DAG);
case ISD::ABS:
@@ -8900,8 +8836,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_INTERLEAVE(Op, DAG);
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
- case ISD::VECTOR_MATCH:
- return LowerVectorMatch(Op, DAG);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
@@ -19998,8 +19932,8 @@ bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
}
bool AArch64TargetLowering::isLegalAddScalableImmediate(int64_t Imm) const {
- // We will only emit addvl/inc* instructions if the subtarget allows it.
- if (!Subtarget->useScalarIncVL())
+ // We will only emit addvl/inc* instructions for SVE2
+ if (!Subtarget->hasSVE2())
return false;
// addvl's immediates are in terms of the number of bytes in a register.
@@ -25332,6 +25266,18 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
+ case Intrinsic::aarch64_sve_saddwb:
+ return DAG.getNode(AArch64ISD::SADDWB, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_saddwt:
+ return DAG.getNode(AArch64ISD::SADDWT, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_uaddwb:
+ return DAG.getNode(AArch64ISD::UADDWB, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_uaddwt:
+ return DAG.getNode(AArch64ISD::UADDWT, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
case Intrinsic::aarch64_sve_eor_u:
return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
@@ -31233,7 +31179,54 @@ static SDValue performPredicateLoadCombine(SDNode *N,
DAG.makeEquivalentMemoryOrdering(Load, LoadPred);
return LoadPred;
}
+static SDValue performFPToIntToDivCombine(SDNode *N, SelectionDAG &DAG) {
+ unsigned Opc = N->getOpcode();
+ bool IsSigned = (Opc == ISD::FP_TO_SINT);
+
+ SDValue FDiv = N->getOperand(0);
+ if (FDiv.getOpcode() != ISD::FDIV)
+ return SDValue();
+
+ EVT IntVT = N->getValueType(0);
+ EVT FPVT = FDiv.getValueType();
+
+ if (!IntVT.isVector() || !FPVT.isVector())
+ return SDValue();
+ if (IntVT.getVectorElementType() != MVT::i32 ||
+ FPVT.getVectorElementType() != MVT::f64)
+ return SDValue();
+
+ unsigned CastOpc = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+ SDValue Op0 = FDiv.getOperand(0);
+ SDValue Op1 = FDiv.getOperand(1);
+ if (Op0.getOpcode() != CastOpc || Op1.getOpcode() != CastOpc)
+ return SDValue();
+
+ if (Op0.getOperand(0).getValueType() != IntVT ||
+ Op1.getOperand(0).getValueType() != IntVT)
+ return SDValue();
+
+ unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV;
+ SDValue N0 = Op0.getOperand(0);
+ SDValue N1 = Op1.getOperand(0);
+
+ if (IntVT == MVT::nxv2i32) {
+ EVT WideVT = MVT::nxv4i32;
+ SDValue Undef = DAG.getUNDEF(WideVT);
+ SDValue ZeroIdx = DAG.getConstant(0, SDLoc(N), MVT::i64);
+
+ N0 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WideVT, Undef, N0,
+ ZeroIdx);
+ N1 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WideVT, Undef, N1,
+ ZeroIdx);
+
+ SDValue Div = DAG.getNode(DivOpc, SDLoc(N), WideVT, N0, N1);
+ return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), IntVT, Div, ZeroIdx);
+ }
+
+ return DAG.getNode(DivOpc, SDLoc(N), IntVT, N0, N1);
+}
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
@@ -31301,6 +31294,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performIntToFpCombine(N, DAG, DCI, Subtarget);
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
+ if (SDValue V = performFPToIntToDivCombine(N, DAG))
+ return V;
+ return performFpToIntCombine(N, DAG, DCI, Subtarget);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
return performFpToIntCombine(N, DAG, DCI, Subtarget);
@@ -32400,21 +32396,6 @@ void AArch64TargetLowering::ReplaceNodeResults(
case ISD::GET_ACTIVE_LANE_MASK:
ReplaceGetActiveLaneMaskResults(N, Results, DAG);
return;
- case ISD::VECTOR_MATCH: {
- EVT VT = N->getValueType(0);
- if (!VT.isFixedLengthVectorOf(MVT::i1))
- return;
-
- // NOTE: Only trivial type promotion is supported.
- EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
- if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
- return;
-
- SDLoc DL(N);
- SDValue V = DAG.getNode(ISD::VECTOR_MATCH, DL, NewVT, N->ops());
- Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
- return;
- }
case ISD::INTRINSIC_WO_CHAIN: {
EVT VT = N->getValueType(0);
@@ -32471,6 +32452,20 @@ void AArch64TargetLowering::ReplaceNodeResults(
DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, RuntimePStateSM));
return;
}
+ case Intrinsic::experimental_vector_match: {
+ if (!VT.isFixedLengthVectorOf(MVT::i1))
+ return;
+
+ // NOTE: Only trivial type promotion is supported.
+ EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
+ if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
+ return;
+
+ SDLoc DL(N);
+ auto V = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, NewVT, N->ops());
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
+ return;
+ }
}
}
case ISD::READ_REGISTER: {
@@ -33990,9 +33985,6 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
SDLoc DL(Op);
auto Pg = getPredicateForVector(DAG, DL, VT);
- bool IsStrictFP = Op->isStrictFPOpcode();
- unsigned OpStartIdx = IsStrictFP ? 1 : 0;
-
if (VT.isFixedLengthVector()) {
assert(isTypeLegal(VT) && "Expected only legal fixed-width types");
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
@@ -34027,18 +34019,8 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
- SmallVector<SDValue, 4> Operands;
-
- // Chain
- if (IsStrictFP)
- Operands.push_back(Op.getOperand(0));
-
- // Predicate
- Operands.push_back(Pg);
-
- // Regular operands
- for (unsigned OpIdx = OpStartIdx; OpIdx < Op.getNumOperands(); OpIdx++) {
- SDValue V = Op.getOperand(OpIdx);
+ SmallVector<SDValue, 4> Operands = {Pg};
+ for (const SDValue &V : Op->op_values()) {
assert((!V.getValueType().isVector() ||
V.getValueType().isScalableVector()) &&
"Only scalable vectors are supported!");
@@ -34048,7 +34030,7 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
if (isMergePassthruOpcode(NewOp))
Operands.push_back(DAG.getPOISON(VT));
- return DAG.getNode(NewOp, DL, Op->getVTList(), Operands, Op->getFlags());
+ return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags());
}
// If a fixed length vector operation has no side effects when applied to
@@ -34799,8 +34781,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
/// Lower a PARTIAL_REDUCE_MLA node. Three cases are handled:
/// 1. (v2i32, v16i8): widen Acc to v4i32 and fold the high half with ADDP.
-/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via (nx)v4i32, using
-/// (U|S)ADALP when available, otherwise add(add(Acc, ext(lo), ext(hi))).
+/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via v4i32, using
+/// (U|S)ADDW(B|T) when available, otherwise add(add(Acc, ext(lo), ext(hi))).
/// 3. SUMLA on (v4i32, v16i8) or (v2i32, v8i8) without +i8mm: rewrite as two
/// UDOTs using the bias-128 identity sext(s) = zext(s ^ 128) - 128.
SDValue
@@ -34827,17 +34809,6 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getExtractSubvector(DL, MVT::v2i32, Reduced, 0);
}
- // Handle (v2i64, v16i8) in two steps via v4i32 and Neon [SU]ADALP.
- if (Subtarget->isNeonAvailable() && ResultVT == MVT::v2i64 &&
- OpVT == MVT::v16i8) {
- SDValue Wide = DAG.getNode(Op.getOpcode(), DL, MVT::v4i32,
- DAG.getConstant(0, DL, MVT::v4i32), LHS, RHS);
- bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- unsigned Opc = IsUnsigned ? AArch64ISD::UADDLP : AArch64ISD::SADDLP;
- return DAG.getNode(ISD::ADD, DL, ResultVT, Acc,
- DAG.getNode(Opc, DL, ResultVT, Wide));
- }
-
// Lower PARTIAL_REDUCE_SUMLA on targets without +i8mm using udot via
// sum(sext(LHS) * zext(RHS)) =
// sum(zext(LHS ^ 128) * zext(RHS)) - sum(128 * zext(RHS))
@@ -34885,12 +34856,10 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
- unsigned IID = IsUnsigned ? Intrinsic::aarch64_sve_uadalp
- : Intrinsic::aarch64_sve_sadalp;
- SDValue Pg = getPredicateForVector(DAG, DL, ResultVT);
- SDValue Res =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResultVT,
- DAG.getConstant(IID, DL, MVT::i64), Pg, Acc, DotNode);
+ unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
+ unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
+ SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
+ SDValue Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res)
: Res;
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
new file mode 100644
index 0000000000000..5c06d8ded424b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 2 x i32> @fptoui_fdiv_uitofp_nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: fptoui_fdiv_uitofp_nxv2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %fa = uitofp <vscale x 2 x i32> %a to <vscale x 2 x double>
+ %fb = uitofp <vscale x 2 x i32> %b to <vscale x 2 x double>
+ %fdiv = fdiv <vscale x 2 x double> %fa, %fb
+ %res = fptoui <vscale x 2 x double> %fdiv to <vscale x 2 x i32>
+ ret <vscale x 2 x i32> %res
+}
+
+define <vscale x 2 x i32> @fptosi_fdiv_sitofp_nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: fptosi_fdiv_sitofp_nxv2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %fa = sitofp <vscale x 2 x i32> %a to <vscale x 2 x double>
+ %fb = sitofp <vscale x 2 x i32> %b to <vscale x 2 x double>
+ %fdiv = fdiv <vscale x 2 x double> %fa, %fb
+ %res = fptosi <vscale x 2 x double> %fdiv to <vscale x 2 x i32>
+ ret <vscale x 2 x i32> %res
+}
+
+define <2 x i32> @fptoui_fdiv_uitofp_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: fptoui_fdiv_uitofp_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %fa = uitofp <2 x i32> %a to <2 x double>
+ %fb = uitofp <2 x i32> %b to <2 x double>
+ %fdiv = fdiv <2 x double> %fa, %fb
+ %res = fptoui <2 x double> %fdiv to <2 x i32>
+ ret <2 x i32> %res
+}
+
+define <2 x i32> @fptosi_fdiv_sitofp_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: fptosi_fdiv_sitofp_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %fa = sitofp <2 x i32> %a to <2 x double>
+ %fb = sitofp <2 x i32> %b to <2 x double>
+ %fdiv = fdiv <2 x double> %fa, %fb
+ %res = fptosi <2 x double> %fdiv to <2 x i32>
+ ret <2 x i32> %res
+}
More information about the llvm-commits
mailing list