[llvm] [AArch64] missed fold vector fptoui(fdiv(uitofp(x), uitofp(y))) or fptosi(fdiv(sitofp(x), sitofp(y))) into udiv(x, y) or sdiv(x, y) for sve- #215146 (PR #215523)
Durgesh Nandan Mohanty via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 04:33:25 PDT 2026
https://github.com/dnmohanty updated https://github.com/llvm/llvm-project/pull/215523
>From cb4f77e8c124b9d7a99dda2420e07414ac7ee509 Mon Sep 17 00:00:00 2001
From: Durgesh Nandan Mohanty <durgeshnandanmohanty at gmail.com>
Date: Tue, 11 Aug 2026 11:15:35 +0000
Subject: [PATCH 1/2] Pre-commit baseline test for SVE fdiv fold (NFC)
---
.../test/CodeGen/AArch64/sve-fdiv-int-fold.ll | 66 +++++++++++++++++++
1 file changed, 66 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
new file mode 100644
index 0000000000000..5c06d8ded424b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 2 x i32> @fptoui_fdiv_uitofp_nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: fptoui_fdiv_uitofp_nxv2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %fa = uitofp <vscale x 2 x i32> %a to <vscale x 2 x double>
+ %fb = uitofp <vscale x 2 x i32> %b to <vscale x 2 x double>
+ %fdiv = fdiv <vscale x 2 x double> %fa, %fb
+ %res = fptoui <vscale x 2 x double> %fdiv to <vscale x 2 x i32>
+ ret <vscale x 2 x i32> %res
+}
+
+define <vscale x 2 x i32> @fptosi_fdiv_sitofp_nxv2i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: fptosi_fdiv_sitofp_nxv2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %fa = sitofp <vscale x 2 x i32> %a to <vscale x 2 x double>
+ %fb = sitofp <vscale x 2 x i32> %b to <vscale x 2 x double>
+ %fdiv = fdiv <vscale x 2 x double> %fa, %fb
+ %res = fptosi <vscale x 2 x double> %fdiv to <vscale x 2 x i32>
+ ret <vscale x 2 x i32> %res
+}
+
+define <2 x i32> @fptoui_fdiv_uitofp_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: fptoui_fdiv_uitofp_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %fa = uitofp <2 x i32> %a to <2 x double>
+ %fb = uitofp <2 x i32> %b to <2 x double>
+ %fdiv = fdiv <2 x double> %fa, %fb
+ %res = fptoui <2 x double> %fdiv to <2 x i32>
+ ret <2 x i32> %res
+}
+
+define <2 x i32> @fptosi_fdiv_sitofp_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: fptosi_fdiv_sitofp_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %fa = sitofp <2 x i32> %a to <2 x double>
+ %fb = sitofp <2 x i32> %b to <2 x double>
+ %fdiv = fdiv <2 x double> %fa, %fb
+ %res = fptosi <2 x double> %fdiv to <2 x i32>
+ ret <2 x i32> %res
+}
>From 76c25a8eb096d43664f9cae5b92e102719f18381 Mon Sep 17 00:00:00 2001
From: Durgesh Nandan Mohanty <durgeshnandanmohanty at gmail.com>
Date: Tue, 11 Aug 2026 11:18:12 +0000
Subject: [PATCH 2/2] [AArch64] Fold vector fptoui/fptosi to udiv/sdiv for SVE
---
.github/workflows/libcxx-benchmark-commit.yml | 158 ---------
.../Target/AArch64/AArch64ISelLowering.cpp | 317 ++++++++----------
2 files changed, 143 insertions(+), 332 deletions(-)
delete mode 100644 .github/workflows/libcxx-benchmark-commit.yml
diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml
deleted file mode 100644
index f55c11b0d93c6..0000000000000
--- a/.github/workflows/libcxx-benchmark-commit.yml
+++ /dev/null
@@ -1,158 +0,0 @@
-# This file defines a workflow that runs the libc++ benchmarks at the given commit.
-# This workflow is intended to be triggered manually or via the Github API. As such,
-# it requires several inputs that allow customizing its behavior.
-
-name: "[libc++] Run benchmark suite against commit"
-
-# Keep in sync with libcxx/utils/ci/lnt/dispatch-benchmarks
-run-name: "[libc++] Run benchmark suite against ${{ inputs.commit }} on ${{ inputs.lnt-machine }}"
-
-permissions:
- contents: read
-
-on:
- workflow_dispatch:
- inputs:
- commit:
- description: 'The libc++ commit to benchmark'
- required: true
- type: string
- lnt-machine:
- description: 'The LNT machine to run the benchmarks on'
- required: true
- type: string
- benchmark-suite-version:
- description: 'The version of the benchmark suite to use (a LLVM monorepo SHA)'
- required: true
- type: string
- filter:
- description: 'An optional filter to determine which benchmarks to run'
- required: false
- type: string
- submit-lnt:
- description: 'Whether to submit the results to LNT -- dry-run unless opted in'
- required: false
- type: boolean
- default: false
- lnt-url:
- description: 'The URL of the LNT instance to submit to'
- required: false
- type: string
- default: http://lnt.llvm.org
-
-jobs:
- # Determine which configuration to run based on the `lnt-machine` input.
- select-machine:
- runs-on: ubuntu-24.04
- outputs:
- matrix: ${{ steps.select.outputs.matrix }}
- steps:
- - name: Checkout the machine definitions
- uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- persist-credentials: false
- # Disabling cone mode allows checking out exactly the single file we need.
- sparse-checkout: libcxx/utils/ci/lnt/machines.json
- sparse-checkout-cone-mode: false
-
- - name: Select the configuration to benchmark on
- id: select
- uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- LNT_MACHINE: ${{ inputs.lnt-machine }}
- with:
- script: |
- const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
-
- const requested = process.env.LNT_MACHINE;
- const selected = config.filter(cfg => cfg['lnt-machine'] === requested);
- if (selected.length === 0) {
- const known = config.map(cfg => cfg['lnt-machine']).join(', ');
- core.setFailed(`Unknown LNT machine '${requested}' (known machines: ${known})`);
- return;
- }
-
- core.setOutput('matrix', JSON.stringify(selected));
-
- run-benchmarks:
- needs:
- - select-machine
- strategy:
- matrix:
- include: ${{ fromJSON(needs.select-machine.outputs.matrix) }}
- fail-fast: false
- runs-on: ${{ matrix.runner }}
- env:
- COMPILER: ${{ matrix.cxx }}
- steps:
- - name: Checkout the LLVM monorepo
- uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- persist-credentials: false
- # We benchmark arbitrary historical commits, which requires the full history to be available.
- fetch-depth: 0
- fetch-tags: true
-
- - name: Install Python
- if: ${{ matrix.running-on == 'linux' }} # installed via Homebrew on macOS
- uses: actions/setup-python at ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
- with:
- python-version: '3.14'
-
- - name: Select Xcode
- if: ${{ matrix.running-on == 'macos' }}
- run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV
-
- - name: Install dependencies via Homebrew
- if: ${{ matrix.running-on == 'macos' }}
- run: |
- brew update
- brew install ninja cmake python at 3.14
- echo "$(brew --prefix python at 3.14)/bin" >> "$GITHUB_PATH"
-
- - name: Diagnose tools in use
- run: |
- cmake --version
- ninja --version
- "${COMPILER}" --version
- python3 --version
-
- - name: Setup virtual environment
- run: |
- python3 -m venv .venv
- source .venv/bin/activate
- pip install -r libcxx/utils/requirements.txt
-
- - name: Run the benchmarks
- env:
- COMMIT: ${{ inputs.commit }}
- BENCHMARK_SUITE_VERSION: ${{ inputs.benchmark-suite-version }}
- FILTER: ${{ inputs.filter }}
- LNT_MACHINE: ${{ matrix.lnt-machine }}
- run: |
- source .venv/bin/activate
- filter_arg=()
- if [ -n "${FILTER}" ]; then
- filter_arg=(--filter "${FILTER}")
- fi
- libcxx/utils/ci/lnt/run-benchmarks \
- --test-suite-commit "${BENCHMARK_SUITE_VERSION}" \
- --machine "${LNT_MACHINE}" \
- --compiler "${COMPILER}" \
- --benchmark-commit "${COMMIT}" \
- "${filter_arg[@]}" \
- --output "${COMMIT}.json"
-
- cat "${COMMIT}.json"
-
- - name: Submit to LNT
- if: ${{ inputs.submit-lnt }}
- env:
- COMMIT: ${{ inputs.commit }}
- LNT_URL: ${{ inputs.lnt-url }}
- run: |
- source .venv/bin/activate
- libcxx/utils/ci/lnt/submit-benchmarks \
- --lnt-url "${LNT_URL}" \
- --test-suite libcxx \
- "${COMMIT}.json"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6003db1c72449..2e6bc7102a184 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -364,13 +364,6 @@ static bool isMergePassthruOpcode(unsigned Opc) {
case AArch64ISD::FSQRT_MERGE_PASSTHRU:
case AArch64ISD::FRECPX_MERGE_PASSTHRU:
case AArch64ISD::FABS_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FCEIL_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FFLOOR_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FNEARBYINT_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FROUND_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FROUNDEVEN_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FTRUNC_MERGE_PASSTHRU:
- case AArch64ISD::STRICT_FSQRT_MERGE_PASSTHRU:
return true;
}
}
@@ -1637,14 +1630,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
}
- if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
- for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
- setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
-
- for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
- setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
- }
-
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
if (Subtarget->isSVEorStreamingSVEAvailable() &&
@@ -1841,30 +1826,30 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
setOperationAction(ISD::SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction({ISD::FADD, ISD::STRICT_FADD}, VT, Custom);
+ setOperationAction(ISD::FADD, VT, Custom);
setOperationAction(ISD::FCANONICALIZE, VT, Custom);
setOperationAction(ISD::FCOPYSIGN, VT, Custom);
- setOperationAction({ISD::FDIV, ISD::STRICT_FDIV}, VT, Custom);
- setOperationAction({ISD::FMA, ISD::STRICT_FMA}, VT, Custom);
- setOperationAction({ISD::FMAXIMUM, ISD::STRICT_FMAXIMUM}, VT, Custom);
- setOperationAction({ISD::FMAXNUM, ISD::STRICT_FMAXNUM}, VT, Custom);
+ setOperationAction(ISD::FDIV, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Custom);
+ setOperationAction(ISD::FMAXIMUM, VT, Custom);
+ setOperationAction(ISD::FMAXNUM, VT, Custom);
setOperationAction(ISD::FMAXNUM_IEEE, VT, Custom);
- setOperationAction({ISD::FMINIMUM, ISD::STRICT_FMINIMUM}, VT, Custom);
- setOperationAction({ISD::FMINNUM, ISD::STRICT_FMINNUM}, VT, Custom);
+ setOperationAction(ISD::FMINIMUM, VT, Custom);
+ setOperationAction(ISD::FMINNUM, VT, Custom);
setOperationAction(ISD::FMINNUM_IEEE, VT, Custom);
- setOperationAction({ISD::FMUL, ISD::STRICT_FMUL}, VT, Custom);
+ setOperationAction(ISD::FMUL, VT, Custom);
setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction({ISD::FSUB, ISD::STRICT_FSUB}, VT, Custom);
- setOperationAction({ISD::FCEIL, ISD::STRICT_FCEIL}, VT, Custom);
- setOperationAction({ISD::FFLOOR, ISD::STRICT_FFLOOR}, VT, Custom);
- setOperationAction({ISD::FNEARBYINT, ISD::STRICT_FNEARBYINT}, VT, Custom);
+ setOperationAction(ISD::FSUB, VT, Custom);
+ setOperationAction(ISD::FCEIL, VT, Custom);
+ setOperationAction(ISD::FFLOOR, VT, Custom);
+ setOperationAction(ISD::FNEARBYINT, VT, Custom);
setOperationAction(ISD::FRINT, VT, Custom);
setOperationAction(ISD::LRINT, VT, Custom);
setOperationAction(ISD::LLRINT, VT, Custom);
- setOperationAction({ISD::FROUND, ISD::STRICT_FROUND}, VT, Custom);
- setOperationAction({ISD::FROUNDEVEN, ISD::STRICT_FROUNDEVEN}, VT, Custom);
- setOperationAction({ISD::FTRUNC, ISD::STRICT_FTRUNC}, VT, Custom);
- setOperationAction({ISD::FSQRT, ISD::STRICT_FSQRT}, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUNDEVEN, VT, Custom);
+ setOperationAction(ISD::FTRUNC, VT, Custom);
+ setOperationAction(ISD::FSQRT, VT, Custom);
setOperationAction(ISD::FABS, VT, Custom);
setOperationAction(ISD::FP_EXTEND, VT, Custom);
setOperationAction(ISD::FP_ROUND, VT, Custom);
@@ -1880,45 +1865,27 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Expand);
- setOperationAction({ISD::FREM, ISD::STRICT_FREM}, VT, Expand);
- setOperationAction({ISD::FPOW, ISD::STRICT_FPOW}, VT, Expand);
- setOperationAction({ISD::FPOWI, ISD::STRICT_FPOWI}, VT, Expand);
+ setOperationAction(ISD::FREM, VT, Expand);
+ setOperationAction(ISD::FPOW, VT, Expand);
+ setOperationAction(ISD::FPOWI, VT, Expand);
setOperationAction(ISD::FCBRT, VT, Expand);
- setOperationAction({ISD::FCOS, ISD::STRICT_FCOS}, VT, Expand);
- setOperationAction({ISD::FSIN, ISD::STRICT_FSIN}, VT, Expand);
+ setOperationAction(ISD::FCOS, VT, Expand);
+ setOperationAction(ISD::FSIN, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
- setOperationAction({ISD::FTAN, ISD::STRICT_FTAN}, VT, Expand);
- setOperationAction({ISD::FACOS, ISD::STRICT_FACOS}, VT, Expand);
- setOperationAction({ISD::FASIN, ISD::STRICT_FASIN}, VT, Expand);
- setOperationAction({ISD::FATAN, ISD::STRICT_FATAN}, VT, Expand);
- setOperationAction({ISD::FATAN2, ISD::STRICT_FATAN2}, VT, Expand);
- setOperationAction({ISD::FCOSH, ISD::STRICT_FCOSH}, VT, Expand);
- setOperationAction({ISD::FSINH, ISD::STRICT_FSINH}, VT, Expand);
- setOperationAction({ISD::FTANH, ISD::STRICT_FTANH}, VT, Expand);
- setOperationAction({ISD::FEXP, ISD::STRICT_FEXP}, VT, Expand);
- setOperationAction({ISD::FEXP2, ISD::STRICT_FEXP2}, VT, Expand);
+ setOperationAction(ISD::FTAN, VT, Expand);
+ setOperationAction(ISD::FACOS, VT, Expand);
+ setOperationAction(ISD::FASIN, VT, Expand);
+ setOperationAction(ISD::FATAN, VT, Expand);
+ setOperationAction(ISD::FATAN2, VT, Expand);
+ setOperationAction(ISD::FCOSH, VT, Expand);
+ setOperationAction(ISD::FSINH, VT, Expand);
+ setOperationAction(ISD::FTANH, VT, Expand);
+ setOperationAction(ISD::FEXP, VT, Expand);
+ setOperationAction(ISD::FEXP2, VT, Expand);
setOperationAction(ISD::FEXP10, VT, Expand);
- setOperationAction({ISD::FLOG, ISD::STRICT_FLOG}, VT, Expand);
- setOperationAction({ISD::FLOG2, ISD::STRICT_FLOG2}, VT, Expand);
- setOperationAction({ISD::FLOG10, ISD::STRICT_FLOG10}, VT, Expand);
-
- // TODO: These require custom lowering.
- setOperationAction(ISD::STRICT_FLDEXP, VT, Expand);
- setOperationAction(ISD::STRICT_FRINT, VT, Expand);
- setOperationAction(ISD::STRICT_PSEUDO_FMIN, VT, Expand);
- setOperationAction(ISD::STRICT_PSEUDO_FMAX, VT, Expand);
- setOperationAction(ISD::STRICT_LROUND, VT, Expand);
- setOperationAction(ISD::STRICT_LLROUND, VT, Expand);
- setOperationAction(ISD::STRICT_LRINT, VT, Expand);
- setOperationAction(ISD::STRICT_LLRINT, VT, Expand);
- setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Expand);
- setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Expand);
- setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Expand);
- setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Expand);
- setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand);
- setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand);
- setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
- setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
+ setOperationAction(ISD::FLOG, VT, Expand);
+ setOperationAction(ISD::FLOG2, VT, Expand);
+ setOperationAction(ISD::FLOG10, VT, Expand);
setCondCodeAction(ISD::SETO, VT, Expand);
setCondCodeAction(ISD::SETOLT, VT, Expand);
@@ -2492,6 +2459,19 @@ bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
VT != MVT::v4i1 && VT != MVT::v2i1;
}
+bool AArch64TargetLowering::shouldExpandVectorMatch(EVT VT,
+ unsigned SearchSize) const {
+ // MATCH is SVE2 and only available in non-streaming mode.
+ if (!Subtarget->hasSVE2() || !Subtarget->isSVEAvailable())
+ return true;
+ // Furthermore, we can only use it for 8-bit or 16-bit elements.
+ if (VT == MVT::nxv8i16 || VT == MVT::v8i16)
+ return SearchSize != 8;
+ if (VT == MVT::nxv16i8 || VT == MVT::v16i8 || VT == MVT::v8i8)
+ return SearchSize != 8 && SearchSize != 16;
+ return true;
+}
+
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
@@ -6479,29 +6459,21 @@ static SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
static SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
SDLoc DL(Op);
- auto Op1 = Op.getOperand(0);
- auto Op2 = Op.getOperand(1);
- auto Mask = Op.getOperand(2);
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
+
+ auto Op1 = Op.getOperand(1);
+ auto Op2 = Op.getOperand(2);
+ auto Mask = Op.getOperand(3);
EVT Op1VT = Op1.getValueType();
EVT Op2VT = Op2.getValueType();
EVT ResVT = Op.getValueType();
- unsigned SearchSize = Op2VT.getVectorNumElements();
assert((Op1VT.getVectorElementType() == MVT::i8 ||
Op1VT.getVectorElementType() == MVT::i16) &&
"Expected 8-bit or 16-bit characters.");
- if ((Op1VT == MVT::nxv8i16 || Op1VT == MVT::v8i16) && SearchSize != 8)
- return SDValue();
-
- if ((Op1VT == MVT::nxv16i8 || Op1VT == MVT::v16i8 || Op1VT == MVT::v8i8) &&
- SearchSize != 8 && SearchSize != 16)
- return SDValue();
-
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
-
// Scalable vector type used to wrap operands.
// A single container is enough for both operands because ultimately the
// operands will have to be wrapped to the same type (nxv16i8 or nxv8i16).
@@ -7228,6 +7200,9 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
ADDLV, DAG.getConstant(0, DL, MVT::i64));
return EXTRACT_VEC_ELT;
}
+ case Intrinsic::experimental_vector_match: {
+ return LowerVectorMatch(Op, DAG);
+ }
case Intrinsic::aarch64_cls:
case Intrinsic::aarch64_cls64: {
SDValue Res = DAG.getNode(ISD::CTLS, DL, Op.getOperand(1).getValueType(),
@@ -8620,63 +8595,32 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerXALUO(Op, DAG);
case ISD::FADD:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
- case ISD::STRICT_FADD:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FADD_PRED);
case ISD::FSUB:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
- case ISD::STRICT_FSUB:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FSUB_PRED);
case ISD::FMUL:
return LowerFMUL(Op, DAG);
- case ISD::STRICT_FMUL:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMUL_PRED);
case ISD::FMA:
return LowerFMA(Op, DAG);
- case ISD::STRICT_FMA:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMA_PRED);
case ISD::FDIV:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
- case ISD::STRICT_FDIV:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FDIV_PRED);
case ISD::FNEG:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEG_MERGE_PASSTHRU);
case ISD::FCEIL:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FCEIL_MERGE_PASSTHRU);
- case ISD::STRICT_FCEIL:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FCEIL_MERGE_PASSTHRU);
case ISD::FFLOOR:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FFLOOR_MERGE_PASSTHRU);
- case ISD::STRICT_FFLOOR:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FFLOOR_MERGE_PASSTHRU);
case ISD::FNEARBYINT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEARBYINT_MERGE_PASSTHRU);
- case ISD::STRICT_FNEARBYINT:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FNEARBYINT_MERGE_PASSTHRU);
case ISD::FRINT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FRINT_MERGE_PASSTHRU);
case ISD::FROUND:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUND_MERGE_PASSTHRU);
- case ISD::STRICT_FROUND:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FROUND_MERGE_PASSTHRU);
case ISD::FROUNDEVEN:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU);
- case ISD::STRICT_FROUNDEVEN:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FROUNDEVEN_MERGE_PASSTHRU);
case ISD::FTRUNC:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FTRUNC_MERGE_PASSTHRU);
- case ISD::STRICT_FTRUNC:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FTRUNC_MERGE_PASSTHRU);
case ISD::FSQRT:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSQRT_MERGE_PASSTHRU);
- case ISD::STRICT_FSQRT:
- return LowerToPredicatedOp(Op, DAG,
- AArch64ISD::STRICT_FSQRT_MERGE_PASSTHRU);
case ISD::FABS:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FABS_MERGE_PASSTHRU);
case ISD::FP_ROUND:
@@ -8851,22 +8795,14 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerToScalableOp(Op, DAG);
case ISD::FMAXIMUM:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAX_PRED);
- case ISD::STRICT_FMAXIMUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMAX_PRED);
case ISD::FMAXNUM:
case ISD::FMAXNUM_IEEE:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAXNM_PRED);
- case ISD::STRICT_FMAXNUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMAXNM_PRED);
case ISD::FMINIMUM:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMIN_PRED);
- case ISD::STRICT_FMINIMUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMIN_PRED);
case ISD::FMINNUM:
case ISD::FMINNUM_IEEE:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);
- case ISD::STRICT_FMINNUM:
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::STRICT_FMINNM_PRED);
case ISD::VSELECT:
return LowerFixedLengthVectorSelectToSVE(Op, DAG);
case ISD::ABS:
@@ -8900,8 +8836,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_INTERLEAVE(Op, DAG);
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
- case ISD::VECTOR_MATCH:
- return LowerVectorMatch(Op, DAG);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
@@ -19998,8 +19932,8 @@ bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
}
bool AArch64TargetLowering::isLegalAddScalableImmediate(int64_t Imm) const {
- // We will only emit addvl/inc* instructions if the subtarget allows it.
- if (!Subtarget->useScalarIncVL())
+ // We will only emit addvl/inc* instructions for SVE2
+ if (!Subtarget->hasSVE2())
return false;
// addvl's immediates are in terms of the number of bytes in a register.
@@ -25332,6 +25266,18 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
+ case Intrinsic::aarch64_sve_saddwb:
+ return DAG.getNode(AArch64ISD::SADDWB, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_saddwt:
+ return DAG.getNode(AArch64ISD::SADDWT, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_uaddwb:
+ return DAG.getNode(AArch64ISD::UADDWB, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
+ case Intrinsic::aarch64_sve_uaddwt:
+ return DAG.getNode(AArch64ISD::UADDWT, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
case Intrinsic::aarch64_sve_eor_u:
return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
@@ -31233,7 +31179,54 @@ static SDValue performPredicateLoadCombine(SDNode *N,
DAG.makeEquivalentMemoryOrdering(Load, LoadPred);
return LoadPred;
}
+static SDValue performFPToIntToDivCombine(SDNode *N, SelectionDAG &DAG) {
+ unsigned Opc = N->getOpcode();
+ bool IsSigned = (Opc == ISD::FP_TO_SINT);
+
+ SDValue FDiv = N->getOperand(0);
+ if (FDiv.getOpcode() != ISD::FDIV)
+ return SDValue();
+
+ EVT IntVT = N->getValueType(0);
+ EVT FPVT = FDiv.getValueType();
+
+ if (!IntVT.isVector() || !FPVT.isVector())
+ return SDValue();
+ if (IntVT.getVectorElementType() != MVT::i32 ||
+ FPVT.getVectorElementType() != MVT::f64)
+ return SDValue();
+
+ unsigned CastOpc = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+ SDValue Op0 = FDiv.getOperand(0);
+ SDValue Op1 = FDiv.getOperand(1);
+ if (Op0.getOpcode() != CastOpc || Op1.getOpcode() != CastOpc)
+ return SDValue();
+
+ if (Op0.getOperand(0).getValueType() != IntVT ||
+ Op1.getOperand(0).getValueType() != IntVT)
+ return SDValue();
+
+ unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV;
+ SDValue N0 = Op0.getOperand(0);
+ SDValue N1 = Op1.getOperand(0);
+
+ if (IntVT == MVT::nxv2i32) {
+ EVT WideVT = MVT::nxv4i32;
+ SDValue Undef = DAG.getUNDEF(WideVT);
+ SDValue ZeroIdx = DAG.getConstant(0, SDLoc(N), MVT::i64);
+
+ N0 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WideVT, Undef, N0,
+ ZeroIdx);
+ N1 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WideVT, Undef, N1,
+ ZeroIdx);
+
+ SDValue Div = DAG.getNode(DivOpc, SDLoc(N), WideVT, N0, N1);
+ return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), IntVT, Div, ZeroIdx);
+ }
+
+ return DAG.getNode(DivOpc, SDLoc(N), IntVT, N0, N1);
+}
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
@@ -31301,6 +31294,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performIntToFpCombine(N, DAG, DCI, Subtarget);
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
+ if (SDValue V = performFPToIntToDivCombine(N, DAG))
+ return V;
+ return performFpToIntCombine(N, DAG, DCI, Subtarget);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
return performFpToIntCombine(N, DAG, DCI, Subtarget);
@@ -32400,21 +32396,6 @@ void AArch64TargetLowering::ReplaceNodeResults(
case ISD::GET_ACTIVE_LANE_MASK:
ReplaceGetActiveLaneMaskResults(N, Results, DAG);
return;
- case ISD::VECTOR_MATCH: {
- EVT VT = N->getValueType(0);
- if (!VT.isFixedLengthVectorOf(MVT::i1))
- return;
-
- // NOTE: Only trivial type promotion is supported.
- EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
- if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
- return;
-
- SDLoc DL(N);
- SDValue V = DAG.getNode(ISD::VECTOR_MATCH, DL, NewVT, N->ops());
- Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
- return;
- }
case ISD::INTRINSIC_WO_CHAIN: {
EVT VT = N->getValueType(0);
@@ -32471,6 +32452,20 @@ void AArch64TargetLowering::ReplaceNodeResults(
DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, RuntimePStateSM));
return;
}
+ case Intrinsic::experimental_vector_match: {
+ if (!VT.isFixedLengthVectorOf(MVT::i1))
+ return;
+
+ // NOTE: Only trivial type promotion is supported.
+ EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
+ if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
+ return;
+
+ SDLoc DL(N);
+ auto V = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, NewVT, N->ops());
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
+ return;
+ }
}
}
case ISD::READ_REGISTER: {
@@ -33990,9 +33985,6 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
SDLoc DL(Op);
auto Pg = getPredicateForVector(DAG, DL, VT);
- bool IsStrictFP = Op->isStrictFPOpcode();
- unsigned OpStartIdx = IsStrictFP ? 1 : 0;
-
if (VT.isFixedLengthVector()) {
assert(isTypeLegal(VT) && "Expected only legal fixed-width types");
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
@@ -34027,18 +34019,8 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
- SmallVector<SDValue, 4> Operands;
-
- // Chain
- if (IsStrictFP)
- Operands.push_back(Op.getOperand(0));
-
- // Predicate
- Operands.push_back(Pg);
-
- // Regular operands
- for (unsigned OpIdx = OpStartIdx; OpIdx < Op.getNumOperands(); OpIdx++) {
- SDValue V = Op.getOperand(OpIdx);
+ SmallVector<SDValue, 4> Operands = {Pg};
+ for (const SDValue &V : Op->op_values()) {
assert((!V.getValueType().isVector() ||
V.getValueType().isScalableVector()) &&
"Only scalable vectors are supported!");
@@ -34048,7 +34030,7 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
if (isMergePassthruOpcode(NewOp))
Operands.push_back(DAG.getPOISON(VT));
- return DAG.getNode(NewOp, DL, Op->getVTList(), Operands, Op->getFlags());
+ return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags());
}
// If a fixed length vector operation has no side effects when applied to
@@ -34799,8 +34781,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
/// Lower a PARTIAL_REDUCE_MLA node. Three cases are handled:
/// 1. (v2i32, v16i8): widen Acc to v4i32 and fold the high half with ADDP.
-/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via (nx)v4i32, using
-/// (U|S)ADALP when available, otherwise add(add(Acc, ext(lo), ext(hi))).
+/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via v4i32, using
+/// (U|S)ADDW(B|T) when available, otherwise add(add(Acc, ext(lo), ext(hi))).
/// 3. SUMLA on (v4i32, v16i8) or (v2i32, v8i8) without +i8mm: rewrite as two
/// UDOTs using the bias-128 identity sext(s) = zext(s ^ 128) - 128.
SDValue
@@ -34827,17 +34809,6 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getExtractSubvector(DL, MVT::v2i32, Reduced, 0);
}
- // Handle (v2i64, v16i8) in two steps via v4i32 and Neon [SU]ADALP.
- if (Subtarget->isNeonAvailable() && ResultVT == MVT::v2i64 &&
- OpVT == MVT::v16i8) {
- SDValue Wide = DAG.getNode(Op.getOpcode(), DL, MVT::v4i32,
- DAG.getConstant(0, DL, MVT::v4i32), LHS, RHS);
- bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- unsigned Opc = IsUnsigned ? AArch64ISD::UADDLP : AArch64ISD::SADDLP;
- return DAG.getNode(ISD::ADD, DL, ResultVT, Acc,
- DAG.getNode(Opc, DL, ResultVT, Wide));
- }
-
// Lower PARTIAL_REDUCE_SUMLA on targets without +i8mm using udot via
// sum(sext(LHS) * zext(RHS)) =
// sum(zext(LHS ^ 128) * zext(RHS)) - sum(128 * zext(RHS))
@@ -34885,12 +34856,10 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
- unsigned IID = IsUnsigned ? Intrinsic::aarch64_sve_uadalp
- : Intrinsic::aarch64_sve_sadalp;
- SDValue Pg = getPredicateForVector(DAG, DL, ResultVT);
- SDValue Res =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResultVT,
- DAG.getConstant(IID, DL, MVT::i64), Pg, Acc, DotNode);
+ unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
+ unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
+ SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
+ SDValue Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res)
: Res;
}
More information about the llvm-commits
mailing list