[llvm-branch-commits] [llvm] [RISCV] Support i64 accumulator for Zvdot4a8i partial reductions (PR #215517)
Pengcheng Wang via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 20 01:23:02 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/215517
>From e252d0cbeb0a1a3ee8c7e03052d1e37035f70d4a Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 11 Aug 2026 18:58:52 +0800
Subject: [PATCH 1/6] Format test
Created using spr 1.3.6-beta.1
---
.../RISCV/partial-reduce-dot-product-i64.ll | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll
index 459c896357809..a1e6d460d32c6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4
-; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefix=NOTAILFOLD
-; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s | FileCheck %s --check-prefix=TAILFOLD
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s \
+; RUN: | FileCheck %s --check-prefix=NOTAILFOLD
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s \
+; RUN: | FileCheck %s --check-prefix=TAILFOLD
; Dot product with an i64 accumulator and i8 inputs. This forms a scale-8
; partial reduction (i8 -> i64), which the RISC-V backend lowers using vdot4a*
@@ -11,7 +13,7 @@
target triple = "riscv64-none-unknown-elf"
-define i64 @vqdot_i64(ptr %a, ptr %b) #0 {
+define i64 @vqdot_i64(ptr %a, ptr %b) {
; NOTAILFOLD-LABEL: define i64 @vqdot_i64(
; NOTAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; NOTAILFOLD-NEXT: entry:
@@ -102,5 +104,3 @@ for.body:
for.exit:
ret i64 %add
}
-
-attributes #0 = { "target-features"="+v,+experimental-zvdot4a8i" }
>From b43a10a9a81bad361f1bb7d3a34f6d76cbef73bf Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Thu, 13 Aug 2026 16:02:14 +0800
Subject: [PATCH 2/6] Address comments: add more tests and do i32 reduction
first
Created using spr 1.3.6-beta.1
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 41 +-
.../CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll | 705 +++++++++++++++++-
2 files changed, 698 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 48b4baca61994..152e04ffb26ca 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -9902,34 +9902,35 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
// vdot4a* only produces an i32 result. For an i64 accumulator, perform the
// dot product into a fresh i32 accumulator (each result is the sum of four
- // i8 products, which cannot overflow i32), then widen the i32 partial sums
- // to i64 and accumulate. This mirrors the AArch64 sdot+sadalp idiom.
+ // i8 products, which cannot overflow i32), reduce those i32 partial sums
+ // down to the accumulator's element count while still in i32 (a sum of eight
+ // i8 products still cannot overflow i32), and only then extend to i64 and add
+ // to the accumulator.
if (VT.getVectorElementType() == MVT::i64) {
assert(Accum.getSimpleValueType() == VT);
// vdot4a* reduces each group of four i8 lanes into one i32 lane, so the
// intermediate i32 result has 1/4 the element count of the i8 inputs.
- MVT I32VT = MVT::getVectorVT(
+ MVT DotVT = MVT::getVectorVT(
MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4));
- SDValue Dot = DAG.getNode(Op.getOpcode(), DL, I32VT,
- {DAG.getConstant(0, DL, I32VT), A, B});
- // Widen the i32 partial sums to i64. They are signed for SMLA/SUMLA and
- // unsigned for UMLA.
+ SDValue Dot = DAG.getNode(Op.getOpcode(), DL, DotVT,
+ {DAG.getConstant(0, DL, DotVT), A, B});
+ // Add the i32 partial sums down to the accumulator's element count by
+ // extracting and summing the subvectors (still in i32 to avoid a wider
+ // extend).
+ MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
+ unsigned Stride = NarrowVT.getVectorMinNumElements();
+ SDValue Sum = DAG.getExtractSubvector(DL, NarrowVT, Dot, 0);
+ for (unsigned I = 1, E = DotVT.getVectorMinNumElements() / Stride; I != E;
+ ++I)
+ Sum = DAG.getNode(ISD::ADD, DL, NarrowVT, Sum,
+ DAG.getExtractSubvector(DL, NarrowVT, Dot, I * Stride));
+ // Extend the reduced i32 sums to i64 and accumulate. They are signed for
+ // SMLA/SUMLA and unsigned for UMLA.
unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
? ISD::ZERO_EXTEND
: ISD::SIGN_EXTEND;
- MVT WideVT = I32VT.changeVectorElementType(MVT::i64);
- SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot);
- // The widened dot result has twice the elements of the i64 accumulator.
- // Reduce it in by splitting into subvectors matching the accumulator and
- // adding them together (the same lowering the generic expander would use
- // for a multiplier-free partial reduction, but without a redundant mul).
- unsigned Stride = VT.getVectorMinNumElements();
- SDValue Res = Accum;
- for (unsigned I = 0, E = WideVT.getVectorMinNumElements() / Stride; I != E;
- ++I)
- Res = DAG.getNode(ISD::ADD, DL, VT, Res,
- DAG.getExtractSubvector(DL, VT, Wide, I * Stride));
- return Res;
+ return DAG.getNode(ISD::ADD, DL, VT, Accum,
+ DAG.getNode(ExtOpc, DL, VT, Sum));
}
assert(Accum.getSimpleValueType() == VT &&
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
index 740da5c6584da..10b0fa0723128 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
@@ -49,13 +49,14 @@ define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8>
; DOT-LABEL: vdot4a_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v12, 0
-; DOT-NEXT: vdot4a.vv v12, v9, v10
-; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
-; DOT-NEXT: vsext.vf2 v10, v12
-; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; DOT-NEXT: vadd.vv v8, v8, v10
-; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4a.vv v11, v9, v10
+; DOT-NEXT: csrr a0, vlenb
+; DOT-NEXT: srli a0, a0, 3
+; DOT-NEXT: vslidedown.vx v9, v11, a0
+; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
@@ -105,13 +106,14 @@ define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8
; DOT-LABEL: vdot4au_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v12, 0
-; DOT-NEXT: vdot4au.vv v12, v9, v10
-; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
-; DOT-NEXT: vzext.vf2 v10, v12
-; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; DOT-NEXT: vadd.vv v8, v8, v10
-; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4au.vv v11, v9, v10
+; DOT-NEXT: csrr a0, vlenb
+; DOT-NEXT: srli a0, a0, 3
+; DOT-NEXT: vslidedown.vx v9, v11, a0
+; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwaddu.wv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64>
@@ -162,13 +164,14 @@ define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i
; DOT-LABEL: vdot4asu_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v12, 0
-; DOT-NEXT: vdot4asu.vv v12, v9, v10
-; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
-; DOT-NEXT: vsext.vf2 v10, v12
-; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; DOT-NEXT: vadd.vv v8, v8, v10
-; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4asu.vv v11, v9, v10
+; DOT-NEXT: csrr a0, vlenb
+; DOT-NEXT: srli a0, a0, 3
+; DOT-NEXT: vslidedown.vx v9, v11, a0
+; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
@@ -228,13 +231,11 @@ define <vscale x 2 x i64> @vdot4a_i64_m2(<vscale x 2 x i64> %acc, <vscale x 16 x
; DOT-LABEL: vdot4a_i64_m2:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
-; DOT-NEXT: vmv.v.i v16, 0
-; DOT-NEXT: vdot4a.vv v16, v10, v12
-; DOT-NEXT: vsetvli zero, zero, e64, m4, ta, ma
-; DOT-NEXT: vsext.vf2 v12, v16
-; DOT-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; DOT-NEXT: vadd.vv v8, v8, v12
-; DOT-NEXT: vadd.vv v8, v8, v14
+; DOT-NEXT: vmv.v.i v14, 0
+; DOT-NEXT: vdot4a.vv v14, v10, v12
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vadd.vv v10, v14, v15
+; DOT-NEXT: vwadd.wv v8, v8, v10
; DOT-NEXT: ret
entry:
%a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
@@ -243,6 +244,654 @@ entry:
%res = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> %acc, <vscale x 16 x i64> %mul)
ret <vscale x 2 x i64> %res
}
+
+; The <vscale x 8 x i64> accumulator has an illegal result type, so the
+; sign/zero extend produced by the lowering must itself be legalized.
+define <vscale x 8 x i64> @vdot4a_i64_m8(<vscale x 8 x i64> %acc, <vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64_m8:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 4
+; NODOT32-NEXT: sub sp, sp, a1
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vl8r.v v8, (a0)
+; NODOT32-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v4, v16
+; NODOT32-NEXT: vsext.vf4 v0, v8
+; NODOT32-NEXT: vwmul.vv v24, v4, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: addi a1, sp, 16
+; NODOT32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v28, v17
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv2r.v v2, v10
+; NODOT32-NEXT: vmv1r.v v1, v9
+; NODOT32-NEXT: vsext.vf4 v24, v9
+; NODOT32-NEXT: vwmul.vv v0, v28, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v18
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv2r.v v2, v10
+; NODOT32-NEXT: vsext.vf4 v28, v2
+; NODOT32-NEXT: vwmul.vv v0, v24, v28
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v19
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv1r.v v3, v11
+; NODOT32-NEXT: vsext.vf4 v16, v11
+; NODOT32-NEXT: vwmul.vv v0, v24, v16
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v20
+; NODOT32-NEXT: vmv4r.v v28, v12
+; NODOT32-NEXT: vsext.vf4 v24, v28
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v21
+; NODOT32-NEXT: vsext.vf4 v24, v13
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v22
+; NODOT32-NEXT: vmv2r.v v30, v14
+; NODOT32-NEXT: vsext.vf4 v24, v30
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v23
+; NODOT32-NEXT: vsext.vf4 v28, v15
+; NODOT32-NEXT: vwmul.vv v16, v24, v28
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v8, a0, v16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64_m8:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vl8r.v v24, (a0)
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v4, v16
+; NODOT64-NEXT: vsext.vf4 v0, v24
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v17
+; NODOT64-NEXT: vsext.vf4 v0, v25
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v18
+; NODOT64-NEXT: vsext.vf4 v0, v26
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v19
+; NODOT64-NEXT: vsext.vf4 v16, v27
+; NODOT64-NEXT: vwmacc.vv v8, v4, v16
+; NODOT64-NEXT: vsext.vf4 v16, v20
+; NODOT64-NEXT: vsext.vf4 v24, v28
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v21
+; NODOT64-NEXT: vsext.vf4 v24, v29
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v22
+; NODOT64-NEXT: vsext.vf4 v24, v30
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v23
+; NODOT64-NEXT: vsext.vf4 v20, v31
+; NODOT64-NEXT: vwmacc.vv v8, v16, v20
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64_m8:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vl8r.v v24, (a0)
+; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma
+; DOT-NEXT: vmv.v.i v0, 0
+; DOT-NEXT: vdot4a.vv v0, v16, v24
+; DOT-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; DOT-NEXT: vadd.vv v16, v0, v4
+; DOT-NEXT: vwadd.wv v8, v8, v16
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 64 x i8> %a to <vscale x 64 x i64>
+ %b.sext = sext <vscale x 64 x i8> %b to <vscale x 64 x i64>
+ %mul = mul <vscale x 64 x i64> %a.sext, %b.sext
+ %res = call <vscale x 8 x i64> @llvm.experimental.vector.partial.reduce.add.nxv8i64.nxv64i64(<vscale x 8 x i64> %acc, <vscale x 64 x i64> %mul)
+ ret <vscale x 8 x i64> %res
+}
+
+define <2 x i64> @vdot4a_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; NODOT32-LABEL: vdot4a_v2i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v9
+; NODOT32-NEXT: vsext.vf4 v20, v10
+; NODOT32-NEXT: vwmul.vv v8, v16, v20
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v8, a0
+; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v16, 14
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v0, v16, 12
+; NODOT32-NEXT: vslidedown.vi v24, v16, 10
+; NODOT32-NEXT: vslidedown.vi v8, v16, 8
+; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 6
+; NODOT32-NEXT: vslidedown.vi v20, v16, 4
+; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v10, v16, 2
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v9, v16
+; NODOT32-NEXT: vadd.vv v11, v24, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v9, v24, v9
+; NODOT32-NEXT: vadd.vv v8, v12, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v20
+; NODOT32-NEXT: vadd.vv v9, v11, v9
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v8, v9
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_v2i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vsext.vf4 v24, v10
+; NODOT64-NEXT: vwmul.vv v16, v12, v24
+; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v16, 14
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v0, v16, 12
+; NODOT64-NEXT: vslidedown.vi v24, v16, 10
+; NODOT64-NEXT: vslidedown.vi v8, v16, 8
+; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 6
+; NODOT64-NEXT: vslidedown.vi v20, v16, 4
+; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v10, v16, 2
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v9, v16
+; NODOT64-NEXT: vadd.vv v11, v24, v0
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v9, v24, v9
+; NODOT64-NEXT: vadd.vv v8, v12, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v20
+; NODOT64-NEXT: vadd.vv v9, v11, v9
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v8, v9
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_v2i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4a.vv v11, v9, v10
+; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; DOT-NEXT: vslidedown.vi v9, v11, 2
+; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <16 x i8> %a to <16 x i64>
+ %b.sext = sext <16 x i8> %b to <16 x i64>
+ %mul = mul <16 x i64> %a.sext, %b.sext
+ %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @vdot4au_v4i64(<4 x i64> %acc, <32 x i8> %a, <32 x i8> %b) {
+; NODOT32-LABEL: vdot4au_v4i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 1
+; NODOT32-NEXT: mv a1, a0
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, a0, a1
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v8, v10, v12
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v24, v8
+; NODOT32-NEXT: vwmulu.vx v16, v24, a0
+; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v24, v16, 12
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v24, v16, 8
+; NODOT32-NEXT: addi a1, sp, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetivli zero, 16, e8, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v10, 16
+; NODOT32-NEXT: vslidedown.vi v10, v12, 16
+; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v12, v8, v10
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v20, v12
+; NODOT32-NEXT: vwmulu.vx v8, v20, a0
+; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v24, v8, 12
+; NODOT32-NEXT: vslidedown.vi v0, v8, 8
+; NODOT32-NEXT: vsetivli zero, 4, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 4
+; NODOT32-NEXT: vslidedown.vi v20, v8, 4
+; NODOT32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; NODOT32-NEXT: vadd.vv v8, v24, v8
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v10, v12, v24
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v12, v12, v16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v12, v24, v12
+; NODOT32-NEXT: vadd.vv v10, v10, v12
+; NODOT32-NEXT: vadd.vv v8, v8, v10
+; NODOT32-NEXT: vadd.vv v10, v20, v0
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 1
+; NODOT32-NEXT: mv a1, a0
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, a0, a1
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4au_v4i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 1
+; NODOT64-NEXT: mv a1, a0
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, a0, a1
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 4
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v8, v10, v12
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v16, v8
+; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v24, v16, 12
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v24, v16, 8
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e8, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v10, 16
+; NODOT64-NEXT: vslidedown.vi v10, v12, 16
+; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v20, v8, v10
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v8, v20
+; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v24, v8, 12
+; NODOT64-NEXT: vslidedown.vi v0, v8, 8
+; NODOT64-NEXT: vsetivli zero, 4, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 4
+; NODOT64-NEXT: vslidedown.vi v20, v8, 4
+; NODOT64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; NODOT64-NEXT: vadd.vv v8, v24, v8
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v10, v12, v24
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 4
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v12, v12, v16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v12, v24, v12
+; NODOT64-NEXT: vadd.vv v10, v10, v12
+; NODOT64-NEXT: vadd.vv v8, v8, v10
+; NODOT64-NEXT: vadd.vv v10, v20, v0
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 1
+; NODOT64-NEXT: mv a1, a0
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, a0, a1
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4au_v4i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v14, 0
+; DOT-NEXT: vdot4au.vv v14, v10, v12
+; DOT-NEXT: vsetivli zero, 4, e32, m2, ta, ma
+; DOT-NEXT: vslidedown.vi v10, v14, 4
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vadd.vv v10, v14, v10
+; DOT-NEXT: vwaddu.wv v8, v8, v10
+; DOT-NEXT: ret
+entry:
+ %a.zext = zext <32 x i8> %a to <32 x i64>
+ %b.zext = zext <32 x i8> %b to <32 x i64>
+ %mul = mul <32 x i64> %a.zext, %b.zext
+ %res = call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v32i64(<4 x i64> %acc, <32 x i64> %mul)
+ ret <4 x i64> %res
+}
+
+define <2 x i64> @vdot4asu_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; NODOT32-LABEL: vdot4asu_v2i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v9
+; NODOT32-NEXT: vzext.vf4 v20, v10
+; NODOT32-NEXT: vwmulsu.vv v8, v16, v20
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v8, a0
+; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v16, 14
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v0, v16, 12
+; NODOT32-NEXT: vslidedown.vi v24, v16, 10
+; NODOT32-NEXT: vslidedown.vi v8, v16, 8
+; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 6
+; NODOT32-NEXT: vslidedown.vi v20, v16, 4
+; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v10, v16, 2
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v9, v16
+; NODOT32-NEXT: vadd.vv v11, v24, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v9, v24, v9
+; NODOT32-NEXT: vadd.vv v8, v12, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v20
+; NODOT32-NEXT: vadd.vv v9, v11, v9
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v8, v9
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4asu_v2i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vzext.vf4 v24, v10
+; NODOT64-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v16, 14
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v0, v16, 12
+; NODOT64-NEXT: vslidedown.vi v24, v16, 10
+; NODOT64-NEXT: vslidedown.vi v8, v16, 8
+; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 6
+; NODOT64-NEXT: vslidedown.vi v20, v16, 4
+; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v10, v16, 2
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v9, v16
+; NODOT64-NEXT: vadd.vv v11, v24, v0
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v9, v24, v9
+; NODOT64-NEXT: vadd.vv v8, v12, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v20
+; NODOT64-NEXT: vadd.vv v9, v11, v9
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v8, v9
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4asu_v2i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4asu.vv v11, v9, v10
+; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; DOT-NEXT: vslidedown.vi v9, v11, 2
+; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <16 x i8> %a to <16 x i64>
+ %b.zext = zext <16 x i8> %b to <16 x i64>
+ %mul = mul <16 x i64> %a.sext, %b.zext
+ %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
+ ret <2 x i64> %res
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
; DOT32: {{.*}}
>From 0e960327f152255f9f890fda16ee18f4fe16cb56 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 18 Aug 2026 20:17:00 +0800
Subject: [PATCH 3/6] Handle M1 case and remove vslide
Created using spr 1.3.6-beta.1
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 32 ++++++++++---
.../CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll | 45 +++++++++----------
2 files changed, 46 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 01392a651428d..f53c536864356 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -50,6 +50,7 @@
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
#include <optional>
using namespace llvm;
@@ -9914,21 +9915,38 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4));
SDValue Dot = DAG.getNode(Op.getOpcode(), DL, DotVT,
{DAG.getConstant(0, DL, DotVT), A, B});
+ // The reduced i32 sums are signed for SMLA/SUMLA and unsigned for UMLA.
+ unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
+ ? ISD::ZERO_EXTEND
+ : ISD::SIGN_EXTEND;
+
+ MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
+ if (VT.isScalableVector() &&
+ RISCVTargetLowering::getLMUL(NarrowVT) == RISCVVType::LMUL_1) {
+ // When the accumulator is a single vector (LMUL 1), the i32 subvectors
+ // are a fractional LMUL, so extracting the high subvector would need a
+ // vslidedown. Widen the i32 sums to i64 first instead; the i64
+ // subvectors are then register-aligned and the reduction is a plain add.
+ MVT WideVT = DotVT.changeVectorElementType(MVT::i64);
+ SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot);
+ unsigned Stride = VT.getVectorMinNumElements();
+ SDValue Sum = DAG.getExtractSubvector(DL, VT, Wide, 0);
+ for (unsigned I = 1, E = WideVT.getVectorMinNumElements() / Stride;
+ I != E; ++I)
+ Sum = DAG.getNode(ISD::ADD, DL, VT, Sum,
+ DAG.getExtractSubvector(DL, VT, Wide, I * Stride));
+ return DAG.getNode(ISD::ADD, DL, VT, Accum, Sum);
+ }
+
// Add the i32 partial sums down to the accumulator's element count by
// extracting and summing the subvectors (still in i32 to avoid a wider
- // extend).
- MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
+ // extend), then extend once to i64 and accumulate.
unsigned Stride = NarrowVT.getVectorMinNumElements();
SDValue Sum = DAG.getExtractSubvector(DL, NarrowVT, Dot, 0);
for (unsigned I = 1, E = DotVT.getVectorMinNumElements() / Stride; I != E;
++I)
Sum = DAG.getNode(ISD::ADD, DL, NarrowVT, Sum,
DAG.getExtractSubvector(DL, NarrowVT, Dot, I * Stride));
- // Extend the reduced i32 sums to i64 and accumulate. They are signed for
- // SMLA/SUMLA and unsigned for UMLA.
- unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
- ? ISD::ZERO_EXTEND
- : ISD::SIGN_EXTEND;
return DAG.getNode(ISD::ADD, DL, VT, Accum,
DAG.getNode(ExtOpc, DL, VT, Sum));
}
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
index 10b0fa0723128..11f307d20cc16 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
@@ -49,14 +49,13 @@ define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8>
; DOT-LABEL: vdot4a_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v11, 0
-; DOT-NEXT: vdot4a.vv v11, v9, v10
-; DOT-NEXT: csrr a0, vlenb
-; DOT-NEXT: srli a0, a0, 3
-; DOT-NEXT: vslidedown.vx v9, v11, a0
-; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
-; DOT-NEXT: vadd.vv v9, v11, v9
-; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4a.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
@@ -106,14 +105,13 @@ define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8
; DOT-LABEL: vdot4au_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v11, 0
-; DOT-NEXT: vdot4au.vv v11, v9, v10
-; DOT-NEXT: csrr a0, vlenb
-; DOT-NEXT: srli a0, a0, 3
-; DOT-NEXT: vslidedown.vx v9, v11, a0
-; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
-; DOT-NEXT: vadd.vv v9, v11, v9
-; DOT-NEXT: vwaddu.wv v8, v8, v9
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4au.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vzext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64>
@@ -164,14 +162,13 @@ define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i
; DOT-LABEL: vdot4asu_i64:
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v11, 0
-; DOT-NEXT: vdot4asu.vv v11, v9, v10
-; DOT-NEXT: csrr a0, vlenb
-; DOT-NEXT: srli a0, a0, 3
-; DOT-NEXT: vslidedown.vx v9, v11, a0
-; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
-; DOT-NEXT: vadd.vv v9, v11, v9
-; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4asu.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
; DOT-NEXT: ret
entry:
%a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
>From a4fefc40c67a4ff44568b64dbbb1a9cbbffd3cee Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 18 Aug 2026 20:18:37 +0800
Subject: [PATCH 4/6] Remove extra include
Created using spr 1.3.6-beta.1
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f53c536864356..e0aa61021dc20 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -50,7 +50,6 @@
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
-#include "llvm/TargetParser/RISCVTargetParser.h"
#include <optional>
using namespace llvm;
>From 7845f03535654b6c9fc382ec234124cbc1adc4a2 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 18 Aug 2026 20:27:41 +0800
Subject: [PATCH 5/6] Use LMUL_RESERVED
Created using spr 1.3.6-beta.1
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e0aa61021dc20..3ad9e80e6d456 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -9921,7 +9921,7 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
if (VT.isScalableVector() &&
- RISCVTargetLowering::getLMUL(NarrowVT) == RISCVVType::LMUL_1) {
+ RISCVTargetLowering::getLMUL(NarrowVT) > RISCVVType::LMUL_RESERVED) {
// When the accumulator is a single vector (LMUL 1), the i32 subvectors
// are a fractional LMUL, so extracting the high subvector would need a
// vslidedown. Widen the i32 sums to i64 first instead; the i64
>From e1e72a21d60600f4c7c00e4481221a67f18f5bab Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Wed, 19 Aug 2026 15:11:19 +0800
Subject: [PATCH 6/6] Use decodeVLMUL and split tests
Created using spr 1.3.6-beta.1
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +-
.../RISCV/rvv/fixed-vectors-zvdot4a8i-i64.ll | 452 ++++++++++++++++++
.../CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll | 443 +----------------
3 files changed, 457 insertions(+), 441 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i-i64.ll
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3ad9e80e6d456..ebd5443751682 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -9921,7 +9921,8 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
if (VT.isScalableVector() &&
- RISCVTargetLowering::getLMUL(NarrowVT) > RISCVVType::LMUL_RESERVED) {
+ RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(NarrowVT))
+ .second) {
// When the accumulator is a single vector (LMUL 1), the i32 subvectors
// are a fractional LMUL, so extracting the high subvector would need a
// vslidedown. Widen the i32 sums to i64 first instead; the i64
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i-i64.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i-i64.ll
new file mode 100644
index 0000000000000..1f32254f3c732
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i-i64.ll
@@ -0,0 +1,452 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT64
+
+; Tests for partial.reduce.add into an i64 accumulator with i8 inputs. When
+; Zvdot4a8i is available these are lowered to a vdot4a* (i8 -> i32) followed by
+; a widening (i32 -> i64) accumulate, mirroring the AArch64 sdot+sadalp idiom.
+
+define <2 x i64> @vdot4a_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; NODOT32-LABEL: vdot4a_v2i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v9
+; NODOT32-NEXT: vsext.vf4 v20, v10
+; NODOT32-NEXT: vwmul.vv v8, v16, v20
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v8, a0
+; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v16, 14
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v0, v16, 12
+; NODOT32-NEXT: vslidedown.vi v24, v16, 10
+; NODOT32-NEXT: vslidedown.vi v8, v16, 8
+; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 6
+; NODOT32-NEXT: vslidedown.vi v20, v16, 4
+; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v10, v16, 2
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v9, v16
+; NODOT32-NEXT: vadd.vv v11, v24, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v9, v24, v9
+; NODOT32-NEXT: vadd.vv v8, v12, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v20
+; NODOT32-NEXT: vadd.vv v9, v11, v9
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v8, v9
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_v2i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vsext.vf4 v24, v10
+; NODOT64-NEXT: vwmul.vv v16, v12, v24
+; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v16, 14
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v0, v16, 12
+; NODOT64-NEXT: vslidedown.vi v24, v16, 10
+; NODOT64-NEXT: vslidedown.vi v8, v16, 8
+; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 6
+; NODOT64-NEXT: vslidedown.vi v20, v16, 4
+; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v10, v16, 2
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v9, v16
+; NODOT64-NEXT: vadd.vv v11, v24, v0
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v9, v24, v9
+; NODOT64-NEXT: vadd.vv v8, v12, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v20
+; NODOT64-NEXT: vadd.vv v9, v11, v9
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v8, v9
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_v2i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4a.vv v11, v9, v10
+; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; DOT-NEXT: vslidedown.vi v9, v11, 2
+; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <16 x i8> %a to <16 x i64>
+ %b.sext = sext <16 x i8> %b to <16 x i64>
+ %mul = mul <16 x i64> %a.sext, %b.sext
+ %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @vdot4au_v4i64(<4 x i64> %acc, <32 x i8> %a, <32 x i8> %b) {
+; NODOT32-LABEL: vdot4au_v4i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 1
+; NODOT32-NEXT: mv a1, a0
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, a0, a1
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v8, v10, v12
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v24, v8
+; NODOT32-NEXT: vwmulu.vx v16, v24, a0
+; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v24, v16, 12
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v24, v16, 8
+; NODOT32-NEXT: addi a1, sp, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetivli zero, 16, e8, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v10, 16
+; NODOT32-NEXT: vslidedown.vi v10, v12, 16
+; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v12, v8, v10
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v20, v12
+; NODOT32-NEXT: vwmulu.vx v8, v20, a0
+; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v24, v8, 12
+; NODOT32-NEXT: vslidedown.vi v0, v8, 8
+; NODOT32-NEXT: vsetivli zero, 4, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 4
+; NODOT32-NEXT: vslidedown.vi v20, v8, 4
+; NODOT32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; NODOT32-NEXT: vadd.vv v8, v24, v8
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v10, v12, v24
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v12, v12, v16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v12, v24, v12
+; NODOT32-NEXT: vadd.vv v10, v10, v12
+; NODOT32-NEXT: vadd.vv v8, v8, v10
+; NODOT32-NEXT: vadd.vv v10, v20, v0
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 1
+; NODOT32-NEXT: mv a1, a0
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, a0, a1
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4au_v4i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 1
+; NODOT64-NEXT: mv a1, a0
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, a0, a1
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 4
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v8, v10, v12
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v16, v8
+; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v24, v16, 12
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v24, v16, 8
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e8, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v10, 16
+; NODOT64-NEXT: vslidedown.vi v10, v12, 16
+; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v20, v8, v10
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v8, v20
+; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v24, v8, 12
+; NODOT64-NEXT: vslidedown.vi v0, v8, 8
+; NODOT64-NEXT: vsetivli zero, 4, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 4
+; NODOT64-NEXT: vslidedown.vi v20, v8, 4
+; NODOT64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; NODOT64-NEXT: vadd.vv v8, v24, v8
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v10, v12, v24
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 4
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v12, v12, v16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v12, v24, v12
+; NODOT64-NEXT: vadd.vv v10, v10, v12
+; NODOT64-NEXT: vadd.vv v8, v8, v10
+; NODOT64-NEXT: vadd.vv v10, v20, v0
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 1
+; NODOT64-NEXT: mv a1, a0
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, a0, a1
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4au_v4i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v14, 0
+; DOT-NEXT: vdot4au.vv v14, v10, v12
+; DOT-NEXT: vsetivli zero, 4, e32, m2, ta, ma
+; DOT-NEXT: vslidedown.vi v10, v14, 4
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vadd.vv v10, v14, v10
+; DOT-NEXT: vwaddu.wv v8, v8, v10
+; DOT-NEXT: ret
+entry:
+ %a.zext = zext <32 x i8> %a to <32 x i64>
+ %b.zext = zext <32 x i8> %b to <32 x i64>
+ %mul = mul <32 x i64> %a.zext, %b.zext
+ %res = call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v32i64(<4 x i64> %acc, <32 x i64> %mul)
+ ret <4 x i64> %res
+}
+
+define <2 x i64> @vdot4asu_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; NODOT32-LABEL: vdot4asu_v2i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: sub sp, sp, a0
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v9
+; NODOT32-NEXT: vzext.vf4 v20, v10
+; NODOT32-NEXT: vwmulsu.vv v8, v16, v20
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v8, a0
+; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT32-NEXT: vslidedown.vi v8, v16, 14
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vslidedown.vi v0, v16, 12
+; NODOT32-NEXT: vslidedown.vi v24, v16, 10
+; NODOT32-NEXT: vslidedown.vi v8, v16, 8
+; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT32-NEXT: vslidedown.vi v12, v16, 6
+; NODOT32-NEXT: vslidedown.vi v20, v16, 4
+; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT32-NEXT: vslidedown.vi v10, v16, 2
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 3
+; NODOT32-NEXT: add a0, sp, a0
+; NODOT32-NEXT: addi a0, a0, 16
+; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v9, v16
+; NODOT32-NEXT: vadd.vv v11, v24, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vadd.vv v9, v24, v9
+; NODOT32-NEXT: vadd.vv v8, v12, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v20
+; NODOT32-NEXT: vadd.vv v9, v11, v9
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v8, v9
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a1, a0, 3
+; NODOT32-NEXT: add a0, a1, a0
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4asu_v2i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: addi sp, sp, -16
+; NODOT64-NEXT: .cfi_def_cfa_offset 16
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: sub sp, sp, a0
+; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vzext.vf4 v24, v10
+; NODOT64-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
+; NODOT64-NEXT: vslidedown.vi v8, v16, 14
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; NODOT64-NEXT: vslidedown.vi v0, v16, 12
+; NODOT64-NEXT: vslidedown.vi v24, v16, 10
+; NODOT64-NEXT: vslidedown.vi v8, v16, 8
+; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
+; NODOT64-NEXT: vslidedown.vi v12, v16, 6
+; NODOT64-NEXT: vslidedown.vi v20, v16, 4
+; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
+; NODOT64-NEXT: vslidedown.vi v10, v16, 2
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a0, a0, 3
+; NODOT64-NEXT: add a0, sp, a0
+; NODOT64-NEXT: addi a0, a0, 16
+; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v9, v16
+; NODOT64-NEXT: vadd.vv v11, v24, v0
+; NODOT64-NEXT: addi a0, sp, 16
+; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; NODOT64-NEXT: vadd.vv v9, v24, v9
+; NODOT64-NEXT: vadd.vv v8, v12, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v20
+; NODOT64-NEXT: vadd.vv v9, v11, v9
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v8, v9
+; NODOT64-NEXT: csrr a0, vlenb
+; NODOT64-NEXT: slli a1, a0, 3
+; NODOT64-NEXT: add a0, a1, a0
+; NODOT64-NEXT: add sp, sp, a0
+; NODOT64-NEXT: .cfi_def_cfa sp, 16
+; NODOT64-NEXT: addi sp, sp, 16
+; NODOT64-NEXT: .cfi_def_cfa_offset 0
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4asu_v2i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v11, 0
+; DOT-NEXT: vdot4asu.vv v11, v9, v10
+; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; DOT-NEXT: vslidedown.vi v9, v11, 2
+; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; DOT-NEXT: vadd.vv v9, v11, v9
+; DOT-NEXT: vwadd.wv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <16 x i8> %a to <16 x i64>
+ %b.zext = zext <16 x i8> %b to <16 x i64>
+ %mul = mul <16 x i64> %a.sext, %b.zext
+ %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
+ ret <2 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; DOT32: {{.*}}
+; DOT64: {{.*}}
+; NODOT: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
index 11f307d20cc16..4b4f2b0eeaedf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
@@ -242,8 +242,9 @@ entry:
ret <vscale x 2 x i64> %res
}
-; The <vscale x 8 x i64> accumulator has an illegal result type, so the
-; sign/zero extend produced by the lowering must itself be legalized.
+; The widest (LMUL 8) i64 accumulator: the i32 partial sums are reduced down
+; to LMUL 4 in i32 and then widened into the accumulator with a single
+; vwadd.wv.
define <vscale x 8 x i64> @vdot4a_i64_m8(<vscale x 8 x i64> %acc, <vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {
; NODOT32-LABEL: vdot4a_i64_m8:
; NODOT32: # %bb.0: # %entry
@@ -451,444 +452,6 @@ entry:
%res = call <vscale x 8 x i64> @llvm.experimental.vector.partial.reduce.add.nxv8i64.nxv64i64(<vscale x 8 x i64> %acc, <vscale x 64 x i64> %mul)
ret <vscale x 8 x i64> %res
}
-
-define <2 x i64> @vdot4a_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
-; NODOT32-LABEL: vdot4a_v2i64:
-; NODOT32: # %bb.0: # %entry
-; NODOT32-NEXT: addi sp, sp, -16
-; NODOT32-NEXT: .cfi_def_cfa_offset 16
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a1, a0, 3
-; NODOT32-NEXT: add a0, a1, a0
-; NODOT32-NEXT: sub sp, sp, a0
-; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; NODOT32-NEXT: li a0, 1
-; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
-; NODOT32-NEXT: vsext.vf4 v16, v9
-; NODOT32-NEXT: vsext.vf4 v20, v10
-; NODOT32-NEXT: vwmul.vv v8, v16, v20
-; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
-; NODOT32-NEXT: vmul.vx v16, v8, a0
-; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
-; NODOT32-NEXT: vslidedown.vi v8, v16, 14
-; NODOT32-NEXT: addi a0, sp, 16
-; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; NODOT32-NEXT: vslidedown.vi v0, v16, 12
-; NODOT32-NEXT: vslidedown.vi v24, v16, 10
-; NODOT32-NEXT: vslidedown.vi v8, v16, 8
-; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
-; NODOT32-NEXT: vslidedown.vi v12, v16, 6
-; NODOT32-NEXT: vslidedown.vi v20, v16, 4
-; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; NODOT32-NEXT: vslidedown.vi v10, v16, 2
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
-; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; NODOT32-NEXT: vadd.vv v9, v9, v16
-; NODOT32-NEXT: vadd.vv v11, v24, v0
-; NODOT32-NEXT: addi a0, sp, 16
-; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT32-NEXT: vadd.vv v9, v24, v9
-; NODOT32-NEXT: vadd.vv v8, v12, v8
-; NODOT32-NEXT: vadd.vv v10, v10, v20
-; NODOT32-NEXT: vadd.vv v9, v11, v9
-; NODOT32-NEXT: vadd.vv v8, v10, v8
-; NODOT32-NEXT: vadd.vv v8, v8, v9
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a1, a0, 3
-; NODOT32-NEXT: add a0, a1, a0
-; NODOT32-NEXT: add sp, sp, a0
-; NODOT32-NEXT: .cfi_def_cfa sp, 16
-; NODOT32-NEXT: addi sp, sp, 16
-; NODOT32-NEXT: .cfi_def_cfa_offset 0
-; NODOT32-NEXT: ret
-;
-; NODOT64-LABEL: vdot4a_v2i64:
-; NODOT64: # %bb.0: # %entry
-; NODOT64-NEXT: addi sp, sp, -16
-; NODOT64-NEXT: .cfi_def_cfa_offset 16
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a1, a0, 3
-; NODOT64-NEXT: add a0, a1, a0
-; NODOT64-NEXT: sub sp, sp, a0
-; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
-; NODOT64-NEXT: vsext.vf4 v12, v9
-; NODOT64-NEXT: vsext.vf4 v24, v10
-; NODOT64-NEXT: vwmul.vv v16, v12, v24
-; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
-; NODOT64-NEXT: vslidedown.vi v8, v16, 14
-; NODOT64-NEXT: addi a0, sp, 16
-; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; NODOT64-NEXT: vslidedown.vi v0, v16, 12
-; NODOT64-NEXT: vslidedown.vi v24, v16, 10
-; NODOT64-NEXT: vslidedown.vi v8, v16, 8
-; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
-; NODOT64-NEXT: vslidedown.vi v12, v16, 6
-; NODOT64-NEXT: vslidedown.vi v20, v16, 4
-; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; NODOT64-NEXT: vslidedown.vi v10, v16, 2
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
-; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; NODOT64-NEXT: vadd.vv v9, v9, v16
-; NODOT64-NEXT: vadd.vv v11, v24, v0
-; NODOT64-NEXT: addi a0, sp, 16
-; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT64-NEXT: vadd.vv v9, v24, v9
-; NODOT64-NEXT: vadd.vv v8, v12, v8
-; NODOT64-NEXT: vadd.vv v10, v10, v20
-; NODOT64-NEXT: vadd.vv v9, v11, v9
-; NODOT64-NEXT: vadd.vv v8, v10, v8
-; NODOT64-NEXT: vadd.vv v8, v8, v9
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a1, a0, 3
-; NODOT64-NEXT: add a0, a1, a0
-; NODOT64-NEXT: add sp, sp, a0
-; NODOT64-NEXT: .cfi_def_cfa sp, 16
-; NODOT64-NEXT: addi sp, sp, 16
-; NODOT64-NEXT: .cfi_def_cfa_offset 0
-; NODOT64-NEXT: ret
-;
-; DOT-LABEL: vdot4a_v2i64:
-; DOT: # %bb.0: # %entry
-; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v11, 0
-; DOT-NEXT: vdot4a.vv v11, v9, v10
-; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; DOT-NEXT: vslidedown.vi v9, v11, 2
-; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; DOT-NEXT: vadd.vv v9, v11, v9
-; DOT-NEXT: vwadd.wv v8, v8, v9
-; DOT-NEXT: ret
-entry:
- %a.sext = sext <16 x i8> %a to <16 x i64>
- %b.sext = sext <16 x i8> %b to <16 x i64>
- %mul = mul <16 x i64> %a.sext, %b.sext
- %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
- ret <2 x i64> %res
-}
-
-define <4 x i64> @vdot4au_v4i64(<4 x i64> %acc, <32 x i8> %a, <32 x i8> %b) {
-; NODOT32-LABEL: vdot4au_v4i64:
-; NODOT32: # %bb.0: # %entry
-; NODOT32-NEXT: addi sp, sp, -16
-; NODOT32-NEXT: .cfi_def_cfa_offset 16
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 1
-; NODOT32-NEXT: mv a1, a0
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, a0, a1
-; NODOT32-NEXT: sub sp, sp, a0
-; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 4
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; NODOT32-NEXT: li a0, 1
-; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT32-NEXT: vwmulu.vv v8, v10, v12
-; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT32-NEXT: vzext.vf2 v24, v8
-; NODOT32-NEXT: vwmulu.vx v16, v24, a0
-; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
-; NODOT32-NEXT: vslidedown.vi v24, v16, 12
-; NODOT32-NEXT: csrr a1, vlenb
-; NODOT32-NEXT: slli a1, a1, 3
-; NODOT32-NEXT: add a1, sp, a1
-; NODOT32-NEXT: addi a1, a1, 16
-; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
-; NODOT32-NEXT: vslidedown.vi v24, v16, 8
-; NODOT32-NEXT: addi a1, sp, 16
-; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
-; NODOT32-NEXT: vsetivli zero, 16, e8, m2, ta, ma
-; NODOT32-NEXT: vslidedown.vi v8, v10, 16
-; NODOT32-NEXT: vslidedown.vi v10, v12, 16
-; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT32-NEXT: vwmulu.vv v12, v8, v10
-; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT32-NEXT: vzext.vf2 v20, v12
-; NODOT32-NEXT: vwmulu.vx v8, v20, a0
-; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma
-; NODOT32-NEXT: vslidedown.vi v24, v8, 12
-; NODOT32-NEXT: vslidedown.vi v0, v8, 8
-; NODOT32-NEXT: vsetivli zero, 4, e64, m4, ta, ma
-; NODOT32-NEXT: vslidedown.vi v12, v16, 4
-; NODOT32-NEXT: vslidedown.vi v20, v8, 4
-; NODOT32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; NODOT32-NEXT: vadd.vv v8, v24, v8
-; NODOT32-NEXT: addi a0, sp, 16
-; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT32-NEXT: vadd.vv v10, v12, v24
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 4
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
-; NODOT32-NEXT: vadd.vv v12, v12, v16
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT32-NEXT: vadd.vv v12, v24, v12
-; NODOT32-NEXT: vadd.vv v10, v10, v12
-; NODOT32-NEXT: vadd.vv v8, v8, v10
-; NODOT32-NEXT: vadd.vv v10, v20, v0
-; NODOT32-NEXT: vadd.vv v8, v10, v8
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 1
-; NODOT32-NEXT: mv a1, a0
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, a0, a1
-; NODOT32-NEXT: add sp, sp, a0
-; NODOT32-NEXT: .cfi_def_cfa sp, 16
-; NODOT32-NEXT: addi sp, sp, 16
-; NODOT32-NEXT: .cfi_def_cfa_offset 0
-; NODOT32-NEXT: ret
-;
-; NODOT64-LABEL: vdot4au_v4i64:
-; NODOT64: # %bb.0: # %entry
-; NODOT64-NEXT: addi sp, sp, -16
-; NODOT64-NEXT: .cfi_def_cfa_offset 16
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 1
-; NODOT64-NEXT: mv a1, a0
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, a0, a1
-; NODOT64-NEXT: sub sp, sp, a0
-; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 4
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT64-NEXT: vwmulu.vv v8, v10, v12
-; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
-; NODOT64-NEXT: vzext.vf4 v16, v8
-; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
-; NODOT64-NEXT: vslidedown.vi v24, v16, 12
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
-; NODOT64-NEXT: vslidedown.vi v24, v16, 8
-; NODOT64-NEXT: addi a0, sp, 16
-; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
-; NODOT64-NEXT: vsetivli zero, 16, e8, m2, ta, ma
-; NODOT64-NEXT: vslidedown.vi v8, v10, 16
-; NODOT64-NEXT: vslidedown.vi v10, v12, 16
-; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT64-NEXT: vwmulu.vv v20, v8, v10
-; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
-; NODOT64-NEXT: vzext.vf4 v8, v20
-; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma
-; NODOT64-NEXT: vslidedown.vi v24, v8, 12
-; NODOT64-NEXT: vslidedown.vi v0, v8, 8
-; NODOT64-NEXT: vsetivli zero, 4, e64, m4, ta, ma
-; NODOT64-NEXT: vslidedown.vi v12, v16, 4
-; NODOT64-NEXT: vslidedown.vi v20, v8, 4
-; NODOT64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; NODOT64-NEXT: vadd.vv v8, v24, v8
-; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT64-NEXT: vadd.vv v10, v12, v24
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 4
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
-; NODOT64-NEXT: vadd.vv v12, v12, v16
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT64-NEXT: vadd.vv v12, v24, v12
-; NODOT64-NEXT: vadd.vv v10, v10, v12
-; NODOT64-NEXT: vadd.vv v8, v8, v10
-; NODOT64-NEXT: vadd.vv v10, v20, v0
-; NODOT64-NEXT: vadd.vv v8, v10, v8
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 1
-; NODOT64-NEXT: mv a1, a0
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, a0, a1
-; NODOT64-NEXT: add sp, sp, a0
-; NODOT64-NEXT: .cfi_def_cfa sp, 16
-; NODOT64-NEXT: addi sp, sp, 16
-; NODOT64-NEXT: .cfi_def_cfa_offset 0
-; NODOT64-NEXT: ret
-;
-; DOT-LABEL: vdot4au_v4i64:
-; DOT: # %bb.0: # %entry
-; DOT-NEXT: vsetivli zero, 8, e32, m2, ta, ma
-; DOT-NEXT: vmv.v.i v14, 0
-; DOT-NEXT: vdot4au.vv v14, v10, v12
-; DOT-NEXT: vsetivli zero, 4, e32, m2, ta, ma
-; DOT-NEXT: vslidedown.vi v10, v14, 4
-; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; DOT-NEXT: vadd.vv v10, v14, v10
-; DOT-NEXT: vwaddu.wv v8, v8, v10
-; DOT-NEXT: ret
-entry:
- %a.zext = zext <32 x i8> %a to <32 x i64>
- %b.zext = zext <32 x i8> %b to <32 x i64>
- %mul = mul <32 x i64> %a.zext, %b.zext
- %res = call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v32i64(<4 x i64> %acc, <32 x i64> %mul)
- ret <4 x i64> %res
-}
-
-define <2 x i64> @vdot4asu_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
-; NODOT32-LABEL: vdot4asu_v2i64:
-; NODOT32: # %bb.0: # %entry
-; NODOT32-NEXT: addi sp, sp, -16
-; NODOT32-NEXT: .cfi_def_cfa_offset 16
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a1, a0, 3
-; NODOT32-NEXT: add a0, a1, a0
-; NODOT32-NEXT: sub sp, sp, a0
-; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; NODOT32-NEXT: li a0, 1
-; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
-; NODOT32-NEXT: vsext.vf4 v16, v9
-; NODOT32-NEXT: vzext.vf4 v20, v10
-; NODOT32-NEXT: vwmulsu.vv v8, v16, v20
-; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
-; NODOT32-NEXT: vmul.vx v16, v8, a0
-; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma
-; NODOT32-NEXT: vslidedown.vi v8, v16, 14
-; NODOT32-NEXT: addi a0, sp, 16
-; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; NODOT32-NEXT: vslidedown.vi v0, v16, 12
-; NODOT32-NEXT: vslidedown.vi v24, v16, 10
-; NODOT32-NEXT: vslidedown.vi v8, v16, 8
-; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma
-; NODOT32-NEXT: vslidedown.vi v12, v16, 6
-; NODOT32-NEXT: vslidedown.vi v20, v16, 4
-; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; NODOT32-NEXT: vslidedown.vi v10, v16, 2
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a0, a0, 3
-; NODOT32-NEXT: add a0, sp, a0
-; NODOT32-NEXT: addi a0, a0, 16
-; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
-; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; NODOT32-NEXT: vadd.vv v9, v9, v16
-; NODOT32-NEXT: vadd.vv v11, v24, v0
-; NODOT32-NEXT: addi a0, sp, 16
-; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT32-NEXT: vadd.vv v9, v24, v9
-; NODOT32-NEXT: vadd.vv v8, v12, v8
-; NODOT32-NEXT: vadd.vv v10, v10, v20
-; NODOT32-NEXT: vadd.vv v9, v11, v9
-; NODOT32-NEXT: vadd.vv v8, v10, v8
-; NODOT32-NEXT: vadd.vv v8, v8, v9
-; NODOT32-NEXT: csrr a0, vlenb
-; NODOT32-NEXT: slli a1, a0, 3
-; NODOT32-NEXT: add a0, a1, a0
-; NODOT32-NEXT: add sp, sp, a0
-; NODOT32-NEXT: .cfi_def_cfa sp, 16
-; NODOT32-NEXT: addi sp, sp, 16
-; NODOT32-NEXT: .cfi_def_cfa_offset 0
-; NODOT32-NEXT: ret
-;
-; NODOT64-LABEL: vdot4asu_v2i64:
-; NODOT64: # %bb.0: # %entry
-; NODOT64-NEXT: addi sp, sp, -16
-; NODOT64-NEXT: .cfi_def_cfa_offset 16
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a1, a0, 3
-; NODOT64-NEXT: add a0, a1, a0
-; NODOT64-NEXT: sub sp, sp, a0
-; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma
-; NODOT64-NEXT: vsext.vf4 v12, v9
-; NODOT64-NEXT: vzext.vf4 v24, v10
-; NODOT64-NEXT: vwmulsu.vv v16, v12, v24
-; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma
-; NODOT64-NEXT: vslidedown.vi v8, v16, 14
-; NODOT64-NEXT: addi a0, sp, 16
-; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; NODOT64-NEXT: vslidedown.vi v0, v16, 12
-; NODOT64-NEXT: vslidedown.vi v24, v16, 10
-; NODOT64-NEXT: vslidedown.vi v8, v16, 8
-; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma
-; NODOT64-NEXT: vslidedown.vi v12, v16, 6
-; NODOT64-NEXT: vslidedown.vi v20, v16, 4
-; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; NODOT64-NEXT: vslidedown.vi v10, v16, 2
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a0, a0, 3
-; NODOT64-NEXT: add a0, sp, a0
-; NODOT64-NEXT: addi a0, a0, 16
-; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
-; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; NODOT64-NEXT: vadd.vv v9, v9, v16
-; NODOT64-NEXT: vadd.vv v11, v24, v0
-; NODOT64-NEXT: addi a0, sp, 16
-; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; NODOT64-NEXT: vadd.vv v9, v24, v9
-; NODOT64-NEXT: vadd.vv v8, v12, v8
-; NODOT64-NEXT: vadd.vv v10, v10, v20
-; NODOT64-NEXT: vadd.vv v9, v11, v9
-; NODOT64-NEXT: vadd.vv v8, v10, v8
-; NODOT64-NEXT: vadd.vv v8, v8, v9
-; NODOT64-NEXT: csrr a0, vlenb
-; NODOT64-NEXT: slli a1, a0, 3
-; NODOT64-NEXT: add a0, a1, a0
-; NODOT64-NEXT: add sp, sp, a0
-; NODOT64-NEXT: .cfi_def_cfa sp, 16
-; NODOT64-NEXT: addi sp, sp, 16
-; NODOT64-NEXT: .cfi_def_cfa_offset 0
-; NODOT64-NEXT: ret
-;
-; DOT-LABEL: vdot4asu_v2i64:
-; DOT: # %bb.0: # %entry
-; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; DOT-NEXT: vmv.v.i v11, 0
-; DOT-NEXT: vdot4asu.vv v11, v9, v10
-; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; DOT-NEXT: vslidedown.vi v9, v11, 2
-; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; DOT-NEXT: vadd.vv v9, v11, v9
-; DOT-NEXT: vwadd.wv v8, v8, v9
-; DOT-NEXT: ret
-entry:
- %a.sext = sext <16 x i8> %a to <16 x i64>
- %b.zext = zext <16 x i8> %b to <16 x i64>
- %mul = mul <16 x i64> %a.sext, %b.zext
- %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
- ret <2 x i64> %res
-}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
; DOT32: {{.*}}
More information about the llvm-branch-commits
mailing list