[llvm-branch-commits] [llvm] [RISCV] Support i64 accumulator for Zvdot4a8i partial reductions (PR #215517)
Craig Topper via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Aug 18 09:02:15 PDT 2026
================
@@ -0,0 +1,896 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT64
+
+; Tests for partial.reduce.add into an i64 accumulator with i8 inputs. When
+; Zvdot4a8i is available these are lowered to a vdot4a* (i8 -> i32) followed by
+; a widening (i32 -> i64) accumulate, mirroring the AArch64 sdot+sadalp idiom.
+
+define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v12, v9
+; NODOT32-NEXT: vsext.vf4 v24, v10
+; NODOT32-NEXT: vwmul.vv v16, v12, v24
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vsext.vf4 v24, v10
+; NODOT64-NEXT: vwmul.vv v16, v12, v24
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4a.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.sext, %b.sext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4au_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v16, v9, v10
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v12, v16
+; NODOT32-NEXT: vwmulu.vx v16, v12, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4au_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v12, v9, v10
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v16, v12
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4au_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4au.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vzext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.zext, %b.zext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4asu_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v12, v9
+; NODOT32-NEXT: vzext.vf4 v24, v10
+; NODOT32-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4asu_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vzext.vf4 v24, v10
+; NODOT64-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4asu_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4asu.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v9, v10, v11
+; DOT-NEXT: vadd.vv v8, v8, v9
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.sext, %b.zext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 2 x i64> @vdot4a_i64_m2(<vscale x 2 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64_m2:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v10
+; NODOT32-NEXT: vsext.vf4 v4, v11
+; NODOT32-NEXT: vsext.vf4 v28, v12
+; NODOT32-NEXT: vsext.vf4 v0, v13
+; NODOT32-NEXT: vwmul.vv v16, v24, v28
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vwmul.vv v24, v4, v0
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v24, v24, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; NODOT32-NEXT: vadd.vv v10, v18, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v8, v22, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v24
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v30, v8
+; NODOT32-NEXT: vadd.vv v10, v26, v28
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64_m2:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v4, v10
+; NODOT64-NEXT: vsext.vf4 v24, v11
+; NODOT64-NEXT: vsext.vf4 v0, v12
+; NODOT64-NEXT: vsext.vf4 v28, v13
+; NODOT64-NEXT: vwmul.vv v16, v24, v28
+; NODOT64-NEXT: vwmul.vv v24, v4, v0
+; NODOT64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; NODOT64-NEXT: vadd.vv v10, v26, v28
+; NODOT64-NEXT: vadd.vv v8, v8, v24
+; NODOT64-NEXT: vadd.vv v8, v30, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v16
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v22, v8
+; NODOT64-NEXT: vadd.vv v10, v18, v20
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64_m2:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v14, 0
+; DOT-NEXT: vdot4a.vv v14, v10, v12
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vadd.vv v10, v14, v15
+; DOT-NEXT: vwadd.wv v8, v8, v10
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
+ %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i64>
+ %mul = mul <vscale x 16 x i64> %a.sext, %b.sext
+ %res = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> %acc, <vscale x 16 x i64> %mul)
+ ret <vscale x 2 x i64> %res
+}
+
+; The <vscale x 8 x i64> accumulator has an illegal result type, so the
+; sign/zero extend produced by the lowering must itself be legalized.
+define <vscale x 8 x i64> @vdot4a_i64_m8(<vscale x 8 x i64> %acc, <vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64_m8:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: addi sp, sp, -16
+; NODOT32-NEXT: .cfi_def_cfa_offset 16
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 4
+; NODOT32-NEXT: sub sp, sp, a1
+; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vl8r.v v8, (a0)
+; NODOT32-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v4, v16
+; NODOT32-NEXT: vsext.vf4 v0, v8
+; NODOT32-NEXT: vwmul.vv v24, v4, v0
+; NODOT32-NEXT: addi a0, sp, 16
+; NODOT32-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: addi a1, sp, 16
+; NODOT32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v28, v17
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv2r.v v2, v10
+; NODOT32-NEXT: vmv1r.v v1, v9
+; NODOT32-NEXT: vsext.vf4 v24, v9
+; NODOT32-NEXT: vwmul.vv v0, v28, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v18
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv2r.v v2, v10
+; NODOT32-NEXT: vsext.vf4 v28, v2
+; NODOT32-NEXT: vwmul.vv v0, v24, v28
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v19
+; NODOT32-NEXT: vmv4r.v v4, v12
+; NODOT32-NEXT: vmv1r.v v3, v11
+; NODOT32-NEXT: vsext.vf4 v16, v11
+; NODOT32-NEXT: vwmul.vv v0, v24, v16
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v20
+; NODOT32-NEXT: vmv4r.v v28, v12
+; NODOT32-NEXT: vsext.vf4 v24, v28
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v21
+; NODOT32-NEXT: vsext.vf4 v24, v13
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v16, v22
+; NODOT32-NEXT: vmv2r.v v30, v14
+; NODOT32-NEXT: vsext.vf4 v24, v30
+; NODOT32-NEXT: vwmul.vv v0, v16, v24
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v24, a0, v0
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v23
+; NODOT32-NEXT: vsext.vf4 v28, v15
+; NODOT32-NEXT: vwmul.vv v16, v24, v28
+; NODOT32-NEXT: csrr a1, vlenb
+; NODOT32-NEXT: slli a1, a1, 3
+; NODOT32-NEXT: add a1, sp, a1
+; NODOT32-NEXT: addi a1, a1, 16
+; NODOT32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmacc.vx v8, a0, v16
+; NODOT32-NEXT: csrr a0, vlenb
+; NODOT32-NEXT: slli a0, a0, 4
+; NODOT32-NEXT: add sp, sp, a0
+; NODOT32-NEXT: .cfi_def_cfa sp, 16
+; NODOT32-NEXT: addi sp, sp, 16
+; NODOT32-NEXT: .cfi_def_cfa_offset 0
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64_m8:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vl8r.v v24, (a0)
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v4, v16
+; NODOT64-NEXT: vsext.vf4 v0, v24
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v17
+; NODOT64-NEXT: vsext.vf4 v0, v25
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v18
+; NODOT64-NEXT: vsext.vf4 v0, v26
+; NODOT64-NEXT: vwmacc.vv v8, v4, v0
+; NODOT64-NEXT: vsext.vf4 v4, v19
+; NODOT64-NEXT: vsext.vf4 v16, v27
+; NODOT64-NEXT: vwmacc.vv v8, v4, v16
+; NODOT64-NEXT: vsext.vf4 v16, v20
+; NODOT64-NEXT: vsext.vf4 v24, v28
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v21
+; NODOT64-NEXT: vsext.vf4 v24, v29
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v22
+; NODOT64-NEXT: vsext.vf4 v24, v30
+; NODOT64-NEXT: vwmacc.vv v8, v16, v24
+; NODOT64-NEXT: vsext.vf4 v16, v23
+; NODOT64-NEXT: vsext.vf4 v20, v31
+; NODOT64-NEXT: vwmacc.vv v8, v16, v20
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64_m8:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vl8r.v v24, (a0)
+; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma
+; DOT-NEXT: vmv.v.i v0, 0
+; DOT-NEXT: vdot4a.vv v0, v16, v24
+; DOT-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; DOT-NEXT: vadd.vv v16, v0, v4
+; DOT-NEXT: vwadd.wv v8, v8, v16
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 64 x i8> %a to <vscale x 64 x i64>
+ %b.sext = sext <vscale x 64 x i8> %b to <vscale x 64 x i64>
+ %mul = mul <vscale x 64 x i64> %a.sext, %b.sext
+ %res = call <vscale x 8 x i64> @llvm.experimental.vector.partial.reduce.add.nxv8i64.nxv64i64(<vscale x 8 x i64> %acc, <vscale x 64 x i64> %mul)
+ ret <vscale x 8 x i64> %res
+}
+
+define <2 x i64> @vdot4a_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
----------------
topperc wrote:
Fixed vectors should be in a different file that starts with fixed-vectors- to keep our established conventions
https://github.com/llvm/llvm-project/pull/215517
More information about the llvm-branch-commits
mailing list