[llvm-branch-commits] [RISCV] Support i64 accumulator for Zvdot4a8i partial reductions (PR #215517)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Aug 11 03:55:29 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Pengcheng Wang (wangpc-pp)
<details>
<summary>Changes</summary>
In `Zvdot4a8i` extension, `vdot4a*` only produces an i32 result.
This PR extends partial-reduction handling so an i64 accumulator
with i8 inputs (a scale-8 partial reduction) is also lowered using
the dot-product instructions: perform the dot product into a fresh
i32 accumulator (each lane is the sum of four i8 products and cannot
overflow i32), then sign/zero-extend the i32 partial sums to i64
and accumulate. This mirrors the AArch64 sdot+sadalp idiom.
- `getPartialReductionCost` accepts an i64 accumulator (reduction
factor 8), costing the `vdot4a*` plus the extra i32->i64 widen
and accumulate.
- `setPartialReduceMLAAction` marks the i8->i64 (scalable and
fixed-length) partial reductions Custom.
- `lowerPARTIAL_REDUCE_MLA` lowers the i64 case via an i32 `vdot4a*`
followed by a widening `partial.reduce.add`, which the generic
expander turns into extract_subvector + add.
The i64 case requires a wide enough VF (LMUL) to reach the scale-8 factor,
matching how AArch64 only forms it under SVE.
Assisted-by: TRAE CLI (Opus 4.8)
---
Patch is 24.50 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/215517.diff
4 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+46-4)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+26-5)
- (added) llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll (+250)
- (added) llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll (+106)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ea4803e59ebc1..48b4baca61994 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1970,13 +1970,22 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
+ // An i64 accumulator is handled by performing an i32 vdot4a* and widening
+ // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
+ setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom);
+ setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
+ setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom);
+ setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom);
+
if (Subtarget.useRVVForFixedLengthVectors()) {
for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
- if (VT.getVectorElementType() != MVT::i32 ||
+ if ((VT.getVectorElementType() != MVT::i32 &&
+ VT.getVectorElementType() != MVT::i64) ||
!useRVVForFixedLengthVectorVT(VT))
continue;
ElementCount EC = VT.getVectorElementCount();
- MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(4));
+ unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
+ MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(Scale));
setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
}
}
@@ -9884,8 +9893,6 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
SDValue Accum = Op.getOperand(0);
- assert(Accum.getSimpleValueType() == VT &&
- VT.getVectorElementType() == MVT::i32);
SDValue A = Op.getOperand(1);
SDValue B = Op.getOperand(2);
MVT ArgVT = A.getSimpleValueType();
@@ -9893,6 +9900,41 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
ArgVT.getVectorElementType() == MVT::i8);
(void)ArgVT;
+ // vdot4a* only produces an i32 result. For an i64 accumulator, perform the
+ // dot product into a fresh i32 accumulator (each result is the sum of four
+ // i8 products, which cannot overflow i32), then widen the i32 partial sums
+ // to i64 and accumulate. This mirrors the AArch64 sdot+sadalp idiom.
+ if (VT.getVectorElementType() == MVT::i64) {
+ assert(Accum.getSimpleValueType() == VT);
+ // vdot4a* reduces each group of four i8 lanes into one i32 lane, so the
+ // intermediate i32 result has 1/4 the element count of the i8 inputs.
+ MVT I32VT = MVT::getVectorVT(
+ MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4));
+ SDValue Dot = DAG.getNode(Op.getOpcode(), DL, I32VT,
+ {DAG.getConstant(0, DL, I32VT), A, B});
+ // Widen the i32 partial sums to i64. They are signed for SMLA/SUMLA and
+ // unsigned for UMLA.
+ unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
+ ? ISD::ZERO_EXTEND
+ : ISD::SIGN_EXTEND;
+ MVT WideVT = I32VT.changeVectorElementType(MVT::i64);
+ SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot);
+ // The widened dot result has twice the elements of the i64 accumulator.
+ // Reduce it in by splitting into subvectors matching the accumulator and
+ // adding them together (the same lowering the generic expander would use
+ // for a multiplier-free partial reduction, but without a redundant mul).
+ unsigned Stride = VT.getVectorMinNumElements();
+ SDValue Res = Accum;
+ for (unsigned I = 0, E = WideVT.getVectorMinNumElements() / Stride; I != E;
+ ++I)
+ Res = DAG.getNode(ISD::ADD, DL, VT, Res,
+ DAG.getExtractSubvector(DL, VT, Wide, I * Stride));
+ return Res;
+ }
+
+ assert(Accum.getSimpleValueType() == VT &&
+ VT.getVectorElementType() == MVT::i32);
+
// The zvdot4a8i pseudos are defined with sources and destination both
// being i32. This cast is needed for correctness to avoid incorrect
// .vx matching of i8 splats.
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9868e4d02f905..62f95797bbf49 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -353,17 +353,38 @@ InstructionCost RISCVTTIImpl::getPartialReductionCost(
// zve32x is broken for partial_reduce_umla, but let's make sure we
// don't generate them.
+ // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
+ // additionally supported by widening the i32 partial sums to i64 (see
+ // lowerPARTIAL_REDUCE_MLA). \p VF is the number of i8 input elements, so the
+ // reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(8) ||
- !AccumType->isIntegerTy(32) || !VF.isKnownMultipleOf(4))
+ (!AccumType->isIntegerTy(32) && !AccumType->isIntegerTy(64)))
return InstructionCost::getInvalid();
- Type *Tp = VectorType::get(AccumType, VF.divideCoefficientBy(4));
- std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
+ unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
+ if (!VF.isKnownMultipleOf(Ratio))
+ return InstructionCost::getInvalid();
+
+ // Cost of the vdot4a* itself, which operates on the i32 intermediate type
+ // holding VF/4 elements.
+ Type *DotTp = VectorType::get(Type::getInt32Ty(AccumType->getContext()),
+ VF.divideCoefficientBy(4));
+ std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(DotTp);
// Note: Asuming all vdot4a* variants are equal cost
- return LT.first *
- getRISCVInstructionCost(RISCV::VDOT4A_VV, LT.second, CostKind);
+ InstructionCost Cost =
+ DotLT.first *
+ getRISCVInstructionCost(RISCV::VDOT4A_VV, DotLT.second, CostKind);
+
+ // Account for the widening extend + accumulate needed for an i64 result.
+ if (AccumType->isIntegerTy(64)) {
+ Type *AccTp = VectorType::get(AccumType, VF.divideCoefficientBy(Ratio));
+ std::pair<InstructionCost, MVT> AccLT = getTypeLegalizationCost(AccTp);
+ Cost += AccLT.first * 2;
+ }
+
+ return Cost;
}
bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
new file mode 100644
index 0000000000000..740da5c6584da
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT64
+
+; Tests for partial.reduce.add into an i64 accumulator with i8 inputs. When
+; Zvdot4a8i is available these are lowered to a vdot4a* (i8 -> i32) followed by
+; a widening (i32 -> i64) accumulate, mirroring the AArch64 sdot+sadalp idiom.
+
+define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v12, v9
+; NODOT32-NEXT: vsext.vf4 v24, v10
+; NODOT32-NEXT: vwmul.vv v16, v12, v24
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vsext.vf4 v24, v10
+; NODOT64-NEXT: vwmul.vv v16, v12, v24
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4a.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v8, v8, v10
+; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.sext, %b.sext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4au_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e8, m1, ta, ma
+; NODOT32-NEXT: vwmulu.vv v16, v9, v10
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vzext.vf2 v12, v16
+; NODOT32-NEXT: vwmulu.vx v16, v12, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4au_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e8, m1, ta, ma
+; NODOT64-NEXT: vwmulu.vv v12, v9, v10
+; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT64-NEXT: vzext.vf4 v16, v12
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4au_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4au.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vzext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v8, v8, v10
+; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: ret
+entry:
+ %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.zext, %b.zext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; NODOT32-LABEL: vdot4asu_i64:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v12, v9
+; NODOT32-NEXT: vzext.vf4 v24, v10
+; NODOT32-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT32-NEXT: vadd.vv v9, v19, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v10, v17, v18
+; NODOT32-NEXT: vadd.vv v8, v23, v8
+; NODOT32-NEXT: vadd.vv v11, v21, v22
+; NODOT32-NEXT: vadd.vv v9, v10, v9
+; NODOT32-NEXT: vadd.vv v8, v11, v8
+; NODOT32-NEXT: vadd.vv v8, v9, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4asu_i64:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v12, v9
+; NODOT64-NEXT: vzext.vf4 v24, v10
+; NODOT64-NEXT: vwmulsu.vv v16, v12, v24
+; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; NODOT64-NEXT: vadd.vv v9, v19, v20
+; NODOT64-NEXT: vadd.vv v8, v8, v16
+; NODOT64-NEXT: vadd.vv v10, v17, v18
+; NODOT64-NEXT: vadd.vv v8, v23, v8
+; NODOT64-NEXT: vadd.vv v11, v21, v22
+; NODOT64-NEXT: vadd.vv v9, v10, v9
+; NODOT64-NEXT: vadd.vv v8, v11, v8
+; NODOT64-NEXT: vadd.vv v8, v9, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4asu_i64:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vdot4asu.vv v12, v9, v10
+; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; DOT-NEXT: vsext.vf2 v10, v12
+; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; DOT-NEXT: vadd.vv v8, v8, v10
+; DOT-NEXT: vadd.vv v8, v8, v11
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64>
+ %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>
+ %mul = mul <vscale x 8 x i64> %a.sext, %b.zext
+ %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul)
+ ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 2 x i64> @vdot4a_i64_m2(<vscale x 2 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; NODOT32-LABEL: vdot4a_i64_m2:
+; NODOT32: # %bb.0: # %entry
+; NODOT32-NEXT: li a0, 1
+; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vsext.vf4 v24, v10
+; NODOT32-NEXT: vsext.vf4 v4, v11
+; NODOT32-NEXT: vsext.vf4 v28, v12
+; NODOT32-NEXT: vsext.vf4 v0, v13
+; NODOT32-NEXT: vwmul.vv v16, v24, v28
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v16, v16, a0
+; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT32-NEXT: vwmul.vv v24, v4, v0
+; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
+; NODOT32-NEXT: vmul.vx v24, v24, a0
+; NODOT32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; NODOT32-NEXT: vadd.vv v10, v18, v20
+; NODOT32-NEXT: vadd.vv v8, v8, v16
+; NODOT32-NEXT: vadd.vv v8, v22, v8
+; NODOT32-NEXT: vadd.vv v10, v10, v24
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: vadd.vv v8, v30, v8
+; NODOT32-NEXT: vadd.vv v10, v26, v28
+; NODOT32-NEXT: vadd.vv v8, v10, v8
+; NODOT32-NEXT: ret
+;
+; NODOT64-LABEL: vdot4a_i64_m2:
+; NODOT64: # %bb.0: # %entry
+; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma
+; NODOT64-NEXT: vsext.vf4 v4, v10
+; NODOT64-NEXT: vsext.vf4 v24, v11
+; NODOT64-NEXT: vsext.vf4 v0, v12
+; NODOT64-NEXT: vsext.vf4 v28, v13
+; NODOT64-NEXT: vwmul.vv v16, v24, v28
+; NODOT64-NEXT: vwmul.vv v24, v4, v0
+; NODOT64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; NODOT64-NEXT: vadd.vv v10, v26, v28
+; NODOT64-NEXT: vadd.vv v8, v8, v24
+; NODOT64-NEXT: vadd.vv v8, v30, v8
+; NODOT64-NEXT: vadd.vv v10, v10, v16
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: vadd.vv v8, v22, v8
+; NODOT64-NEXT: vadd.vv v10, v18, v20
+; NODOT64-NEXT: vadd.vv v8, v10, v8
+; NODOT64-NEXT: ret
+;
+; DOT-LABEL: vdot4a_i64_m2:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v16, 0
+; DOT-NEXT: vdot4a.vv v16, v10, v12
+; DOT-NEXT: vsetvli zero, zero, e64, m4, ta, ma
+; DOT-NEXT: vsext.vf2 v12, v16
+; DOT-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; DOT-NEXT: vadd.vv v8, v8, v12
+; DOT-NEXT: vadd.vv v8, v8, v14
+; DOT-NEXT: ret
+entry:
+ %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
+ %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i64>
+ %mul = mul <vscale x 16 x i64> %a.sext, %b.sext
+ %res = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> %acc, <vscale x 16 x i64> %mul)
+ ret <vscale x 2 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; DOT32: {{.*}}
+; DOT64: {{.*}}
+; NODOT: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll
new file mode 100644
index 0000000000000..459c896357809
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefix=NOTAILFOLD
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s | FileCheck %s --check-prefix=TAILFOLD
+
+; Dot product with an i64 accumulator and i8 inputs. This forms a scale-8
+; partial reduction (i8 -> i64), which the RISC-V backend lowers using vdot4a*
+; plus a widening accumulate. The i64 accumulator is 8x wider than the i8
+; input, so the scale-8 reduction needs a VF of at least (vscale x) 8; the
+; -riscv-v-register-bit-width-lmul=8 override raises the VF range high enough
+; to reach it, both with and without EVL tail folding.
+
+target triple = "riscv64-none-unknown-elf"
+
+define i64 @vqdot_i64(ptr %a, ptr %b) #0 {
+; NOTAILFOLD-LABEL: define i64 @vqdot_i64(
+; NOTAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOTAILFOLD-NEXT: entry:
+; NOTAILFOLD-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NOTAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP4]], 3
+; NOTAILFOLD-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 16)
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP5]]
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NOTAILFOLD: vector.ph:
+; NOTAILFOLD-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 3
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]
+; NOTAILFOLD: vector.body:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; NOTAILFOLD-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP0]], align 1
+; NOTAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; NOTAILFOLD-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP2]], align 1
+; NOTAILFOLD-NEXT: [[TMP6:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; NOTAILFOLD-NEXT: [[TMP7:%.*]] = sext <vscal...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/215517
More information about the llvm-branch-commits
mailing list