[llvm] [RISCV] Optimize scalable multiply reductions lowerings (PR #193528)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 08:47:57 PDT 2026
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/193528
>From 96a12b20d1352006c3affe043f7b915959f2bb71 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Wed, 22 Apr 2026 08:10:51 -0700
Subject: [PATCH 1/6] [RISCV] Optimize scalable multiply reductions lowerings
Now that we can correctly lower a scalable multiply reduction,
lets improve the lowering for two common cases.
1) If we have exact VLEN, just convert to the fixed vector
form and lower so that we get shuffles instead of the
slow painful loop.
2) Handle the high LMUL case by splitting down to m1 via
a reduce tree. It's only at the final stage that we need
to use the loop to handle the unknown number of elements.
For context, this is mostly for completeness. I don't plan on
going any further to improve the code quality here. There's
more we can do (e.g. exploiting minimum element count to use
shuffles for the first couple stages), but that can be future
work.
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 91 ++++++--
.../test/CodeGen/RISCV/rvv/vreductions-int.ll | 221 +++++++++++++++---
2 files changed, 260 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 4ce5432d67fcf..9bfbd206b22a5 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -264,9 +264,32 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
return true;
}
-// Partially expand a vector_reduce_mul wider than M1 to reduce the
-// number of vsetvlis required when VLEN is exactly known, and
-// reducing register pressure in all cases.
+// Extract pieces of size PieceEC from Vec, then build a binary tree of
+// element-wise multiplies reducing to a single piece.
+static Value *buildMulTree(IRBuilder<> &Builder, ElementCount PieceEC,
+ Value *Vec) {
+ auto *VecTy = cast<VectorType>(Vec->getType());
+ auto *PieceTy = VectorType::get(VecTy->getElementType(), PieceEC);
+ unsigned PieceElts = PieceEC.getKnownMinValue();
+ unsigned NumPieces = VecTy->getElementCount().getKnownMinValue() / PieceElts;
+ assert(isPowerOf2_32(NumPieces));
+
+ SmallVector<Value *, 8> Pieces(NumPieces);
+ for (unsigned i = 0; i < NumPieces; i++)
+ Pieces[i] =
+ Builder.CreateExtractVector(PieceTy, Vec, (uint64_t)(i * PieceElts));
+
+ while (Pieces.size() > 1) {
+ for (unsigned i = 0; i < Pieces.size() / 2; i++)
+ Pieces[i] =
+ Builder.CreateMul(Pieces[i * 2], Pieces[i * 2 + 1], "bin.rdx");
+ Pieces.truncate(Pieces.size() / 2);
+ }
+ return Pieces[0];
+}
+
+// Partially expand a vector_reduce_mul wider than M1 to reduce
+// register pressure and the number of vsetvlis required.
bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
if (II.getIntrinsicID() != Intrinsic::vector_reduce_mul)
return false;
@@ -275,12 +298,44 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
return false;
Value *TmpVec = II.getArgOperand(0);
- auto *VecTy = dyn_cast<FixedVectorType>(TmpVec->getType());
- if (!VecTy)
- return false;
-
+ auto *VecTy = cast<VectorType>(TmpVec->getType());
unsigned EltSize = VecTy->getScalarSizeInBits();
- unsigned VF = VecTy->getNumElements();
+
+ if (auto *ScalTy = dyn_cast<ScalableVectorType>(VecTy)) {
+ unsigned MinElts = ScalTy->getMinNumElements();
+
+ if (auto VLen = ST->getRealVLen()) {
+ // If VLEN is exactly known, convert to a fixed vector reduction and
+ // recurse to let the fixed path handle it (shuffle reduction instead
+ // of a scalar loop).
+ unsigned VScale = *VLen / RISCV::RVVBitsPerBlock;
+ auto *FixedTy =
+ FixedVectorType::get(VecTy->getElementType(), MinElts * VScale);
+ IRBuilder<> Builder(&II);
+ Value *Fixed = Builder.CreateExtractVector(FixedTy, TmpVec, (uint64_t)0);
+ auto *FixedRdx = cast<IntrinsicInst>(Builder.CreateIntrinsic(
+ Intrinsic::vector_reduce_mul, {FixedTy}, {Fixed}));
+ II.replaceAllUsesWith(FixedRdx);
+ II.eraseFromParent();
+ expandMulReduction(*FixedRdx);
+ return true;
+ }
+
+ unsigned M1MinElts = RISCV::RVVBitsPerBlock / EltSize;
+ if (MinElts <= M1MinElts)
+ return false;
+
+ IRBuilder<> Builder(&II);
+ auto M1EC = ElementCount::getScalable(M1MinElts);
+ Value *Reduced = buildMulTree(Builder, M1EC, TmpVec);
+ Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul,
+ {Reduced->getType()}, {Reduced});
+ II.replaceAllUsesWith(Rdx);
+ II.eraseFromParent();
+ return true;
+ }
+
+ unsigned VF = cast<FixedVectorType>(VecTy)->getNumElements();
unsigned MinVLen = ST->getRealMinVLen();
unsigned M1VF = MinVLen / EltSize;
@@ -288,31 +343,19 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
return false;
IRBuilder<> Builder(&II);
- auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
+ auto M1EC = ElementCount::getFixed(M1VF);
+ auto *M1Ty = VectorType::get(VecTy->getElementType(), M1EC);
// When VLEN is exactly known, extract m1 pieces and build a mul tree.
// This greatly reduces register pressure during the reduction, and
// avoids all but one vsetvli (the one from original LMUL to m1).
// TODO: Generalize to handle the splitting case.
if (MinVLen == ST->getRealMaxVLen() && VF <= 8 * M1VF) {
- unsigned NumM1 = VF / M1VF;
- assert(isPowerOf2_32(NumM1) && NumM1 <= 8);
- SmallVector<Value *, 8> Pieces(NumM1);
- for (unsigned i = 0; i < NumM1; i++)
- Pieces[i] =
- Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)(i * M1VF));
-
- while (Pieces.size() > 1) {
- for (unsigned i = 0; i < Pieces.size() / 2; i++)
- Pieces[i] =
- Builder.CreateMul(Pieces[i * 2], Pieces[i * 2 + 1], "bin.rdx");
- Pieces.truncate(Pieces.size() / 2);
- }
- TmpVec = Pieces[0];
+ TmpVec = buildMulTree(Builder, M1EC, TmpVec);
} else {
// For non-exact VLEN, shuffle-reduce at the original vector width down to
// m1, then extract. This prioritizes reducing the number of vsetvli
- // over maximual reduction of LMUL for the intermediate states.
+ // over maximal reduction of LMUL for the intermediate states.
SmallVector<int, 32> ShuffleMask(VF);
for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
unsigned Half = LiveElts / 2;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index 267d9b557a785..2b49be2dbe568 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2034,14 +2034,16 @@ define signext i32 @vreduce_mul_nxv4i32(<vscale x 4 x i32> %v) {
; RV32-LABEL: vreduce_mul_nxv4i32:
; RV32: # %bb.0:
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: csrr a2, vlenb
-; RV32-NEXT: srli a2, a2, 1
+; RV32-NEXT: srli a2, a2, 2
; RV32-NEXT: li a0, 1
-; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; RV32-NEXT: .LBB122_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v10, v8, a1
-; RV32-NEXT: vmv.x.s a3, v10
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a1
+; RV32-NEXT: vmv.x.s a3, v9
; RV32-NEXT: addi a1, a1, 1
; RV32-NEXT: mul a0, a0, a3
; RV32-NEXT: bne a1, a2, .LBB122_1
@@ -2051,14 +2053,16 @@ define signext i32 @vreduce_mul_nxv4i32(<vscale x 4 x i32> %v) {
; RV64-LABEL: vreduce_mul_nxv4i32:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 1
+; RV64-NEXT: srli a2, a2, 2
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; RV64-NEXT: .LBB122_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v10, v8, a1
-; RV64-NEXT: vmv.x.s a3, v10
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mulw a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB122_1
@@ -2120,18 +2124,20 @@ define i64 @vreduce_mul_nxv2i64(<vscale x 2 x i64> %v) {
; RV32: # %bb.0:
; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: csrr a3, vlenb
; RV32-NEXT: li a0, 1
-; RV32-NEXT: srli a3, a3, 2
+; RV32-NEXT: srli a3, a3, 3
; RV32-NEXT: li a4, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
; RV32-NEXT: .LBB124_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v10, v8, a2
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v12, v10, a4
-; RV32-NEXT: vmv.x.s a5, v10
-; RV32-NEXT: vmv.x.s a6, v12
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
; RV32-NEXT: mulhu a7, a0, a5
; RV32-NEXT: mul a1, a1, a5
; RV32-NEXT: mul a6, a0, a6
@@ -2145,14 +2151,16 @@ define i64 @vreduce_mul_nxv2i64(<vscale x 2 x i64> %v) {
; RV64-LABEL: vreduce_mul_nxv2i64:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 2
+; RV64-NEXT: srli a2, a2, 3
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma
; RV64-NEXT: .LBB124_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v10, v8, a1
-; RV64-NEXT: vmv.x.s a3, v10
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mul a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB124_1
@@ -2167,18 +2175,22 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
; RV32: # %bb.0:
; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: csrr a3, vlenb
; RV32-NEXT: li a0, 1
-; RV32-NEXT: srli a3, a3, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: srli a3, a3, 3
; RV32-NEXT: li a4, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
; RV32-NEXT: .LBB125_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v12, v8, a2
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v16, v12, a4
-; RV32-NEXT: vmv.x.s a5, v12
-; RV32-NEXT: vmv.x.s a6, v16
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
; RV32-NEXT: mulhu a7, a0, a5
; RV32-NEXT: mul a1, a1, a5
; RV32-NEXT: mul a6, a0, a6
@@ -2192,14 +2204,18 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
; RV64-LABEL: vreduce_mul_nxv4i64:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: srli a2, a2, 3
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m4, ta, ma
; RV64-NEXT: .LBB125_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v12, v8, a1
-; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mul a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB125_1
@@ -2208,3 +2224,152 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
%red = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v)
ret i64 %red
}
+
+define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
+; RV32-LABEL: vreduce_mul_nxv16i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a2, 0
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v10, v12, v13
+; RV32-NEXT: vmul.vv v11, v14, v15
+; RV32-NEXT: vmul.vv v12, v16, v17
+; RV32-NEXT: vmul.vv v13, v18, v19
+; RV32-NEXT: vmul.vv v14, v20, v21
+; RV32-NEXT: vmul.vv v15, v22, v23
+; RV32-NEXT: csrr a3, vlenb
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v10, v12, v13
+; RV32-NEXT: vmul.vv v11, v14, v15
+; RV32-NEXT: srli a3, a3, 3
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: li a4, 32
+; RV32-NEXT: .LBB126_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
+; RV32-NEXT: addi a2, a2, 1
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
+; RV32-NEXT: mulhu a7, a0, a5
+; RV32-NEXT: mul a1, a1, a5
+; RV32-NEXT: mul a6, a0, a6
+; RV32-NEXT: add a1, a7, a1
+; RV32-NEXT: add a1, a1, a6
+; RV32-NEXT: mul a0, a0, a5
+; RV32-NEXT: bne a2, a3, .LBB126_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv16i64:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v10, v12, v13
+; RV64-NEXT: vmul.vv v11, v14, v15
+; RV64-NEXT: vmul.vv v12, v16, v17
+; RV64-NEXT: vmul.vv v13, v18, v19
+; RV64-NEXT: vmul.vv v14, v20, v21
+; RV64-NEXT: vmul.vv v15, v22, v23
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v10, v12, v13
+; RV64-NEXT: vmul.vv v11, v14, v15
+; RV64-NEXT: srli a2, a2, 3
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: .LBB126_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: addi a1, a1, 1
+; RV64-NEXT: mul a0, a0, a3
+; RV64-NEXT: bne a1, a2, .LBB126_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: ret
+ %red = call i64 @llvm.vector.reduce.mul.nxv16i64(<vscale x 16 x i64> %v)
+ ret i64 %red
+}
+
+; With known VLEN, scalable mul reductions can use fixed-vector shuffle
+; reduction instead of a scalar loop.
+
+define signext i32 @vreduce_mul_nxv2i32_exact_vlen(<vscale x 2 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv2i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv2i32(<vscale x 2 x i32> %v)
+ ret i32 %red
+}
+
+define signext i32 @vreduce_mul_nxv4i32_exact_vlen(<vscale x 4 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv4i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v)
+ ret i32 %red
+}
+
+define i64 @vreduce_mul_nxv2i64_exact_vlen(<vscale x 2 x i64> %v) vscale_range(2,2) {
+; RV32-LABEL: vreduce_mul_nxv2i64_exact_vlen:
+; RV32: # %bb.0:
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv2i64_exact_vlen:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %red = call i64 @llvm.vector.reduce.mul.nxv2i64(<vscale x 2 x i64> %v)
+ ret i64 %red
+}
+
+define signext i32 @vreduce_mul_nxv1i32_exact_vlen(<vscale x 1 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv1i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> %v)
+ ret i32 %red
+}
>From 834ae32521aa6922f75f202ea7a0ed2963a2f27a Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 23 Apr 2026 09:09:49 -0700
Subject: [PATCH 2/6] Address review comment
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 9bfbd206b22a5..f28c866cd1953 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -276,8 +276,7 @@ static Value *buildMulTree(IRBuilder<> &Builder, ElementCount PieceEC,
SmallVector<Value *, 8> Pieces(NumPieces);
for (unsigned i = 0; i < NumPieces; i++)
- Pieces[i] =
- Builder.CreateExtractVector(PieceTy, Vec, (uint64_t)(i * PieceElts));
+ Pieces[i] = Builder.CreateExtractVector(PieceTy, Vec, i * PieceElts);
while (Pieces.size() > 1) {
for (unsigned i = 0; i < Pieces.size() / 2; i++)
>From f06a487578fcb8573a5dad15aed0db0023044946 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 23 Apr 2026 09:52:25 -0700
Subject: [PATCH 3/6] Fix illgal type handling and add tests
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 2 +-
.../test/CodeGen/RISCV/rvv/vreductions-int.ll | 328 +++++++++++++++++-
2 files changed, 325 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index f28c866cd1953..3ac4a200323c3 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -321,7 +321,7 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
}
unsigned M1MinElts = RISCV::RVVBitsPerBlock / EltSize;
- if (MinElts <= M1MinElts)
+ if (MinElts <= M1MinElts || !isPowerOf2_32(MinElts / M1MinElts))
return false;
IRBuilder<> Builder(&II);
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index 2b49be2dbe568..c91d018ac51d1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2225,6 +2225,115 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
ret i64 %red
}
+; Non-power-of-two number of M1 pieces; should not be split by buildMulTree.
+define i64 @vreduce_mul_nxv3i64(<vscale x 3 x i64> %v) {
+; RV32-LABEL: vreduce_mul_nxv3i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a2, 0
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: srli a3, a0, 3
+; RV32-NEXT: srli a0, a0, 2
+; RV32-NEXT: add a3, a0, a3
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: vsetivli zero, 1, e32, m4, ta, ma
+; RV32-NEXT: .LBB126_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: add a4, a2, a2
+; RV32-NEXT: addi a2, a2, 1
+; RV32-NEXT: addi a5, a4, 1
+; RV32-NEXT: vslidedown.vx v12, v8, a4
+; RV32-NEXT: vmv.x.s a4, v12
+; RV32-NEXT: vslidedown.vx v12, v8, a5
+; RV32-NEXT: vmv.x.s a5, v12
+; RV32-NEXT: mulhu a6, a0, a4
+; RV32-NEXT: mul a1, a1, a4
+; RV32-NEXT: mul a5, a0, a5
+; RV32-NEXT: add a1, a6, a1
+; RV32-NEXT: add a1, a1, a5
+; RV32-NEXT: mul a0, a0, a4
+; RV32-NEXT: bne a2, a3, .LBB126_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv3i64:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 0
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: srli a2, a0, 3
+; RV64-NEXT: srli a0, a0, 2
+; RV64-NEXT: add a2, a0, a2
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT: .LBB126_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vslidedown.vx v12, v8, a1
+; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: addi a1, a1, 1
+; RV64-NEXT: mul a0, a0, a3
+; RV64-NEXT: bne a1, a2, .LBB126_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: ret
+ %red = call i64 @llvm.vector.reduce.mul.nxv3i64(<vscale x 3 x i64> %v)
+ ret i64 %red
+}
+
+; Illegal element type; should not be split by buildMulTree.
+define i63 @vreduce_mul_nxv4i63(<vscale x 4 x i63> %v) {
+; RV32-LABEL: vreduce_mul_nxv4i63:
+; RV32: # %bb.0:
+; RV32-NEXT: li a2, 0
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: csrr a3, vlenb
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: srli a3, a3, 3
+; RV32-NEXT: li a4, 32
+; RV32-NEXT: .LBB127_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
+; RV32-NEXT: addi a2, a2, 1
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
+; RV32-NEXT: mulhu a7, a0, a5
+; RV32-NEXT: mul a1, a1, a5
+; RV32-NEXT: mul a6, a0, a6
+; RV32-NEXT: add a1, a7, a1
+; RV32-NEXT: add a1, a1, a6
+; RV32-NEXT: mul a0, a0, a5
+; RV32-NEXT: bne a2, a3, .LBB127_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv4i63:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: srli a2, a2, 3
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: .LBB127_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: addi a1, a1, 1
+; RV64-NEXT: mul a0, a0, a3
+; RV64-NEXT: bne a1, a2, .LBB127_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: ret
+ %red = call i63 @llvm.vector.reduce.mul.nxv4i63(<vscale x 4 x i63> %v)
+ ret i63 %red
+}
+
define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV32-LABEL: vreduce_mul_nxv16i64:
; RV32: # %bb.0:
@@ -2250,7 +2359,7 @@ define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: li a4, 32
-; RV32-NEXT: .LBB126_1: # %rdx.loop
+; RV32-NEXT: .LBB128_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vslidedown.vx v9, v8, a2
@@ -2264,7 +2373,7 @@ define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV32-NEXT: add a1, a7, a1
; RV32-NEXT: add a1, a1, a6
; RV32-NEXT: mul a0, a0, a5
-; RV32-NEXT: bne a2, a3, .LBB126_1
+; RV32-NEXT: bne a2, a3, .LBB128_1
; RV32-NEXT: # %bb.2: # %rdx.exit
; RV32-NEXT: ret
;
@@ -2290,14 +2399,14 @@ define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: li a0, 1
-; RV64-NEXT: .LBB126_1: # %rdx.loop
+; RV64-NEXT: .LBB128_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV64-NEXT: vslidedown.vx v9, v8, a1
; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mul a0, a0, a3
-; RV64-NEXT: bne a1, a2, .LBB126_1
+; RV64-NEXT: bne a1, a2, .LBB128_1
; RV64-NEXT: # %bb.2: # %rdx.exit
; RV64-NEXT: ret
%red = call i64 @llvm.vector.reduce.mul.nxv16i64(<vscale x 16 x i64> %v)
@@ -2362,6 +2471,217 @@ define i64 @vreduce_mul_nxv2i64_exact_vlen(<vscale x 2 x i64> %v) vscale_range(2
ret i64 %red
}
+define i64 @vreduce_mul_nxv3i64_exact_vlen(<vscale x 3 x i64> %v) vscale_range(2,2) {
+; RV32-LABEL: vreduce_mul_nxv3i64_exact_vlen:
+; RV32: # %bb.0:
+; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vslidedown.vi v12, v8, 5
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vslidedown.vi v16, v8, 4
+; RV32-NEXT: vslidedown.vi v20, v8, 3
+; RV32-NEXT: vslidedown.vi v24, v8, 1
+; RV32-NEXT: vslidedown.vi v28, v8, 2
+; RV32-NEXT: vmv.x.s a2, v8
+; RV32-NEXT: vsrl.vx v4, v12, a1
+; RV32-NEXT: vmv.x.s a0, v12
+; RV32-NEXT: vsrl.vx v12, v16, a1
+; RV32-NEXT: vmv.x.s a3, v16
+; RV32-NEXT: vsrl.vx v16, v20, a1
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a4, v20
+; RV32-NEXT: vsrl.vx v20, v24, a1
+; RV32-NEXT: vmv.x.s a5, v28
+; RV32-NEXT: vsrl.vx v28, v28, a1
+; RV32-NEXT: vmv.x.s a1, v24
+; RV32-NEXT: vmv.x.s a6, v4
+; RV32-NEXT: vmv.x.s a7, v12
+; RV32-NEXT: vmv.x.s t0, v16
+; RV32-NEXT: vmv.x.s t1, v8
+; RV32-NEXT: vmv.x.s t2, v20
+; RV32-NEXT: vmv.x.s t3, v28
+; RV32-NEXT: mul t4, a2, a1
+; RV32-NEXT: mulhu t5, a2, a1
+; RV32-NEXT: mul t3, t4, t3
+; RV32-NEXT: mul a2, a2, t2
+; RV32-NEXT: mulhu t2, t4, a5
+; RV32-NEXT: mul a1, t1, a1
+; RV32-NEXT: mul t1, t4, a5
+; RV32-NEXT: add t2, t2, t3
+; RV32-NEXT: add a2, t5, a2
+; RV32-NEXT: mul t0, t1, t0
+; RV32-NEXT: mulhu t3, t1, a4
+; RV32-NEXT: mul t1, t1, a4
+; RV32-NEXT: add a1, a2, a1
+; RV32-NEXT: add t0, t3, t0
+; RV32-NEXT: mul a2, t1, a7
+; RV32-NEXT: mulhu a7, t1, a3
+; RV32-NEXT: mul t1, t1, a3
+; RV32-NEXT: mul a1, a1, a5
+; RV32-NEXT: add a2, a7, a2
+; RV32-NEXT: mul a5, t1, a6
+; RV32-NEXT: mulhu a6, t1, a0
+; RV32-NEXT: add a1, t2, a1
+; RV32-NEXT: add a5, a6, a5
+; RV32-NEXT: mul a1, a1, a4
+; RV32-NEXT: add a1, t0, a1
+; RV32-NEXT: mul a1, a1, a3
+; RV32-NEXT: add a1, a2, a1
+; RV32-NEXT: mul a1, a1, a0
+; RV32-NEXT: add a1, a5, a1
+; RV32-NEXT: mul a0, t1, a0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv3i64_exact_vlen:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v11, v10, 1
+; RV64-NEXT: vmv.x.s a0, v10
+; RV64-NEXT: vslidedown.vi v10, v8, 1
+; RV64-NEXT: vmv.x.s a1, v8
+; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vslidedown.vi v8, v9, 1
+; RV64-NEXT: vmv.x.s a3, v11
+; RV64-NEXT: vmv.x.s a4, v10
+; RV64-NEXT: vmv.x.s a5, v8
+; RV64-NEXT: vmv.v.x v8, a2
+; RV64-NEXT: vmv.v.x v11, a1
+; RV64-NEXT: vmv.v.x v10, a0
+; RV64-NEXT: vslide1down.vx v9, v8, a5
+; RV64-NEXT: vslide1down.vx v10, v10, a3
+; RV64-NEXT: vslide1down.vx v8, v11, a4
+; RV64-NEXT: vmv.v.i v11, 1
+; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 1
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %red = call i64 @llvm.vector.reduce.mul.nxv3i64(<vscale x 3 x i64> %v)
+ ret i64 %red
+}
+
+define i63 @vreduce_mul_nxv4i63_exact_vlen(<vscale x 4 x i63> %v) vscale_range(2,2) {
+; RV32-LABEL: vreduce_mul_nxv4i63_exact_vlen:
+; RV32: # %bb.0:
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v12, v9, 1
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vslidedown.vi v13, v9, 2
+; RV32-NEXT: vslidedown.vi v9, v9, 3
+; RV32-NEXT: vslidedown.vi v14, v8, 1
+; RV32-NEXT: vmv.x.s a5, v8
+; RV32-NEXT: vslidedown.vi v15, v8, 2
+; RV32-NEXT: vslidedown.vi v8, v8, 3
+; RV32-NEXT: vslidedown.vi v16, v11, 3
+; RV32-NEXT: vslidedown.vi v17, v11, 2
+; RV32-NEXT: vslidedown.vi v18, v11, 1
+; RV32-NEXT: vmv.x.s a6, v11
+; RV32-NEXT: vslidedown.vi v11, v10, 3
+; RV32-NEXT: vslidedown.vi v19, v10, 2
+; RV32-NEXT: vmv.x.s a7, v10
+; RV32-NEXT: vslidedown.vi v10, v10, 1
+; RV32-NEXT: vmv.x.s t0, v12
+; RV32-NEXT: vmv.x.s a2, v13
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: vmv.x.s t1, v14
+; RV32-NEXT: vmv.x.s a4, v15
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: vmv.v.x v8, a3
+; RV32-NEXT: vmv.x.s a3, v16
+; RV32-NEXT: vmv.v.x v9, a5
+; RV32-NEXT: vmv.x.s a5, v17
+; RV32-NEXT: vmv.v.x v12, a7
+; RV32-NEXT: vmv.x.s a7, v18
+; RV32-NEXT: vmv.v.x v13, a6
+; RV32-NEXT: li a6, 32
+; RV32-NEXT: vslide1down.vx v8, v8, t0
+; RV32-NEXT: vmv.x.s t0, v11
+; RV32-NEXT: vslide1down.vx v9, v9, t1
+; RV32-NEXT: vmv.x.s t1, v10
+; RV32-NEXT: vslide1down.vx v10, v12, t1
+; RV32-NEXT: vmv.x.s t1, v19
+; RV32-NEXT: vslide1down.vx v11, v13, a7
+; RV32-NEXT: vslide1down.vx v8, v8, a2
+; RV32-NEXT: vslide1down.vx v9, v9, a4
+; RV32-NEXT: vslide1down.vx v10, v10, t1
+; RV32-NEXT: vslide1down.vx v11, v11, a5
+; RV32-NEXT: vslide1down.vx v8, v8, a0
+; RV32-NEXT: vslide1down.vx v9, v9, a1
+; RV32-NEXT: vslide1down.vx v10, v10, t0
+; RV32-NEXT: vslide1down.vx v11, v11, a3
+; RV32-NEXT: vslidedown.vi v12, v9, 1
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: vslidedown.vi v13, v9, 2
+; RV32-NEXT: vslidedown.vi v9, v9, 3
+; RV32-NEXT: vslidedown.vi v14, v8, 1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v10, v10, v11
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v11, v8, 2
+; RV32-NEXT: vslidedown.vi v8, v8, 3
+; RV32-NEXT: vmv.x.s a2, v12
+; RV32-NEXT: vmv.v.x v12, a0
+; RV32-NEXT: vmv.x.s a0, v13
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vmv.x.s a4, v14
+; RV32-NEXT: vmv.v.x v9, a1
+; RV32-NEXT: vmv.x.s a1, v11
+; RV32-NEXT: vmv.x.s a5, v8
+; RV32-NEXT: vslide1down.vx v8, v12, a2
+; RV32-NEXT: vslide1down.vx v9, v9, a4
+; RV32-NEXT: vslide1down.vx v8, v8, a0
+; RV32-NEXT: vslide1down.vx v9, v9, a1
+; RV32-NEXT: vslide1down.vx v8, v8, a3
+; RV32-NEXT: vslide1down.vx v9, v9, a5
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a6
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv4i63_exact_vlen:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v12, v11, 1
+; RV64-NEXT: vmv.x.s a0, v11
+; RV64-NEXT: vslidedown.vi v11, v10, 1
+; RV64-NEXT: vmv.x.s a1, v10
+; RV64-NEXT: vslidedown.vi v10, v9, 1
+; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vmv.x.s a3, v8
+; RV64-NEXT: vslidedown.vi v8, v8, 1
+; RV64-NEXT: vmv.x.s a4, v12
+; RV64-NEXT: vmv.x.s a5, v11
+; RV64-NEXT: vmv.x.s a6, v10
+; RV64-NEXT: vmv.x.s a7, v8
+; RV64-NEXT: vmv.v.x v8, a3
+; RV64-NEXT: vmv.v.x v9, a2
+; RV64-NEXT: vmv.v.x v10, a1
+; RV64-NEXT: vmv.v.x v11, a0
+; RV64-NEXT: vslide1down.vx v8, v8, a7
+; RV64-NEXT: vslide1down.vx v9, v9, a6
+; RV64-NEXT: vslide1down.vx v10, v10, a5
+; RV64-NEXT: vslide1down.vx v11, v11, a4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %red = call i63 @llvm.vector.reduce.mul.nxv4i63(<vscale x 4 x i63> %v)
+ ret i63 %red
+}
+
define signext i32 @vreduce_mul_nxv1i32_exact_vlen(<vscale x 1 x i32> %v) vscale_range(2,2) {
; CHECK-LABEL: vreduce_mul_nxv1i32_exact_vlen:
; CHECK: # %bb.0:
>From 08e2ea66f5bf8101485b867c7f5e2319827a424b Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Fri, 24 Apr 2026 18:54:05 -0700
Subject: [PATCH 4/6] Update after merge
---
llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index c91d018ac51d1..3d8eef761aade 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2239,7 +2239,7 @@ define i64 @vreduce_mul_nxv3i64(<vscale x 3 x i64> %v) {
; RV32-NEXT: vsetivli zero, 1, e32, m4, ta, ma
; RV32-NEXT: .LBB126_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: add a4, a2, a2
+; RV32-NEXT: slli a4, a2, 1
; RV32-NEXT: addi a2, a2, 1
; RV32-NEXT: addi a5, a4, 1
; RV32-NEXT: vslidedown.vx v12, v8, a4
>From 8702450b4b87d4c568463b564ee73ac6f94f78ee Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 30 Apr 2026 08:38:36 -0700
Subject: [PATCH 5/6] rework-tests
---
.../test/CodeGen/RISCV/rvv/vreductions-int.ll | 514 ++++++++----------
1 file changed, 222 insertions(+), 292 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index 3d8eef761aade..f697dfc9637cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2226,120 +2226,102 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
}
; Non-power-of-two number of M1 pieces; should not be split by buildMulTree.
-define i64 @vreduce_mul_nxv3i64(<vscale x 3 x i64> %v) {
-; RV32-LABEL: vreduce_mul_nxv3i64:
+define signext i32 @vreduce_mul_nxv6i32(<vscale x 6 x i32> %v) {
+; RV32-LABEL: vreduce_mul_nxv6i32:
; RV32: # %bb.0:
-; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
; RV32-NEXT: csrr a0, vlenb
-; RV32-NEXT: srli a3, a0, 3
-; RV32-NEXT: srli a0, a0, 2
-; RV32-NEXT: add a3, a0, a3
+; RV32-NEXT: srli a2, a0, 2
+; RV32-NEXT: sub a2, a0, a2
; RV32-NEXT: li a0, 1
; RV32-NEXT: vsetivli zero, 1, e32, m4, ta, ma
; RV32-NEXT: .LBB126_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: slli a4, a2, 1
-; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: addi a5, a4, 1
-; RV32-NEXT: vslidedown.vx v12, v8, a4
-; RV32-NEXT: vmv.x.s a4, v12
-; RV32-NEXT: vslidedown.vx v12, v8, a5
-; RV32-NEXT: vmv.x.s a5, v12
-; RV32-NEXT: mulhu a6, a0, a4
-; RV32-NEXT: mul a1, a1, a4
-; RV32-NEXT: mul a5, a0, a5
-; RV32-NEXT: add a1, a6, a1
-; RV32-NEXT: add a1, a1, a5
-; RV32-NEXT: mul a0, a0, a4
-; RV32-NEXT: bne a2, a3, .LBB126_1
+; RV32-NEXT: vslidedown.vx v12, v8, a1
+; RV32-NEXT: vmv.x.s a3, v12
+; RV32-NEXT: addi a1, a1, 1
+; RV32-NEXT: mul a0, a0, a3
+; RV32-NEXT: bne a1, a2, .LBB126_1
; RV32-NEXT: # %bb.2: # %rdx.exit
; RV32-NEXT: ret
;
-; RV64-LABEL: vreduce_mul_nxv3i64:
+; RV64-LABEL: vreduce_mul_nxv6i32:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
; RV64-NEXT: csrr a0, vlenb
-; RV64-NEXT: srli a2, a0, 3
-; RV64-NEXT: srli a0, a0, 2
-; RV64-NEXT: add a2, a0, a2
+; RV64-NEXT: srli a2, a0, 2
+; RV64-NEXT: sub a2, a0, a2
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT: vsetivli zero, 1, e32, m4, ta, ma
; RV64-NEXT: .LBB126_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
; RV64-NEXT: vslidedown.vx v12, v8, a1
; RV64-NEXT: vmv.x.s a3, v12
; RV64-NEXT: addi a1, a1, 1
-; RV64-NEXT: mul a0, a0, a3
+; RV64-NEXT: mulw a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB126_1
; RV64-NEXT: # %bb.2: # %rdx.exit
; RV64-NEXT: ret
- %red = call i64 @llvm.vector.reduce.mul.nxv3i64(<vscale x 3 x i64> %v)
- ret i64 %red
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %v)
+ ret i32 %red
}
; Illegal element type; should not be split by buildMulTree.
-define i63 @vreduce_mul_nxv4i63(<vscale x 4 x i63> %v) {
-; RV32-LABEL: vreduce_mul_nxv4i63:
+define signext i31 @vreduce_mul_nxv8i31(<vscale x 8 x i31> %v) {
+; RV32-LABEL: vreduce_mul_nxv8i31:
; RV32: # %bb.0:
-; RV32-NEXT: li a2, 0
-; RV32-NEXT: li a1, 0
-; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: li a0, 0
+; RV32-NEXT: vsetvli a1, zero, e32, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmul.vv v9, v10, v11
-; RV32-NEXT: csrr a3, vlenb
-; RV32-NEXT: li a0, 1
+; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: vmul.vv v8, v8, v9
-; RV32-NEXT: srli a3, a3, 3
-; RV32-NEXT: li a4, 32
+; RV32-NEXT: srli a2, a2, 2
+; RV32-NEXT: li a1, 1
; RV32-NEXT: .LBB127_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vx v9, v8, a2
-; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v10, v9, a4
-; RV32-NEXT: vmv.x.s a5, v9
-; RV32-NEXT: vmv.x.s a6, v10
-; RV32-NEXT: mulhu a7, a0, a5
-; RV32-NEXT: mul a1, a1, a5
-; RV32-NEXT: mul a6, a0, a6
-; RV32-NEXT: add a1, a7, a1
-; RV32-NEXT: add a1, a1, a6
-; RV32-NEXT: mul a0, a0, a5
-; RV32-NEXT: bne a2, a3, .LBB127_1
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a0
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: addi a0, a0, 1
+; RV32-NEXT: mul a1, a1, a3
+; RV32-NEXT: bne a0, a2, .LBB127_1
; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: slli a0, a1, 1
+; RV32-NEXT: srai a0, a0, 1
; RV32-NEXT: ret
;
-; RV64-LABEL: vreduce_mul_nxv4i63:
+; RV64-LABEL: vreduce_mul_nxv8i31:
; RV64: # %bb.0:
-; RV64-NEXT: li a1, 0
-; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: li a0, 0
+; RV64-NEXT: vsetvli a1, zero, e32, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: vmul.vv v8, v8, v9
-; RV64-NEXT: srli a2, a2, 3
-; RV64-NEXT: li a0, 1
+; RV64-NEXT: srli a2, a2, 2
+; RV64-NEXT: li a1, 1
; RV64-NEXT: .LBB127_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a0
; RV64-NEXT: vmv.x.s a3, v9
-; RV64-NEXT: addi a1, a1, 1
-; RV64-NEXT: mul a0, a0, a3
-; RV64-NEXT: bne a1, a2, .LBB127_1
+; RV64-NEXT: addi a0, a0, 1
+; RV64-NEXT: mul a1, a1, a3
+; RV64-NEXT: bne a0, a2, .LBB127_1
; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: slli a0, a1, 33
+; RV64-NEXT: srai a0, a0, 33
; RV64-NEXT: ret
- %red = call i63 @llvm.vector.reduce.mul.nxv4i63(<vscale x 4 x i63> %v)
- ret i63 %red
+ %red = call i31 @llvm.vector.reduce.mul.nxv8i31(<vscale x 8 x i31> %v)
+ ret i31 %red
}
-define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
-; RV32-LABEL: vreduce_mul_nxv16i64:
+define signext i32 @vreduce_mul_nxv32i32(<vscale x 32 x i32> %v) {
+; RV32-LABEL: vreduce_mul_nxv32i32:
; RV32: # %bb.0:
-; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
-; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vsetvli a0, zero, e32, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: vmul.vv v10, v12, v13
@@ -2348,39 +2330,31 @@ define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV32-NEXT: vmul.vv v13, v18, v19
; RV32-NEXT: vmul.vv v14, v20, v21
; RV32-NEXT: vmul.vv v15, v22, v23
-; RV32-NEXT: csrr a3, vlenb
-; RV32-NEXT: li a0, 1
+; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: vmul.vv v10, v12, v13
; RV32-NEXT: vmul.vv v11, v14, v15
-; RV32-NEXT: srli a3, a3, 3
+; RV32-NEXT: srli a2, a2, 2
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: vmul.vv v8, v8, v9
-; RV32-NEXT: li a4, 32
+; RV32-NEXT: li a0, 1
; RV32-NEXT: .LBB128_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vx v9, v8, a2
-; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v10, v9, a4
-; RV32-NEXT: vmv.x.s a5, v9
-; RV32-NEXT: vmv.x.s a6, v10
-; RV32-NEXT: mulhu a7, a0, a5
-; RV32-NEXT: mul a1, a1, a5
-; RV32-NEXT: mul a6, a0, a6
-; RV32-NEXT: add a1, a7, a1
-; RV32-NEXT: add a1, a1, a6
-; RV32-NEXT: mul a0, a0, a5
-; RV32-NEXT: bne a2, a3, .LBB128_1
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a1
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: addi a1, a1, 1
+; RV32-NEXT: mul a0, a0, a3
+; RV32-NEXT: bne a1, a2, .LBB128_1
; RV32-NEXT: # %bb.2: # %rdx.exit
; RV32-NEXT: ret
;
-; RV64-LABEL: vreduce_mul_nxv16i64:
+; RV64-LABEL: vreduce_mul_nxv32i32:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
-; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vsetvli a0, zero, e32, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: vmul.vv v10, v12, v13
@@ -2394,23 +2368,23 @@ define i64 @vreduce_mul_nxv16i64(<vscale x 16 x i64> %v) {
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: vmul.vv v10, v12, v13
; RV64-NEXT: vmul.vv v11, v14, v15
-; RV64-NEXT: srli a2, a2, 3
+; RV64-NEXT: srli a2, a2, 2
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: li a0, 1
; RV64-NEXT: .LBB128_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
; RV64-NEXT: vslidedown.vx v9, v8, a1
; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
-; RV64-NEXT: mul a0, a0, a3
+; RV64-NEXT: mulw a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB128_1
; RV64-NEXT: # %bb.2: # %rdx.exit
; RV64-NEXT: ret
- %red = call i64 @llvm.vector.reduce.mul.nxv16i64(<vscale x 16 x i64> %v)
- ret i64 %red
+ %red = call i32 @llvm.vector.reduce.mul.nxv32i32(<vscale x 32 x i32> %v)
+ ret i32 %red
}
; With known VLEN, scalable mul reductions can use fixed-vector shuffle
@@ -2445,241 +2419,197 @@ define signext i32 @vreduce_mul_nxv4i32_exact_vlen(<vscale x 4 x i32> %v) vscale
ret i32 %red
}
-define i64 @vreduce_mul_nxv2i64_exact_vlen(<vscale x 2 x i64> %v) vscale_range(2,2) {
-; RV32-LABEL: vreduce_mul_nxv2i64_exact_vlen:
-; RV32: # %bb.0:
-; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v9
-; RV32-NEXT: li a1, 32
-; RV32-NEXT: vrgather.vi v9, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v9
-; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vsrl.vx v8, v8, a1
-; RV32-NEXT: vmv.x.s a1, v8
-; RV32-NEXT: ret
-;
-; RV64-LABEL: vreduce_mul_nxv2i64_exact_vlen:
-; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v9
-; RV64-NEXT: vrgather.vi v9, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v9
-; RV64-NEXT: vmv.x.s a0, v8
-; RV64-NEXT: ret
- %red = call i64 @llvm.vector.reduce.mul.nxv2i64(<vscale x 2 x i64> %v)
- ret i64 %red
-}
-
-define i64 @vreduce_mul_nxv3i64_exact_vlen(<vscale x 3 x i64> %v) vscale_range(2,2) {
-; RV32-LABEL: vreduce_mul_nxv3i64_exact_vlen:
-; RV32: # %bb.0:
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
-; RV32-NEXT: vslidedown.vi v12, v8, 5
-; RV32-NEXT: li a1, 32
-; RV32-NEXT: vslidedown.vi v16, v8, 4
-; RV32-NEXT: vslidedown.vi v20, v8, 3
-; RV32-NEXT: vslidedown.vi v24, v8, 1
-; RV32-NEXT: vslidedown.vi v28, v8, 2
-; RV32-NEXT: vmv.x.s a2, v8
-; RV32-NEXT: vsrl.vx v4, v12, a1
-; RV32-NEXT: vmv.x.s a0, v12
-; RV32-NEXT: vsrl.vx v12, v16, a1
-; RV32-NEXT: vmv.x.s a3, v16
-; RV32-NEXT: vsrl.vx v16, v20, a1
-; RV32-NEXT: vsrl.vx v8, v8, a1
-; RV32-NEXT: vmv.x.s a4, v20
-; RV32-NEXT: vsrl.vx v20, v24, a1
-; RV32-NEXT: vmv.x.s a5, v28
-; RV32-NEXT: vsrl.vx v28, v28, a1
-; RV32-NEXT: vmv.x.s a1, v24
-; RV32-NEXT: vmv.x.s a6, v4
-; RV32-NEXT: vmv.x.s a7, v12
-; RV32-NEXT: vmv.x.s t0, v16
-; RV32-NEXT: vmv.x.s t1, v8
-; RV32-NEXT: vmv.x.s t2, v20
-; RV32-NEXT: vmv.x.s t3, v28
-; RV32-NEXT: mul t4, a2, a1
-; RV32-NEXT: mulhu t5, a2, a1
-; RV32-NEXT: mul t3, t4, t3
-; RV32-NEXT: mul a2, a2, t2
-; RV32-NEXT: mulhu t2, t4, a5
-; RV32-NEXT: mul a1, t1, a1
-; RV32-NEXT: mul t1, t4, a5
-; RV32-NEXT: add t2, t2, t3
-; RV32-NEXT: add a2, t5, a2
-; RV32-NEXT: mul t0, t1, t0
-; RV32-NEXT: mulhu t3, t1, a4
-; RV32-NEXT: mul t1, t1, a4
-; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: add t0, t3, t0
-; RV32-NEXT: mul a2, t1, a7
-; RV32-NEXT: mulhu a7, t1, a3
-; RV32-NEXT: mul t1, t1, a3
-; RV32-NEXT: mul a1, a1, a5
-; RV32-NEXT: add a2, a7, a2
-; RV32-NEXT: mul a5, t1, a6
-; RV32-NEXT: mulhu a6, t1, a0
-; RV32-NEXT: add a1, t2, a1
-; RV32-NEXT: add a5, a6, a5
-; RV32-NEXT: mul a1, a1, a4
-; RV32-NEXT: add a1, t0, a1
-; RV32-NEXT: mul a1, a1, a3
-; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: mul a1, a1, a0
-; RV32-NEXT: add a1, a5, a1
-; RV32-NEXT: mul a0, t1, a0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: vreduce_mul_nxv3i64_exact_vlen:
-; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v11, v10, 1
-; RV64-NEXT: vmv.x.s a0, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 1
-; RV64-NEXT: vmv.x.s a1, v8
-; RV64-NEXT: vmv.x.s a2, v9
-; RV64-NEXT: vslidedown.vi v8, v9, 1
-; RV64-NEXT: vmv.x.s a3, v11
-; RV64-NEXT: vmv.x.s a4, v10
-; RV64-NEXT: vmv.x.s a5, v8
-; RV64-NEXT: vmv.v.x v8, a2
-; RV64-NEXT: vmv.v.x v11, a1
-; RV64-NEXT: vmv.v.x v10, a0
-; RV64-NEXT: vslide1down.vx v9, v8, a5
-; RV64-NEXT: vslide1down.vx v10, v10, a3
-; RV64-NEXT: vslide1down.vx v8, v11, a4
-; RV64-NEXT: vmv.v.i v11, 1
-; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v9
-; RV64-NEXT: vslidedown.vi v9, v8, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v9
-; RV64-NEXT: vmv.x.s a0, v8
-; RV64-NEXT: ret
- %red = call i64 @llvm.vector.reduce.mul.nxv3i64(<vscale x 3 x i64> %v)
- ret i64 %red
+define signext i32 @vreduce_mul_nxv6i32_exact_vlen(<vscale x 6 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv6i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v11, v10, 3
+; CHECK-NEXT: vslidedown.vi v12, v10, 2
+; CHECK-NEXT: vslidedown.vi v13, v10, 1
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v8, 3
+; CHECK-NEXT: vslidedown.vi v14, v8, 2
+; CHECK-NEXT: vslidedown.vi v15, v8, 1
+; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: vslidedown.vi v8, v9, 3
+; CHECK-NEXT: vslidedown.vi v16, v9, 2
+; CHECK-NEXT: vmv.x.s a2, v9
+; CHECK-NEXT: vslidedown.vi v9, v9, 1
+; CHECK-NEXT: vmv.x.s a3, v11
+; CHECK-NEXT: vmv.x.s a4, v12
+; CHECK-NEXT: vmv.x.s a5, v13
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vmv.x.s a7, v14
+; CHECK-NEXT: vmv.x.s t0, v15
+; CHECK-NEXT: vmv.x.s t1, v8
+; CHECK-NEXT: vmv.x.s t2, v16
+; CHECK-NEXT: vmv.x.s t3, v9
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: vmv.v.x v9, a1
+; CHECK-NEXT: vmv.v.x v10, a0
+; CHECK-NEXT: vslide1down.vx v8, v8, t3
+; CHECK-NEXT: vslide1down.vx v9, v9, t0
+; CHECK-NEXT: vslide1down.vx v10, v10, a5
+; CHECK-NEXT: vslide1down.vx v8, v8, t2
+; CHECK-NEXT: vslide1down.vx v11, v9, a7
+; CHECK-NEXT: vslide1down.vx v10, v10, a4
+; CHECK-NEXT: vslide1down.vx v9, v8, t1
+; CHECK-NEXT: vslide1down.vx v10, v10, a3
+; CHECK-NEXT: vslide1down.vx v8, v11, a6
+; CHECK-NEXT: vmv.v.i v11, 1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 1
+; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %v)
+ ret i32 %red
}
-define i63 @vreduce_mul_nxv4i63_exact_vlen(<vscale x 4 x i63> %v) vscale_range(2,2) {
-; RV32-LABEL: vreduce_mul_nxv4i63_exact_vlen:
+define signext i31 @vreduce_mul_nxv8i31_exact_vlen(<vscale x 8 x i31> %v) vscale_range(2,2) {
+; RV32-LABEL: vreduce_mul_nxv8i31_exact_vlen:
; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset s0, -4
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v12, v9, 1
-; RV32-NEXT: vmv.x.s a3, v9
-; RV32-NEXT: vslidedown.vi v13, v9, 2
-; RV32-NEXT: vslidedown.vi v9, v9, 3
-; RV32-NEXT: vslidedown.vi v14, v8, 1
-; RV32-NEXT: vmv.x.s a5, v8
-; RV32-NEXT: vslidedown.vi v15, v8, 2
-; RV32-NEXT: vslidedown.vi v8, v8, 3
-; RV32-NEXT: vslidedown.vi v16, v11, 3
-; RV32-NEXT: vslidedown.vi v17, v11, 2
-; RV32-NEXT: vslidedown.vi v18, v11, 1
-; RV32-NEXT: vmv.x.s a6, v11
+; RV32-NEXT: vslidedown.vi v12, v11, 3
+; RV32-NEXT: vslidedown.vi v13, v11, 2
+; RV32-NEXT: vslidedown.vi v14, v11, 1
+; RV32-NEXT: vmv.x.s a0, v11
; RV32-NEXT: vslidedown.vi v11, v10, 3
-; RV32-NEXT: vslidedown.vi v19, v10, 2
-; RV32-NEXT: vmv.x.s a7, v10
-; RV32-NEXT: vslidedown.vi v10, v10, 1
-; RV32-NEXT: vmv.x.s t0, v12
-; RV32-NEXT: vmv.x.s a2, v13
-; RV32-NEXT: vmv.x.s a0, v9
-; RV32-NEXT: vmv.x.s t1, v14
-; RV32-NEXT: vmv.x.s a4, v15
-; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: vslidedown.vi v15, v10, 2
+; RV32-NEXT: vslidedown.vi v16, v10, 1
+; RV32-NEXT: vmv.x.s a1, v10
+; RV32-NEXT: vslidedown.vi v10, v9, 3
+; RV32-NEXT: vslidedown.vi v17, v9, 2
+; RV32-NEXT: vslidedown.vi v18, v9, 1
+; RV32-NEXT: vmv.x.s a2, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 3
+; RV32-NEXT: vslidedown.vi v19, v8, 2
+; RV32-NEXT: vmv.x.s a3, v8
+; RV32-NEXT: vslidedown.vi v8, v8, 1
+; RV32-NEXT: vmv.x.s a4, v12
+; RV32-NEXT: vmv.x.s a5, v13
+; RV32-NEXT: vmv.x.s a6, v14
+; RV32-NEXT: vmv.x.s a7, v11
+; RV32-NEXT: vmv.x.s t0, v15
+; RV32-NEXT: vmv.x.s t1, v16
+; RV32-NEXT: vmv.x.s t2, v10
+; RV32-NEXT: vmv.x.s t3, v17
+; RV32-NEXT: vmv.x.s t4, v18
+; RV32-NEXT: vmv.x.s t5, v9
+; RV32-NEXT: vmv.x.s t6, v19
+; RV32-NEXT: vmv.x.s s0, v8
; RV32-NEXT: vmv.v.x v8, a3
-; RV32-NEXT: vmv.x.s a3, v16
-; RV32-NEXT: vmv.v.x v9, a5
-; RV32-NEXT: vmv.x.s a5, v17
-; RV32-NEXT: vmv.v.x v12, a7
-; RV32-NEXT: vmv.x.s a7, v18
-; RV32-NEXT: vmv.v.x v13, a6
-; RV32-NEXT: li a6, 32
-; RV32-NEXT: vslide1down.vx v8, v8, t0
-; RV32-NEXT: vmv.x.s t0, v11
-; RV32-NEXT: vslide1down.vx v9, v9, t1
-; RV32-NEXT: vmv.x.s t1, v10
-; RV32-NEXT: vslide1down.vx v10, v12, t1
-; RV32-NEXT: vmv.x.s t1, v19
-; RV32-NEXT: vslide1down.vx v11, v13, a7
-; RV32-NEXT: vslide1down.vx v8, v8, a2
-; RV32-NEXT: vslide1down.vx v9, v9, a4
+; RV32-NEXT: vmv.v.x v9, a2
+; RV32-NEXT: vmv.v.x v10, a1
+; RV32-NEXT: vmv.v.x v11, a0
+; RV32-NEXT: vslide1down.vx v8, v8, s0
+; RV32-NEXT: vslide1down.vx v9, v9, t4
; RV32-NEXT: vslide1down.vx v10, v10, t1
-; RV32-NEXT: vslide1down.vx v11, v11, a5
-; RV32-NEXT: vslide1down.vx v8, v8, a0
-; RV32-NEXT: vslide1down.vx v9, v9, a1
+; RV32-NEXT: vslide1down.vx v11, v11, a6
+; RV32-NEXT: vslide1down.vx v8, v8, t6
+; RV32-NEXT: vslide1down.vx v9, v9, t3
; RV32-NEXT: vslide1down.vx v10, v10, t0
-; RV32-NEXT: vslide1down.vx v11, v11, a3
-; RV32-NEXT: vslidedown.vi v12, v9, 1
-; RV32-NEXT: vmv.x.s a0, v9
-; RV32-NEXT: vslidedown.vi v13, v9, 2
-; RV32-NEXT: vslidedown.vi v9, v9, 3
-; RV32-NEXT: vslidedown.vi v14, v8, 1
-; RV32-NEXT: vmv.x.s a1, v8
-; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV32-NEXT: vmul.vv v10, v10, v11
-; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v11, v8, 2
-; RV32-NEXT: vslidedown.vi v8, v8, 3
-; RV32-NEXT: vmv.x.s a2, v12
-; RV32-NEXT: vmv.v.x v12, a0
-; RV32-NEXT: vmv.x.s a0, v13
-; RV32-NEXT: vmv.x.s a3, v9
-; RV32-NEXT: vmv.x.s a4, v14
-; RV32-NEXT: vmv.v.x v9, a1
-; RV32-NEXT: vmv.x.s a1, v11
-; RV32-NEXT: vmv.x.s a5, v8
-; RV32-NEXT: vslide1down.vx v8, v12, a2
-; RV32-NEXT: vslide1down.vx v9, v9, a4
-; RV32-NEXT: vslide1down.vx v8, v8, a0
-; RV32-NEXT: vslide1down.vx v9, v9, a1
-; RV32-NEXT: vslide1down.vx v8, v8, a3
-; RV32-NEXT: vslide1down.vx v9, v9, a5
-; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vslide1down.vx v11, v11, a5
+; RV32-NEXT: vslide1down.vx v8, v8, t5
+; RV32-NEXT: vslide1down.vx v9, v9, t2
+; RV32-NEXT: vslide1down.vx v10, v10, a7
+; RV32-NEXT: vslide1down.vx v11, v11, a4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
; RV32-NEXT: vmul.vv v8, v8, v9
-; RV32-NEXT: vmul.vv v8, v8, v10
; RV32-NEXT: vrgather.vi v9, v8, 1
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vsrl.vx v8, v8, a6
-; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: slli a0, a0, 1
+; RV32-NEXT: srai a0, a0, 1
+; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore s0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
;
-; RV64-LABEL: vreduce_mul_nxv4i63_exact_vlen:
+; RV64-LABEL: vreduce_mul_nxv8i31_exact_vlen:
; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v12, v11, 1
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset s0, -8
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v12, v11, 3
+; RV64-NEXT: vslidedown.vi v13, v11, 2
+; RV64-NEXT: vslidedown.vi v14, v11, 1
; RV64-NEXT: vmv.x.s a0, v11
-; RV64-NEXT: vslidedown.vi v11, v10, 1
+; RV64-NEXT: vslidedown.vi v11, v10, 3
+; RV64-NEXT: vslidedown.vi v15, v10, 2
+; RV64-NEXT: vslidedown.vi v16, v10, 1
; RV64-NEXT: vmv.x.s a1, v10
-; RV64-NEXT: vslidedown.vi v10, v9, 1
+; RV64-NEXT: vslidedown.vi v10, v9, 3
+; RV64-NEXT: vslidedown.vi v17, v9, 2
+; RV64-NEXT: vslidedown.vi v18, v9, 1
; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 3
+; RV64-NEXT: vslidedown.vi v19, v8, 2
; RV64-NEXT: vmv.x.s a3, v8
; RV64-NEXT: vslidedown.vi v8, v8, 1
; RV64-NEXT: vmv.x.s a4, v12
-; RV64-NEXT: vmv.x.s a5, v11
-; RV64-NEXT: vmv.x.s a6, v10
-; RV64-NEXT: vmv.x.s a7, v8
+; RV64-NEXT: vmv.x.s a5, v13
+; RV64-NEXT: vmv.x.s a6, v14
+; RV64-NEXT: vmv.x.s a7, v11
+; RV64-NEXT: vmv.x.s t0, v15
+; RV64-NEXT: vmv.x.s t1, v16
+; RV64-NEXT: vmv.x.s t2, v10
+; RV64-NEXT: vmv.x.s t3, v17
+; RV64-NEXT: vmv.x.s t4, v18
+; RV64-NEXT: vmv.x.s t5, v9
+; RV64-NEXT: vmv.x.s t6, v19
+; RV64-NEXT: vmv.x.s s0, v8
; RV64-NEXT: vmv.v.x v8, a3
; RV64-NEXT: vmv.v.x v9, a2
; RV64-NEXT: vmv.v.x v10, a1
; RV64-NEXT: vmv.v.x v11, a0
-; RV64-NEXT: vslide1down.vx v8, v8, a7
-; RV64-NEXT: vslide1down.vx v9, v9, a6
-; RV64-NEXT: vslide1down.vx v10, v10, a5
+; RV64-NEXT: vslide1down.vx v8, v8, s0
+; RV64-NEXT: vslide1down.vx v9, v9, t4
+; RV64-NEXT: vslide1down.vx v10, v10, t1
+; RV64-NEXT: vslide1down.vx v11, v11, a6
+; RV64-NEXT: vslide1down.vx v8, v8, t6
+; RV64-NEXT: vslide1down.vx v9, v9, t3
+; RV64-NEXT: vslide1down.vx v10, v10, t0
+; RV64-NEXT: vslide1down.vx v11, v11, a5
+; RV64-NEXT: vslide1down.vx v8, v8, t5
+; RV64-NEXT: vslide1down.vx v9, v9, t2
+; RV64-NEXT: vslide1down.vx v10, v10, a7
; RV64-NEXT: vslide1down.vx v11, v11, a4
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vrgather.vi v9, v8, 1
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: slli a0, a0, 33
+; RV64-NEXT: srai a0, a0, 33
+; RV64-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore s0
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
- %red = call i63 @llvm.vector.reduce.mul.nxv4i63(<vscale x 4 x i63> %v)
- ret i63 %red
+ %red = call i31 @llvm.vector.reduce.mul.nxv8i31(<vscale x 8 x i31> %v)
+ ret i31 %red
}
define signext i32 @vreduce_mul_nxv1i32_exact_vlen(<vscale x 1 x i32> %v) vscale_range(2,2) {
>From 6fe6ed53584314b9725fad6905479602c2c9f18f Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 30 Apr 2026 08:45:01 -0700
Subject: [PATCH 6/6] Add an extra test for scalable illegal types more
explicitly
---
.../test/CodeGen/RISCV/rvv/vreductions-int.ll | 56 +++++++++++++++++++
1 file changed, 56 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index f697dfc9637cd..5753fbbb5b7d0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2419,6 +2419,62 @@ define signext i32 @vreduce_mul_nxv4i32_exact_vlen(<vscale x 4 x i32> %v) vscale
ret i32 %red
}
+define signext i32 @vreduce_mul_nxv6i32_from_nxv8i32_exact_vlen(<vscale x 8 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv6i32_from_nxv8i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v11, v10, 3
+; CHECK-NEXT: vslidedown.vi v12, v10, 2
+; CHECK-NEXT: vslidedown.vi v13, v10, 1
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v8, 3
+; CHECK-NEXT: vslidedown.vi v14, v8, 2
+; CHECK-NEXT: vslidedown.vi v15, v8, 1
+; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: vslidedown.vi v8, v9, 3
+; CHECK-NEXT: vslidedown.vi v16, v9, 2
+; CHECK-NEXT: vmv.x.s a2, v9
+; CHECK-NEXT: vslidedown.vi v9, v9, 1
+; CHECK-NEXT: vmv.x.s a3, v11
+; CHECK-NEXT: vmv.x.s a4, v12
+; CHECK-NEXT: vmv.x.s a5, v13
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vmv.x.s a7, v14
+; CHECK-NEXT: vmv.x.s t0, v15
+; CHECK-NEXT: vmv.x.s t1, v8
+; CHECK-NEXT: vmv.x.s t2, v16
+; CHECK-NEXT: vmv.x.s t3, v9
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: vmv.v.x v9, a1
+; CHECK-NEXT: vmv.v.x v10, a0
+; CHECK-NEXT: vslide1down.vx v8, v8, t3
+; CHECK-NEXT: vslide1down.vx v9, v9, t0
+; CHECK-NEXT: vslide1down.vx v10, v10, a5
+; CHECK-NEXT: vslide1down.vx v8, v8, t2
+; CHECK-NEXT: vslide1down.vx v11, v9, a7
+; CHECK-NEXT: vslide1down.vx v10, v10, a4
+; CHECK-NEXT: vslide1down.vx v9, v8, t1
+; CHECK-NEXT: vslide1down.vx v10, v10, a3
+; CHECK-NEXT: vslide1down.vx v8, v11, a6
+; CHECK-NEXT: vmv.v.i v11, 1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 1
+; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %sub = call <vscale x 6 x i32> @llvm.vector.extract.nxv6i32.nxv8i32(<vscale x 8 x i32> %v, i64 0)
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %sub)
+ ret i32 %red
+}
+
define signext i32 @vreduce_mul_nxv6i32_exact_vlen(<vscale x 6 x i32> %v) vscale_range(2,2) {
; CHECK-LABEL: vreduce_mul_nxv6i32_exact_vlen:
; CHECK: # %bb.0:
More information about the llvm-commits
mailing list