[llvm] [RISCV] Improve lowering of llvm.vector.reduce.mul (PR #190628)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 10:06:40 PDT 2026
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/190628
>From f1321ed37c4c34558f654ac334445b57596e3a9d Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Mon, 6 Apr 2026 08:14:29 -0700
Subject: [PATCH 1/7] [RISCV] Add coverage for mul expansion with higher VLEN
---
.../RISCV/rvv/fixed-vectors-reduction-int.ll | 295 ++++++++++++++++++
1 file changed, 295 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
index 9725bb37c679b..9b46c93edaaea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
@@ -6058,3 +6058,298 @@ define i64 @vreduce_mul_v64i64(ptr %x) nounwind {
%red = call i64 @llvm.vector.reduce.mul.v64i64(<64 x i64> %v)
ret i64 %red
}
+
+define i64 @vreduce_mul_v32i64_vscale4(ptr %x) nounwind vscale_range(4,1024) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale4:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT: vle64.v v8, (a0)
+; RV32-NEXT: vslidedown.vi v16, v8, 16
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vslidedown.vi v16, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vslidedown.vi v16, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vslidedown.vi v16, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vrgather.vi v16, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m8, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale4:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 32
+; RV64-NEXT: vsetvli zero, a1, e64, m8, ta, ma
+; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vslidedown.vi v16, v8, 16
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vslidedown.vi v16, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vslidedown.vi v16, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vslidedown.vi v16, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vrgather.vi v16, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
+
+define i64 @vreduce_mul_v32i64_vscale8(ptr %x) nounwind vscale_range(8,1024) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale8:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetvli zero, a1, e64, m4, ta, ma
+; RV32-NEXT: vle64.v v8, (a0)
+; RV32-NEXT: vslidedown.vi v12, v8, 16
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v12, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v12, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v12, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vrgather.vi v12, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale8:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 32
+; RV64-NEXT: vsetvli zero, a1, e64, m4, ta, ma
+; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vslidedown.vi v12, v8, 16
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v12, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v12, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v12, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vrgather.vi v12, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
+
+define i64 @vreduce_mul_v32i64_vscale16(ptr %x) nounwind vscale_range(16,1024) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale16:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetvli zero, a1, e64, m2, ta, ma
+; RV32-NEXT: vle64.v v8, (a0)
+; RV32-NEXT: vslidedown.vi v10, v8, 16
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v10, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v10, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v10, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vrgather.vi v10, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale16:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 32
+; RV64-NEXT: vsetvli zero, a1, e64, m2, ta, ma
+; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vslidedown.vi v10, v8, 16
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v10, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v10, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v10, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vrgather.vi v10, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
+
+define i64 @vreduce_mul_v32i64_vscale4_exact(ptr %x) nounwind vscale_range(4,4) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale4_exact:
+; RV32: # %bb.0:
+; RV32-NEXT: vl8re64.v v8, (a0)
+; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, ma
+; RV32-NEXT: vmv1r.v v16, v12
+; RV32-NEXT: vmv1r.v v17, v13
+; RV32-NEXT: vmv1r.v v18, v14
+; RV32-NEXT: vmv1r.v v19, v15
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vmv1r.v v16, v10
+; RV32-NEXT: vmv1r.v v17, v11
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vmv1r.v v16, v9
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vsetivli zero, 4, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v16, v8, 2
+; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vrgather.vi v16, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m8, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale4_exact:
+; RV64: # %bb.0:
+; RV64-NEXT: vl8re64.v v8, (a0)
+; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, ma
+; RV64-NEXT: vmv1r.v v16, v12
+; RV64-NEXT: vmv1r.v v17, v13
+; RV64-NEXT: vmv1r.v v18, v14
+; RV64-NEXT: vmv1r.v v19, v15
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vmv1r.v v16, v10
+; RV64-NEXT: vmv1r.v v17, v11
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vmv1r.v v16, v9
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vsetivli zero, 4, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v16, v8, 2
+; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vrgather.vi v16, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
+
+define i64 @vreduce_mul_v32i64_vscale8_exact(ptr %x) nounwind vscale_range(8,8) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale8_exact:
+; RV32: # %bb.0:
+; RV32-NEXT: vl4re64.v v8, (a0)
+; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV32-NEXT: vmv1r.v v12, v10
+; RV32-NEXT: vmv1r.v v13, v11
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vmv1r.v v12, v9
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v12, v8, 4
+; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v12, v8, 2
+; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vrgather.vi v12, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale8_exact:
+; RV64: # %bb.0:
+; RV64-NEXT: vl4re64.v v8, (a0)
+; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV64-NEXT: vmv1r.v v12, v10
+; RV64-NEXT: vmv1r.v v13, v11
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vmv1r.v v12, v9
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v12, v8, 4
+; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v12, v8, 2
+; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vrgather.vi v12, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
+
+define i64 @vreduce_mul_v32i64_vscale16_exact(ptr %x) nounwind vscale_range(16,16) {
+; RV32-LABEL: vreduce_mul_v32i64_vscale16_exact:
+; RV32: # %bb.0:
+; RV32-NEXT: vl2re64.v v8, (a0)
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV32-NEXT: vmv1r.v v10, v9
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v10, v8, 8
+; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v10, v8, 4
+; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v10, v8, 2
+; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vrgather.vi v10, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
+; RV32-NEXT: vmv.x.s a1, v8
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_v32i64_vscale16_exact:
+; RV64: # %bb.0:
+; RV64-NEXT: vl2re64.v v8, (a0)
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vmv1r.v v10, v9
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v10, v8, 8
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v10, v8, 4
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v10, v8, 2
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vrgather.vi v10, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: ret
+ %v = load <32 x i64>, ptr %x
+ %red = call i64 @llvm.vector.reduce.mul.v32i64(<32 x i64> %v)
+ ret i64 %red
+}
>From 016383e812b726d90663e62aa66e48ce6d89f4f3 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Mon, 6 Apr 2026 08:49:48 -0700
Subject: [PATCH 2/7] [RISCV] Improve lowering of llvm.vector.reduce.mul
RVV doesn't have a vredprod instruction, so we're forced to emulate these.
The current ExpandReductions lowering which gets used trips an unfortunate
behavior with exact VLEN - we end up with a bunch of vsetvli toggles caused
by an interaction with lowerShuffleViaVRegSplitting. We end up doing all
the sub-m1 shuffles at m1, but don't recognize that we could do the
multiplies in m1 as well. As a result, we end up toggling back and forth
between LMULs.
This change is somewhat of a blunt hammer; it adjusts the lowering to
do the greater than m1 lowering via the current strategy, then do
a vector.extract, then do the same shuffler reduction lowering on
the smaller type. This has the effect of fixing the exact VLEN case,
and improves register pressure for all cases. The cost is that we end
up with an exact vsetvli to toggle to the m1 type before the sub-m1
portion of the reduction.
This helps all configurations, but particularly helps machines
with large VLENs as there are more reduction steps which can be
performed at m1.
An alternative design would be to try and reduce the LMUL of the
multiply steps, but we've generally shied away from that to date, so
this is side stepping that issue.
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 53 +++
.../rvv/fixed-vectors-reduction-int-vp.ll | 70 +--
.../RISCV/rvv/fixed-vectors-reduction-int.ll | 441 +++++++++---------
3 files changed, 317 insertions(+), 247 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index e8269677fb344..cf7b0b940522b 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -21,6 +21,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
@@ -48,6 +49,7 @@ class RISCVCodeGenPrepare : public InstVisitor<RISCVCodeGenPrepare, bool> {
bool visitAnd(BinaryOperator &BO);
bool visitIntrinsicInst(IntrinsicInst &I);
bool expandVPStrideLoad(IntrinsicInst &I);
+ bool expandMulReduction(IntrinsicInst &I);
bool widenVPMerge(Instruction *I);
bool visitFreezeInst(FreezeInst &BO);
};
@@ -228,6 +230,9 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
if (expandVPStrideLoad(I))
return true;
+ if (expandMulReduction(I))
+ return true;
+
if (widenVPMerge(&I))
return true;
@@ -259,6 +264,54 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
return true;
}
+// Partially expand a vector_reduce_mul wider than M1 to reduce the
+// number of vsetvlis required when VLEN is exactly known.
+bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
+ if (II.getIntrinsicID() != Intrinsic::vector_reduce_mul)
+ return false;
+
+ if (!ST->hasVInstructions())
+ return false;
+
+ Value *TmpVec = II.getArgOperand(0);
+ auto *VecTy = dyn_cast<FixedVectorType>(TmpVec->getType());
+ if (!VecTy)
+ return false;
+
+ unsigned EltSize = VecTy->getScalarSizeInBits();
+ unsigned VF = VecTy->getNumElements();
+ unsigned MinVLen = ST->getRealMinVLen();
+ unsigned M1VF = MinVLen / EltSize;
+
+ if (!isPowerOf2_32(VF) || VF <= M1VF)
+ return false;
+
+ IRBuilder<> Builder(&II);
+
+ // Shuffle-reduce at the original vector width. This just duplicates the
+ // default lowering down to m1.
+ SmallVector<int, 32> ShuffleMask(VF);
+ for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
+ unsigned Half = LiveElts / 2;
+ for (unsigned j = 0; j < Half; ++j)
+ ShuffleMask[j] = Half + j;
+ std::fill(ShuffleMask.begin() + Half, ShuffleMask.end(), -1);
+ Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
+ TmpVec = Builder.CreateMul(TmpVec, Shuf, "bin.rdx");
+ }
+
+ // Extract the M1-sized subvector and emit the final reduction intrinsic.
+ // This is the reason we're here - to force a vsetvli toggle once at m1.
+ auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
+ Value *Sub =
+ Builder.CreateExtractVector(M1Ty, TmpVec, Builder.getInt64(0), "rdx.sub");
+ Value *Rdx =
+ Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
+ II.replaceAllUsesWith(Rdx);
+ II.eraseFromParent();
+ return true;
+}
+
// Always expand zero strided loads so we match more .vx splat patterns, even if
// we have +optimized-zero-stride-loads. RISCVDAGToDAGISel::Select will convert
// it back to a strided load if it's optimized.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 3e77020ed0213..ed3025cac1fab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -1519,16 +1519,18 @@ define signext i8 @vpreduce_mul_v32i8(i8 signext %s, <32 x i8> %v, <32 x i1> %m,
; RV32-NEXT: vsetvli zero, zero, e8, m2, ta, ma
; RV32-NEXT: vmv.v.i v10, 1
; RV32-NEXT: vmerge.vvm v8, v10, v8, v0
+; RV32-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; RV32-NEXT: vslidedown.vi v10, v8, 16
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 8
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
; RV32-NEXT: mv a1, a2
; RV32-NEXT: call __mulsi3
@@ -1555,16 +1557,18 @@ define signext i8 @vpreduce_mul_v32i8(i8 signext %s, <32 x i8> %v, <32 x i1> %m,
; RV64-NEXT: vsetvli zero, zero, e8, m2, ta, ma
; RV64-NEXT: vmv.v.i v10, 1
; RV64-NEXT: vmerge.vvm v8, v10, v8, v0
+; RV64-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; RV64-NEXT: vslidedown.vi v10, v8, 16
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 8
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: mv a1, a2
; RV64-NEXT: call __muldi3
@@ -1603,15 +1607,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
; RV32-NEXT: vslidedown.vx v12, v8, a0
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vslidedown.vi v12, v8, 16
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 8
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
; RV32-NEXT: mv a1, a2
; RV32-NEXT: call __mulsi3
@@ -1646,15 +1651,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
; RV64-NEXT: vslidedown.vx v12, v8, a0
; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vslidedown.vi v12, v8, 16
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 8
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vrgather.vi v12, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: mv a1, a2
; RV64-NEXT: call __muldi3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
index 9b46c93edaaea..d63e7023d2e50 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
@@ -5420,16 +5420,18 @@ define i8 @vreduce_mul_v32i8(ptr %x) {
; CHECK-NEXT: li a1, 32
; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i8>, ptr %x
@@ -5447,15 +5449,16 @@ define i8 @vreduce_mul_v64i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v12, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i8>, ptr %x
@@ -5476,15 +5479,16 @@ define i8 @vreduce_mul_v128i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v16, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <128 x i8>, ptr %x
@@ -5508,15 +5512,16 @@ define i8 @vreduce_mul_v256i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v16, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <256 x i8>, ptr %x
@@ -5587,14 +5592,16 @@ define i16 @vreduce_mul_v16i16(ptr %x) {
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle16.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 8, e16, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <16 x i16>, ptr %x
@@ -5611,13 +5618,14 @@ define i16 @vreduce_mul_v32i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v12, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i16>, ptr %x
@@ -5637,13 +5645,14 @@ define i16 @vreduce_mul_v64i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i16>, ptr %x
@@ -5666,13 +5675,14 @@ define i16 @vreduce_mul_v128i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <128 x i16>, ptr %x
@@ -5725,12 +5735,14 @@ define i32 @vreduce_mul_v8i32(ptr %x) {
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 4, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <8 x i32>, ptr %x
@@ -5746,11 +5758,12 @@ define i32 @vreduce_mul_v16i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v12, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <16 x i32>, ptr %x
@@ -5769,11 +5782,12 @@ define i32 @vreduce_mul_v32i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i32>, ptr %x
@@ -5795,11 +5809,12 @@ define i32 @vreduce_mul_v64i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i32>, ptr %x
@@ -5860,12 +5875,14 @@ define i64 @vreduce_mul_v4i64(ptr %x) {
; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; RV32-NEXT: vle64.v v8, (a0)
; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
; RV32-NEXT: vslidedown.vi v10, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -5874,10 +5891,12 @@ define i64 @vreduce_mul_v4i64(ptr %x) {
; RV64: # %bb.0:
; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
; RV64-NEXT: vslidedown.vi v10, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <4 x i64>, ptr %x
@@ -5894,11 +5913,12 @@ define i64 @vreduce_mul_v8i64(ptr %x) {
; RV32-NEXT: vslidedown.vi v12, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vslidedown.vi v12, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -5910,9 +5930,10 @@ define i64 @vreduce_mul_v8i64(ptr %x) {
; RV64-NEXT: vslidedown.vi v12, v8, 4
; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vslidedown.vi v12, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vrgather.vi v12, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <8 x i64>, ptr %x
@@ -5931,11 +5952,12 @@ define i64 @vreduce_mul_v16i64(ptr %x) {
; RV32-NEXT: vslidedown.vi v16, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vrgather.vi v16, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m8, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -5949,9 +5971,10 @@ define i64 @vreduce_mul_v16i64(ptr %x) {
; RV64-NEXT: vslidedown.vi v16, v8, 4
; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vslidedown.vi v16, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vrgather.vi v16, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <16 x i64>, ptr %x
@@ -5966,18 +5989,20 @@ define i64 @vreduce_mul_v32i64(ptr %x) {
; RV32-NEXT: vle64.v v8, (a0)
; RV32-NEXT: addi a0, a0, 128
; RV32-NEXT: vle64.v v16, (a0)
+; RV32-NEXT: li a1, 32
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 8
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vrgather.vi v16, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vslidedown.vi v8, v8, 1
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
;
@@ -5993,9 +6018,10 @@ define i64 @vreduce_mul_v32i64(ptr %x) {
; RV64-NEXT: vslidedown.vi v16, v8, 4
; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vslidedown.vi v16, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vrgather.vi v16, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6012,22 +6038,24 @@ define i64 @vreduce_mul_v64i64(ptr %x) nounwind {
; RV32-NEXT: vle64.v v16, (a1)
; RV32-NEXT: addi a1, a0, 256
; RV32-NEXT: addi a0, a0, 128
-; RV32-NEXT: vle64.v v24, (a0)
-; RV32-NEXT: vle64.v v0, (a1)
-; RV32-NEXT: vmul.vv v16, v24, v16
-; RV32-NEXT: vmul.vv v8, v8, v0
+; RV32-NEXT: vle64.v v0, (a0)
+; RV32-NEXT: vle64.v v24, (a1)
+; RV32-NEXT: li a1, 32
+; RV32-NEXT: vmul.vv v16, v0, v16
+; RV32-NEXT: vmul.vv v8, v8, v24
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 8
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vrgather.vi v16, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vslidedown.vi v8, v8, 1
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
;
@@ -6049,9 +6077,10 @@ define i64 @vreduce_mul_v64i64(ptr %x) nounwind {
; RV64-NEXT: vslidedown.vi v16, v8, 4
; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vslidedown.vi v16, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vrgather.vi v16, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <64 x i64>, ptr %x
@@ -6070,13 +6099,14 @@ define i64 @vreduce_mul_v32i64_vscale4(ptr %x) nounwind vscale_range(4,1024) {
; RV32-NEXT: vslidedown.vi v16, v8, 8
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vslidedown.vi v16, v8, 4
+; RV32-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vslidedown.vi v16, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vrgather.vi v16, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m8, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6091,11 +6121,12 @@ define i64 @vreduce_mul_v32i64_vscale4(ptr %x) nounwind vscale_range(4,1024) {
; RV64-NEXT: vslidedown.vi v16, v8, 8
; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vslidedown.vi v16, v8, 4
+; RV64-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vslidedown.vi v16, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vrgather.vi v16, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6112,15 +6143,16 @@ define i64 @vreduce_mul_v32i64_vscale8(ptr %x) nounwind vscale_range(8,1024) {
; RV32-NEXT: vslidedown.vi v12, v8, 16
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vslidedown.vi v12, v8, 8
+; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6133,13 +6165,14 @@ define i64 @vreduce_mul_v32i64_vscale8(ptr %x) nounwind vscale_range(8,1024) {
; RV64-NEXT: vslidedown.vi v12, v8, 16
; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vslidedown.vi v12, v8, 8
+; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vrgather.vi v12, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6153,18 +6186,20 @@ define i64 @vreduce_mul_v32i64_vscale16(ptr %x) nounwind vscale_range(16,1024) {
; RV32-NEXT: li a1, 32
; RV32-NEXT: vsetvli zero, a1, e64, m2, ta, ma
; RV32-NEXT: vle64.v v8, (a0)
+; RV32-NEXT: vsetivli zero, 16, e64, m2, ta, ma
; RV32-NEXT: vslidedown.vi v10, v8, 16
+; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 8
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6174,16 +6209,18 @@ define i64 @vreduce_mul_v32i64_vscale16(ptr %x) nounwind vscale_range(16,1024) {
; RV64-NEXT: li a1, 32
; RV64-NEXT: vsetvli zero, a1, e64, m2, ta, ma
; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vsetivli zero, 16, e64, m2, ta, ma
; RV64-NEXT: vslidedown.vi v10, v8, 16
+; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 8
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6202,19 +6239,17 @@ define i64 @vreduce_mul_v32i64_vscale4_exact(ptr %x) nounwind vscale_range(4,4)
; RV32-NEXT: vmv1r.v v19, v15
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vmv1r.v v16, v10
-; RV32-NEXT: vmv1r.v v17, v11
-; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: li a1, 32
-; RV32-NEXT: vmv1r.v v16, v9
+; RV32-NEXT: vmv1r.v v17, v11
; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vsetivli zero, 4, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v16, v8, 2
-; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vrgather.vi v16, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v16
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m8, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6231,14 +6266,12 @@ define i64 @vreduce_mul_v32i64_vscale4_exact(ptr %x) nounwind vscale_range(4,4)
; RV64-NEXT: vmv1r.v v16, v10
; RV64-NEXT: vmv1r.v v17, v11
; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vmv1r.v v16, v9
-; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vsetivli zero, 4, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v16, v8, 2
-; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vrgather.vi v16, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v16
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6252,23 +6285,19 @@ define i64 @vreduce_mul_v32i64_vscale8_exact(ptr %x) nounwind vscale_range(8,8)
; RV32-NEXT: vl4re64.v v8, (a0)
; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
; RV32-NEXT: vmv1r.v v12, v10
-; RV32-NEXT: vmv1r.v v13, v11
-; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: li a1, 32
-; RV32-NEXT: vmv1r.v v12, v9
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v12, v8, 4
-; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
+; RV32-NEXT: vmv1r.v v13, v11
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v12, v8, 2
-; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6280,18 +6309,14 @@ define i64 @vreduce_mul_v32i64_vscale8_exact(ptr %x) nounwind vscale_range(8,8)
; RV64-NEXT: vmv1r.v v12, v10
; RV64-NEXT: vmv1r.v v13, v11
; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vmv1r.v v12, v9
-; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v12, v8, 4
-; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v12, v8, 2
-; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vrgather.vi v12, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
@@ -6304,25 +6329,18 @@ define i64 @vreduce_mul_v32i64_vscale16_exact(ptr %x) nounwind vscale_range(16,1
; RV32: # %bb.0:
; RV32-NEXT: vl2re64.v v8, (a0)
; RV32-NEXT: li a1, 32
-; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV32-NEXT: vmv1r.v v10, v9
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v10, v8, 8
-; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v10, v8, 4
-; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v10
; RV32-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v10, v8, 2
-; RV32-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -6330,23 +6348,16 @@ define i64 @vreduce_mul_v32i64_vscale16_exact(ptr %x) nounwind vscale_range(16,1
; RV64-LABEL: vreduce_mul_v32i64_vscale16_exact:
; RV64: # %bb.0:
; RV64-NEXT: vl2re64.v v8, (a0)
-; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV64-NEXT: vmv1r.v v10, v9
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v10, v8, 8
-; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v10, v8, 4
-; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v10
; RV64-NEXT: vsetivli zero, 16, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v10, v8, 2
-; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <32 x i64>, ptr %x
>From b60dab9925e2918b157ea092279ce088f9bc79cc Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Wed, 8 Apr 2026 19:15:00 -0700
Subject: [PATCH 3/7] Address review comment
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index cf7b0b940522b..c4f50eb767296 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -293,8 +293,7 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
SmallVector<int, 32> ShuffleMask(VF);
for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
unsigned Half = LiveElts / 2;
- for (unsigned j = 0; j < Half; ++j)
- ShuffleMask[j] = Half + j;
+ std::iota(ShuffleMask.begin(), ShuffleMask.begin() + Half, Half);
std::fill(ShuffleMask.begin() + Half, ShuffleMask.end(), -1);
Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
TmpVec = Builder.CreateMul(TmpVec, Shuf, "bin.rdx");
>From d773ee7d53d23582fede7941219d69d92196f1f0 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 9 Apr 2026 09:39:35 -0700
Subject: [PATCH 4/7] Update llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
Co-authored-by: Luke Lau <luke_lau at icloud.com>
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index c4f50eb767296..82f4eccc87d9f 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -303,7 +303,7 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
// This is the reason we're here - to force a vsetvli toggle once at m1.
auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
Value *Sub =
- Builder.CreateExtractVector(M1Ty, TmpVec, Builder.getInt64(0), "rdx.sub");
+ Builder.CreateExtractVector(M1Ty, TmpVec, 0, "rdx.sub");
Value *Rdx =
Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
II.replaceAllUsesWith(Rdx);
>From 8111dbb83f86de85b3f39a50e42338e60b84c892 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 9 Apr 2026 09:40:27 -0700
Subject: [PATCH 5/7] Improve comment
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 82f4eccc87d9f..2d43a02d0ee99 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -265,7 +265,8 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
}
// Partially expand a vector_reduce_mul wider than M1 to reduce the
-// number of vsetvlis required when VLEN is exactly known.
+// number of vsetvlis required when VLEN is exactly known, and
+// reducing register pressure in all cases.
bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
if (II.getIntrinsicID() != Intrinsic::vector_reduce_mul)
return false;
>From 70be67cef35e73d8c25eeffdc0c1c3649a197aa3 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 9 Apr 2026 09:42:39 -0700
Subject: [PATCH 6/7] Build fix
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 2d43a02d0ee99..c20006e86b529 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -304,7 +304,7 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
// This is the reason we're here - to force a vsetvli toggle once at m1.
auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
Value *Sub =
- Builder.CreateExtractVector(M1Ty, TmpVec, 0, "rdx.sub");
+ Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)0, "rdx.sub");
Value *Rdx =
Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
II.replaceAllUsesWith(Rdx);
>From 6b954f8d93084834536838cd1ee24cd51c279c92 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail at philipreames.com>
Date: Thu, 9 Apr 2026 09:59:14 -0700
Subject: [PATCH 7/7] Clang format fix
---
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index c20006e86b529..3880d0e68a764 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -304,7 +304,7 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
// This is the reason we're here - to force a vsetvli toggle once at m1.
auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
Value *Sub =
- Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)0, "rdx.sub");
+ Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)0, "rdx.sub");
Value *Rdx =
Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
II.replaceAllUsesWith(Rdx);
More information about the llvm-commits
mailing list