[llvm] [RISCV] Improve lowering of llvm.vector.reduce.mul (PR #190628)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 09:32:37 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Philip Reames (preames)
<details>
<summary>Changes</summary>
RVV doesn't have a vredprod instruction, so we're forced to emulate these.
The current ExpandReductions lowering which gets used trips an unfortunate
behavior with exact VLEN - we end up with a bunch of vsetvli toggles caused
by an interaction with lowerShuffleViaVRegSplitting. We end up doing all
the sub-m1 shuffles at m1, but don't recognize that we could do the
multiplies in m1 as well. As a result, we end up toggling back and forth
between LMULs.
This change is somewhat of a blunt hammer; it adjusts the lowering to
do the greater than m1 lowering via the current strategy, then do
a vector.extract, then do the same shuffler reduction lowering on
the smaller type. This has the effect of fixing the exact VLEN case,
and improves register pressure for all cases. The cost is that we end
up with an exact vsetvli to toggle to the m1 type before the sub-m1
portion of the reduction.
This helps all configurations, but particularly helps machines
with large VLENs as there are more reduction steps which can be
performed at m1.
An alternative design would be to try and reduce the LMUL of the
multiply steps, but we've generally shied away from that to date, so
this is side stepping that issue.
---
Patch is 34.55 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/190628.diff
3 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp (+53)
- (modified) llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll (+38-32)
- (modified) llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll (+409-103)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index e8269677fb344..cf7b0b940522b 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -21,6 +21,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
@@ -48,6 +49,7 @@ class RISCVCodeGenPrepare : public InstVisitor<RISCVCodeGenPrepare, bool> {
bool visitAnd(BinaryOperator &BO);
bool visitIntrinsicInst(IntrinsicInst &I);
bool expandVPStrideLoad(IntrinsicInst &I);
+ bool expandMulReduction(IntrinsicInst &I);
bool widenVPMerge(Instruction *I);
bool visitFreezeInst(FreezeInst &BO);
};
@@ -228,6 +230,9 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
if (expandVPStrideLoad(I))
return true;
+ if (expandMulReduction(I))
+ return true;
+
if (widenVPMerge(&I))
return true;
@@ -259,6 +264,54 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
return true;
}
+// Partially expand a vector_reduce_mul wider than M1 to reduce the
+// number of vsetvlis required when VLEN is exactly known.
+bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
+ if (II.getIntrinsicID() != Intrinsic::vector_reduce_mul)
+ return false;
+
+ if (!ST->hasVInstructions())
+ return false;
+
+ Value *TmpVec = II.getArgOperand(0);
+ auto *VecTy = dyn_cast<FixedVectorType>(TmpVec->getType());
+ if (!VecTy)
+ return false;
+
+ unsigned EltSize = VecTy->getScalarSizeInBits();
+ unsigned VF = VecTy->getNumElements();
+ unsigned MinVLen = ST->getRealMinVLen();
+ unsigned M1VF = MinVLen / EltSize;
+
+ if (!isPowerOf2_32(VF) || VF <= M1VF)
+ return false;
+
+ IRBuilder<> Builder(&II);
+
+ // Shuffle-reduce at the original vector width. This just duplicates the
+ // default lowering down to m1.
+ SmallVector<int, 32> ShuffleMask(VF);
+ for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
+ unsigned Half = LiveElts / 2;
+ for (unsigned j = 0; j < Half; ++j)
+ ShuffleMask[j] = Half + j;
+ std::fill(ShuffleMask.begin() + Half, ShuffleMask.end(), -1);
+ Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
+ TmpVec = Builder.CreateMul(TmpVec, Shuf, "bin.rdx");
+ }
+
+ // Extract the M1-sized subvector and emit the final reduction intrinsic.
+ // This is the reason we're here - to force a vsetvli toggle once at m1.
+ auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
+ Value *Sub =
+ Builder.CreateExtractVector(M1Ty, TmpVec, Builder.getInt64(0), "rdx.sub");
+ Value *Rdx =
+ Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
+ II.replaceAllUsesWith(Rdx);
+ II.eraseFromParent();
+ return true;
+}
+
// Always expand zero strided loads so we match more .vx splat patterns, even if
// we have +optimized-zero-stride-loads. RISCVDAGToDAGISel::Select will convert
// it back to a strided load if it's optimized.
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 3e77020ed0213..ed3025cac1fab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -1519,16 +1519,18 @@ define signext i8 @vpreduce_mul_v32i8(i8 signext %s, <32 x i8> %v, <32 x i1> %m,
; RV32-NEXT: vsetvli zero, zero, e8, m2, ta, ma
; RV32-NEXT: vmv.v.i v10, 1
; RV32-NEXT: vmerge.vvm v8, v10, v8, v0
+; RV32-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; RV32-NEXT: vslidedown.vi v10, v8, 16
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 8
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
; RV32-NEXT: mv a1, a2
; RV32-NEXT: call __mulsi3
@@ -1555,16 +1557,18 @@ define signext i8 @vpreduce_mul_v32i8(i8 signext %s, <32 x i8> %v, <32 x i1> %m,
; RV64-NEXT: vsetvli zero, zero, e8, m2, ta, ma
; RV64-NEXT: vmv.v.i v10, 1
; RV64-NEXT: vmerge.vvm v8, v10, v8, v0
+; RV64-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; RV64-NEXT: vslidedown.vi v10, v8, 16
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 8
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vslidedown.vi v10, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: mv a1, a2
; RV64-NEXT: call __muldi3
@@ -1603,15 +1607,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
; RV32-NEXT: vslidedown.vx v12, v8, a0
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vslidedown.vi v12, v8, 16
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 8
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 4
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vslidedown.vi v12, v8, 2
-; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vslidedown.vi v9, v8, 8
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
; RV32-NEXT: mv a1, a2
; RV32-NEXT: call __mulsi3
@@ -1646,15 +1651,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
; RV64-NEXT: vslidedown.vx v12, v8, a0
; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vslidedown.vi v12, v8, 16
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 8
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 4
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vslidedown.vi v12, v8, 2
-; RV64-NEXT: vmul.vv v8, v8, v12
-; RV64-NEXT: vrgather.vi v12, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v12
+; RV64-NEXT: vslidedown.vi v9, v8, 8
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: mv a1, a2
; RV64-NEXT: call __muldi3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
index 9725bb37c679b..d63e7023d2e50 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
@@ -5420,16 +5420,18 @@ define i8 @vreduce_mul_v32i8(ptr %x) {
; CHECK-NEXT: li a1, 32
; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 16, e8, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i8>, ptr %x
@@ -5447,15 +5449,16 @@ define i8 @vreduce_mul_v64i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v12, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i8>, ptr %x
@@ -5476,15 +5479,16 @@ define i8 @vreduce_mul_v128i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v16, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <128 x i8>, ptr %x
@@ -5508,15 +5512,16 @@ define i8 @vreduce_mul_v256i8(ptr %x) {
; CHECK-NEXT: vslidedown.vx v16, v8, a0
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 16
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 8
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 8
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <256 x i8>, ptr %x
@@ -5587,14 +5592,16 @@ define i16 @vreduce_mul_v16i16(ptr %x) {
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle16.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 8, e16, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <16 x i16>, ptr %x
@@ -5611,13 +5618,14 @@ define i16 @vreduce_mul_v32i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v12, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i16>, ptr %x
@@ -5637,13 +5645,14 @@ define i16 @vreduce_mul_v64i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i16>, ptr %x
@@ -5666,13 +5675,14 @@ define i16 @vreduce_mul_v128i16(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 16
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 4
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 4
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <128 x i16>, ptr %x
@@ -5725,12 +5735,14 @@ define i32 @vreduce_mul_v8i32(ptr %x) {
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
+; CHECK-NEXT: vsetivli zero, 4, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v10, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v10
-; CHECK-NEXT: vrgather.vi v10, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <8 x i32>, ptr %x
@@ -5746,11 +5758,12 @@ define i32 @vreduce_mul_v16i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v12, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v12
; CHECK-NEXT: vslidedown.vi v12, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vslidedown.vi v12, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v12
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <16 x i32>, ptr %x
@@ -5769,11 +5782,12 @@ define i32 @vreduce_mul_v32i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <32 x i32>, ptr %x
@@ -5795,11 +5809,12 @@ define i32 @vreduce_mul_v64i32(ptr %x) {
; CHECK-NEXT: vslidedown.vi v16, v8, 8
; CHECK-NEXT: vmul.vv v8, v8, v16
; CHECK-NEXT: vslidedown.vi v16, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vslidedown.vi v16, v8, 2
-; CHECK-NEXT: vmul.vv v8, v8, v16
-; CHECK-NEXT: vrgather.vi v16, v8, 1
-; CHECK-NEXT: vmul.vv v8, v8, v16
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%v = load <64 x i32>, ptr %x
@@ -5860,12 +5875,14 @@ define i64 @vreduce_mul_v4i64(ptr %x) {
; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; RV32-NEXT: vle64.v v8, (a0)
; RV32-NEXT: li a1, 32
+; RV32-NEXT: vsetivli zero, 2, e64, m2, ta, ma
; RV32-NEXT: vslidedown.vi v10, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v10
-; RV32-NEXT: vrgather.vi v10, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v10
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -5874,10 +5891,12 @@ define i64 @vreduce_mul_v4i64(ptr %x) {
; RV64: # %bb.0:
; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; RV64-NEXT: vle64.v v8, (a0)
+; RV64-NEXT: vsetivli zero, 2, e64, m2, ta, ma
; RV64-NEXT: vslidedown.vi v10, v8, 2
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v10
-; RV64-NEXT: vrgather.vi v10, v8, 1
-; RV64-NEXT: vmul.vv v8, v8, v10
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vmv.x.s a0, v8
; RV64-NEXT: ret
%v = load <4 x i64>, ptr %x
@@ -5894,11 +5913,12 @@ define i64 @vreduce_mul_v8i64(ptr %x) {
; RV32-NEXT: vslidedown.vi v12, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vslidedown.vi v12, v8, 2
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v12
-; RV32-NEXT: vrgather.vi v12, v8, 1
-; RV32-NEXT: vmul.vv v8, v8, v12
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; RV32-NEXT: vsrl.vx v8, v8, a1
; RV32-NEXT: vmv.x.s a1, v8
; RV32-NEXT: ret
@@ -5910,9 +5930,10 @@ define i64 @vreduce_mu...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/190628
More information about the llvm-commits
mailing list