[llvm] 899ab68 - [RISCV] Optimize scalable multiply reductions lowerings (#193528)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 07:54:11 PDT 2026
Author: Philip Reames
Date: 2026-05-01T07:54:06-07:00
New Revision: 899ab68e299b7416b46b5f7d39536c1a6e8eb616
URL: https://github.com/llvm/llvm-project/commit/899ab68e299b7416b46b5f7d39536c1a6e8eb616
DIFF: https://github.com/llvm/llvm-project/commit/899ab68e299b7416b46b5f7d39536c1a6e8eb616.diff
LOG: [RISCV] Optimize scalable multiply reductions lowerings (#193528)
Now that we can correctly lower a scalable multiply reduction, lets
improve the lowering for two common cases.
1) If we have exact VLEN, just convert to the fixed vector
form and lower so that we get shuffles instead of the
slow painful loop.
2) Handle the high LMUL case by splitting down to m1 via
a reduce tree. It's only at the final stage that we need
to use the loop to handle the unknown number of elements.
For context, this is mostly for completeness. I don't plan on going any
further to improve the code quality here. There's more we can do (e.g.
exploiting minimum element count to use shuffles for the first couple
stages), but that can be future work.
Code written by Claude with heavy guidance and review by me.
Added:
Modified:
llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 4ce5432d67fcf..3ac4a200323c3 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -264,9 +264,31 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
return true;
}
-// Partially expand a vector_reduce_mul wider than M1 to reduce the
-// number of vsetvlis required when VLEN is exactly known, and
-// reducing register pressure in all cases.
+// Extract pieces of size PieceEC from Vec, then build a binary tree of
+// element-wise multiplies reducing to a single piece.
+static Value *buildMulTree(IRBuilder<> &Builder, ElementCount PieceEC,
+ Value *Vec) {
+ auto *VecTy = cast<VectorType>(Vec->getType());
+ auto *PieceTy = VectorType::get(VecTy->getElementType(), PieceEC);
+ unsigned PieceElts = PieceEC.getKnownMinValue();
+ unsigned NumPieces = VecTy->getElementCount().getKnownMinValue() / PieceElts;
+ assert(isPowerOf2_32(NumPieces));
+
+ SmallVector<Value *, 8> Pieces(NumPieces);
+ for (unsigned i = 0; i < NumPieces; i++)
+ Pieces[i] = Builder.CreateExtractVector(PieceTy, Vec, i * PieceElts);
+
+ while (Pieces.size() > 1) {
+ for (unsigned i = 0; i < Pieces.size() / 2; i++)
+ Pieces[i] =
+ Builder.CreateMul(Pieces[i * 2], Pieces[i * 2 + 1], "bin.rdx");
+ Pieces.truncate(Pieces.size() / 2);
+ }
+ return Pieces[0];
+}
+
+// Partially expand a vector_reduce_mul wider than M1 to reduce
+// register pressure and the number of vsetvlis required.
bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
if (II.getIntrinsicID() != Intrinsic::vector_reduce_mul)
return false;
@@ -275,12 +297,44 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
return false;
Value *TmpVec = II.getArgOperand(0);
- auto *VecTy = dyn_cast<FixedVectorType>(TmpVec->getType());
- if (!VecTy)
- return false;
-
+ auto *VecTy = cast<VectorType>(TmpVec->getType());
unsigned EltSize = VecTy->getScalarSizeInBits();
- unsigned VF = VecTy->getNumElements();
+
+ if (auto *ScalTy = dyn_cast<ScalableVectorType>(VecTy)) {
+ unsigned MinElts = ScalTy->getMinNumElements();
+
+ if (auto VLen = ST->getRealVLen()) {
+ // If VLEN is exactly known, convert to a fixed vector reduction and
+ // recurse to let the fixed path handle it (shuffle reduction instead
+ // of a scalar loop).
+ unsigned VScale = *VLen / RISCV::RVVBitsPerBlock;
+ auto *FixedTy =
+ FixedVectorType::get(VecTy->getElementType(), MinElts * VScale);
+ IRBuilder<> Builder(&II);
+ Value *Fixed = Builder.CreateExtractVector(FixedTy, TmpVec, (uint64_t)0);
+ auto *FixedRdx = cast<IntrinsicInst>(Builder.CreateIntrinsic(
+ Intrinsic::vector_reduce_mul, {FixedTy}, {Fixed}));
+ II.replaceAllUsesWith(FixedRdx);
+ II.eraseFromParent();
+ expandMulReduction(*FixedRdx);
+ return true;
+ }
+
+ unsigned M1MinElts = RISCV::RVVBitsPerBlock / EltSize;
+ if (MinElts <= M1MinElts || !isPowerOf2_32(MinElts / M1MinElts))
+ return false;
+
+ IRBuilder<> Builder(&II);
+ auto M1EC = ElementCount::getScalable(M1MinElts);
+ Value *Reduced = buildMulTree(Builder, M1EC, TmpVec);
+ Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul,
+ {Reduced->getType()}, {Reduced});
+ II.replaceAllUsesWith(Rdx);
+ II.eraseFromParent();
+ return true;
+ }
+
+ unsigned VF = cast<FixedVectorType>(VecTy)->getNumElements();
unsigned MinVLen = ST->getRealMinVLen();
unsigned M1VF = MinVLen / EltSize;
@@ -288,31 +342,19 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
return false;
IRBuilder<> Builder(&II);
- auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
+ auto M1EC = ElementCount::getFixed(M1VF);
+ auto *M1Ty = VectorType::get(VecTy->getElementType(), M1EC);
// When VLEN is exactly known, extract m1 pieces and build a mul tree.
// This greatly reduces register pressure during the reduction, and
// avoids all but one vsetvli (the one from original LMUL to m1).
// TODO: Generalize to handle the splitting case.
if (MinVLen == ST->getRealMaxVLen() && VF <= 8 * M1VF) {
- unsigned NumM1 = VF / M1VF;
- assert(isPowerOf2_32(NumM1) && NumM1 <= 8);
- SmallVector<Value *, 8> Pieces(NumM1);
- for (unsigned i = 0; i < NumM1; i++)
- Pieces[i] =
- Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)(i * M1VF));
-
- while (Pieces.size() > 1) {
- for (unsigned i = 0; i < Pieces.size() / 2; i++)
- Pieces[i] =
- Builder.CreateMul(Pieces[i * 2], Pieces[i * 2 + 1], "bin.rdx");
- Pieces.truncate(Pieces.size() / 2);
- }
- TmpVec = Pieces[0];
+ TmpVec = buildMulTree(Builder, M1EC, TmpVec);
} else {
// For non-exact VLEN, shuffle-reduce at the original vector width down to
// m1, then extract. This prioritizes reducing the number of vsetvli
- // over maximual reduction of LMUL for the intermediate states.
+ // over maximal reduction of LMUL for the intermediate states.
SmallVector<int, 32> ShuffleMask(VF);
for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
unsigned Half = LiveElts / 2;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
index 267d9b557a785..5753fbbb5b7d0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int.ll
@@ -2034,14 +2034,16 @@ define signext i32 @vreduce_mul_nxv4i32(<vscale x 4 x i32> %v) {
; RV32-LABEL: vreduce_mul_nxv4i32:
; RV32: # %bb.0:
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: csrr a2, vlenb
-; RV32-NEXT: srli a2, a2, 1
+; RV32-NEXT: srli a2, a2, 2
; RV32-NEXT: li a0, 1
-; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; RV32-NEXT: .LBB122_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v10, v8, a1
-; RV32-NEXT: vmv.x.s a3, v10
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a1
+; RV32-NEXT: vmv.x.s a3, v9
; RV32-NEXT: addi a1, a1, 1
; RV32-NEXT: mul a0, a0, a3
; RV32-NEXT: bne a1, a2, .LBB122_1
@@ -2051,14 +2053,16 @@ define signext i32 @vreduce_mul_nxv4i32(<vscale x 4 x i32> %v) {
; RV64-LABEL: vreduce_mul_nxv4i32:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 1
+; RV64-NEXT: srli a2, a2, 2
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; RV64-NEXT: .LBB122_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v10, v8, a1
-; RV64-NEXT: vmv.x.s a3, v10
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mulw a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB122_1
@@ -2120,18 +2124,20 @@ define i64 @vreduce_mul_nxv2i64(<vscale x 2 x i64> %v) {
; RV32: # %bb.0:
; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: csrr a3, vlenb
; RV32-NEXT: li a0, 1
-; RV32-NEXT: srli a3, a3, 2
+; RV32-NEXT: srli a3, a3, 3
; RV32-NEXT: li a4, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma
; RV32-NEXT: .LBB124_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v10, v8, a2
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v12, v10, a4
-; RV32-NEXT: vmv.x.s a5, v10
-; RV32-NEXT: vmv.x.s a6, v12
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
; RV32-NEXT: mulhu a7, a0, a5
; RV32-NEXT: mul a1, a1, a5
; RV32-NEXT: mul a6, a0, a6
@@ -2145,14 +2151,16 @@ define i64 @vreduce_mul_nxv2i64(<vscale x 2 x i64> %v) {
; RV64-LABEL: vreduce_mul_nxv2i64:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 2
+; RV64-NEXT: srli a2, a2, 3
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma
; RV64-NEXT: .LBB124_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v10, v8, a1
-; RV64-NEXT: vmv.x.s a3, v10
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mul a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB124_1
@@ -2167,18 +2175,22 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
; RV32: # %bb.0:
; RV32-NEXT: li a2, 0
; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
; RV32-NEXT: csrr a3, vlenb
; RV32-NEXT: li a0, 1
-; RV32-NEXT: srli a3, a3, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: srli a3, a3, 3
; RV32-NEXT: li a4, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma
; RV32-NEXT: .LBB125_1: # %rdx.loop
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: vslidedown.vx v12, v8, a2
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a2
; RV32-NEXT: addi a2, a2, 1
-; RV32-NEXT: vsrl.vx v16, v12, a4
-; RV32-NEXT: vmv.x.s a5, v12
-; RV32-NEXT: vmv.x.s a6, v16
+; RV32-NEXT: vsrl.vx v10, v9, a4
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vmv.x.s a6, v10
; RV32-NEXT: mulhu a7, a0, a5
; RV32-NEXT: mul a1, a1, a5
; RV32-NEXT: mul a6, a0, a6
@@ -2192,14 +2204,18 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
; RV64-LABEL: vreduce_mul_nxv4i64:
; RV64: # %bb.0:
; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
; RV64-NEXT: csrr a2, vlenb
-; RV64-NEXT: srli a2, a2, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: srli a2, a2, 3
; RV64-NEXT: li a0, 1
-; RV64-NEXT: vsetivli zero, 1, e64, m4, ta, ma
; RV64-NEXT: .LBB125_1: # %rdx.loop
; RV64-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64-NEXT: vslidedown.vx v12, v8, a1
-; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
; RV64-NEXT: addi a1, a1, 1
; RV64-NEXT: mul a0, a0, a3
; RV64-NEXT: bne a1, a2, .LBB125_1
@@ -2208,3 +2224,458 @@ define i64 @vreduce_mul_nxv4i64(<vscale x 4 x i64> %v) {
%red = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v)
ret i64 %red
}
+
+; Non-power-of-two number of M1 pieces; should not be split by buildMulTree.
+define signext i32 @vreduce_mul_nxv6i32(<vscale x 6 x i32> %v) {
+; RV32-LABEL: vreduce_mul_nxv6i32:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: srli a2, a0, 2
+; RV32-NEXT: sub a2, a0, a2
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: vsetivli zero, 1, e32, m4, ta, ma
+; RV32-NEXT: .LBB126_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: vslidedown.vx v12, v8, a1
+; RV32-NEXT: vmv.x.s a3, v12
+; RV32-NEXT: addi a1, a1, 1
+; RV32-NEXT: mul a0, a0, a3
+; RV32-NEXT: bne a1, a2, .LBB126_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv6i32:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 0
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: srli a2, a0, 2
+; RV64-NEXT: sub a2, a0, a2
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: vsetivli zero, 1, e32, m4, ta, ma
+; RV64-NEXT: .LBB126_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vslidedown.vx v12, v8, a1
+; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: addi a1, a1, 1
+; RV64-NEXT: mulw a0, a0, a3
+; RV64-NEXT: bne a1, a2, .LBB126_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %v)
+ ret i32 %red
+}
+
+; Illegal element type; should not be split by buildMulTree.
+define signext i31 @vreduce_mul_nxv8i31(<vscale x 8 x i31> %v) {
+; RV32-LABEL: vreduce_mul_nxv8i31:
+; RV32: # %bb.0:
+; RV32-NEXT: li a0, 0
+; RV32-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: srli a2, a2, 2
+; RV32-NEXT: li a1, 1
+; RV32-NEXT: .LBB127_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a0
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: addi a0, a0, 1
+; RV32-NEXT: mul a1, a1, a3
+; RV32-NEXT: bne a0, a2, .LBB127_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: slli a0, a1, 1
+; RV32-NEXT: srai a0, a0, 1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv8i31:
+; RV64: # %bb.0:
+; RV64-NEXT: li a0, 0
+; RV64-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: srli a2, a2, 2
+; RV64-NEXT: li a1, 1
+; RV64-NEXT: .LBB127_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a0
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: addi a0, a0, 1
+; RV64-NEXT: mul a1, a1, a3
+; RV64-NEXT: bne a0, a2, .LBB127_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: slli a0, a1, 33
+; RV64-NEXT: srai a0, a0, 33
+; RV64-NEXT: ret
+ %red = call i31 @llvm.vector.reduce.mul.nxv8i31(<vscale x 8 x i31> %v)
+ ret i31 %red
+}
+
+define signext i32 @vreduce_mul_nxv32i32(<vscale x 32 x i32> %v) {
+; RV32-LABEL: vreduce_mul_nxv32i32:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v10, v12, v13
+; RV32-NEXT: vmul.vv v11, v14, v15
+; RV32-NEXT: vmul.vv v12, v16, v17
+; RV32-NEXT: vmul.vv v13, v18, v19
+; RV32-NEXT: vmul.vv v14, v20, v21
+; RV32-NEXT: vmul.vv v15, v22, v23
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v10, v12, v13
+; RV32-NEXT: vmul.vv v11, v14, v15
+; RV32-NEXT: srli a2, a2, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: .LBB128_1: # %rdx.loop
+; RV32-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vx v9, v8, a1
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: addi a1, a1, 1
+; RV32-NEXT: mul a0, a0, a3
+; RV32-NEXT: bne a1, a2, .LBB128_1
+; RV32-NEXT: # %bb.2: # %rdx.exit
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv32i32:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 0
+; RV64-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v10, v12, v13
+; RV64-NEXT: vmul.vv v11, v14, v15
+; RV64-NEXT: vmul.vv v12, v16, v17
+; RV64-NEXT: vmul.vv v13, v18, v19
+; RV64-NEXT: vmul.vv v14, v20, v21
+; RV64-NEXT: vmul.vv v15, v22, v23
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v10, v12, v13
+; RV64-NEXT: vmul.vv v11, v14, v15
+; RV64-NEXT: srli a2, a2, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: .LBB128_1: # %rdx.loop
+; RV64-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vx v9, v8, a1
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: addi a1, a1, 1
+; RV64-NEXT: mulw a0, a0, a3
+; RV64-NEXT: bne a1, a2, .LBB128_1
+; RV64-NEXT: # %bb.2: # %rdx.exit
+; RV64-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv32i32(<vscale x 32 x i32> %v)
+ ret i32 %red
+}
+
+; With known VLEN, scalable mul reductions can use fixed-vector shuffle
+; reduction instead of a scalar loop.
+
+define signext i32 @vreduce_mul_nxv2i32_exact_vlen(<vscale x 2 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv2i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv2i32(<vscale x 2 x i32> %v)
+ ret i32 %red
+}
+
+define signext i32 @vreduce_mul_nxv4i32_exact_vlen(<vscale x 4 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv4i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v)
+ ret i32 %red
+}
+
+define signext i32 @vreduce_mul_nxv6i32_from_nxv8i32_exact_vlen(<vscale x 8 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv6i32_from_nxv8i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v11, v10, 3
+; CHECK-NEXT: vslidedown.vi v12, v10, 2
+; CHECK-NEXT: vslidedown.vi v13, v10, 1
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v8, 3
+; CHECK-NEXT: vslidedown.vi v14, v8, 2
+; CHECK-NEXT: vslidedown.vi v15, v8, 1
+; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: vslidedown.vi v8, v9, 3
+; CHECK-NEXT: vslidedown.vi v16, v9, 2
+; CHECK-NEXT: vmv.x.s a2, v9
+; CHECK-NEXT: vslidedown.vi v9, v9, 1
+; CHECK-NEXT: vmv.x.s a3, v11
+; CHECK-NEXT: vmv.x.s a4, v12
+; CHECK-NEXT: vmv.x.s a5, v13
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vmv.x.s a7, v14
+; CHECK-NEXT: vmv.x.s t0, v15
+; CHECK-NEXT: vmv.x.s t1, v8
+; CHECK-NEXT: vmv.x.s t2, v16
+; CHECK-NEXT: vmv.x.s t3, v9
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: vmv.v.x v9, a1
+; CHECK-NEXT: vmv.v.x v10, a0
+; CHECK-NEXT: vslide1down.vx v8, v8, t3
+; CHECK-NEXT: vslide1down.vx v9, v9, t0
+; CHECK-NEXT: vslide1down.vx v10, v10, a5
+; CHECK-NEXT: vslide1down.vx v8, v8, t2
+; CHECK-NEXT: vslide1down.vx v11, v9, a7
+; CHECK-NEXT: vslide1down.vx v10, v10, a4
+; CHECK-NEXT: vslide1down.vx v9, v8, t1
+; CHECK-NEXT: vslide1down.vx v10, v10, a3
+; CHECK-NEXT: vslide1down.vx v8, v11, a6
+; CHECK-NEXT: vmv.v.i v11, 1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 1
+; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %sub = call <vscale x 6 x i32> @llvm.vector.extract.nxv6i32.nxv8i32(<vscale x 8 x i32> %v, i64 0)
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %sub)
+ ret i32 %red
+}
+
+define signext i32 @vreduce_mul_nxv6i32_exact_vlen(<vscale x 6 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv6i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v11, v10, 3
+; CHECK-NEXT: vslidedown.vi v12, v10, 2
+; CHECK-NEXT: vslidedown.vi v13, v10, 1
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v8, 3
+; CHECK-NEXT: vslidedown.vi v14, v8, 2
+; CHECK-NEXT: vslidedown.vi v15, v8, 1
+; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: vslidedown.vi v8, v9, 3
+; CHECK-NEXT: vslidedown.vi v16, v9, 2
+; CHECK-NEXT: vmv.x.s a2, v9
+; CHECK-NEXT: vslidedown.vi v9, v9, 1
+; CHECK-NEXT: vmv.x.s a3, v11
+; CHECK-NEXT: vmv.x.s a4, v12
+; CHECK-NEXT: vmv.x.s a5, v13
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vmv.x.s a7, v14
+; CHECK-NEXT: vmv.x.s t0, v15
+; CHECK-NEXT: vmv.x.s t1, v8
+; CHECK-NEXT: vmv.x.s t2, v16
+; CHECK-NEXT: vmv.x.s t3, v9
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: vmv.v.x v9, a1
+; CHECK-NEXT: vmv.v.x v10, a0
+; CHECK-NEXT: vslide1down.vx v8, v8, t3
+; CHECK-NEXT: vslide1down.vx v9, v9, t0
+; CHECK-NEXT: vslide1down.vx v10, v10, a5
+; CHECK-NEXT: vslide1down.vx v8, v8, t2
+; CHECK-NEXT: vslide1down.vx v11, v9, a7
+; CHECK-NEXT: vslide1down.vx v10, v10, a4
+; CHECK-NEXT: vslide1down.vx v9, v8, t1
+; CHECK-NEXT: vslide1down.vx v10, v10, a3
+; CHECK-NEXT: vslide1down.vx v8, v11, a6
+; CHECK-NEXT: vmv.v.i v11, 1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vsetivli zero, 2, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v9, v8, 2
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vslidedown.vi v9, v8, 1
+; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv6i32(<vscale x 6 x i32> %v)
+ ret i32 %red
+}
+
+define signext i31 @vreduce_mul_nxv8i31_exact_vlen(<vscale x 8 x i31> %v) vscale_range(2,2) {
+; RV32-LABEL: vreduce_mul_nxv8i31_exact_vlen:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset s0, -4
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v12, v11, 3
+; RV32-NEXT: vslidedown.vi v13, v11, 2
+; RV32-NEXT: vslidedown.vi v14, v11, 1
+; RV32-NEXT: vmv.x.s a0, v11
+; RV32-NEXT: vslidedown.vi v11, v10, 3
+; RV32-NEXT: vslidedown.vi v15, v10, 2
+; RV32-NEXT: vslidedown.vi v16, v10, 1
+; RV32-NEXT: vmv.x.s a1, v10
+; RV32-NEXT: vslidedown.vi v10, v9, 3
+; RV32-NEXT: vslidedown.vi v17, v9, 2
+; RV32-NEXT: vslidedown.vi v18, v9, 1
+; RV32-NEXT: vmv.x.s a2, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 3
+; RV32-NEXT: vslidedown.vi v19, v8, 2
+; RV32-NEXT: vmv.x.s a3, v8
+; RV32-NEXT: vslidedown.vi v8, v8, 1
+; RV32-NEXT: vmv.x.s a4, v12
+; RV32-NEXT: vmv.x.s a5, v13
+; RV32-NEXT: vmv.x.s a6, v14
+; RV32-NEXT: vmv.x.s a7, v11
+; RV32-NEXT: vmv.x.s t0, v15
+; RV32-NEXT: vmv.x.s t1, v16
+; RV32-NEXT: vmv.x.s t2, v10
+; RV32-NEXT: vmv.x.s t3, v17
+; RV32-NEXT: vmv.x.s t4, v18
+; RV32-NEXT: vmv.x.s t5, v9
+; RV32-NEXT: vmv.x.s t6, v19
+; RV32-NEXT: vmv.x.s s0, v8
+; RV32-NEXT: vmv.v.x v8, a3
+; RV32-NEXT: vmv.v.x v9, a2
+; RV32-NEXT: vmv.v.x v10, a1
+; RV32-NEXT: vmv.v.x v11, a0
+; RV32-NEXT: vslide1down.vx v8, v8, s0
+; RV32-NEXT: vslide1down.vx v9, v9, t4
+; RV32-NEXT: vslide1down.vx v10, v10, t1
+; RV32-NEXT: vslide1down.vx v11, v11, a6
+; RV32-NEXT: vslide1down.vx v8, v8, t6
+; RV32-NEXT: vslide1down.vx v9, v9, t3
+; RV32-NEXT: vslide1down.vx v10, v10, t0
+; RV32-NEXT: vslide1down.vx v11, v11, a5
+; RV32-NEXT: vslide1down.vx v8, v8, t5
+; RV32-NEXT: vslide1down.vx v9, v9, t2
+; RV32-NEXT: vslide1down.vx v10, v10, a7
+; RV32-NEXT: vslide1down.vx v11, v11, a4
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vslidedown.vi v9, v8, 2
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vrgather.vi v9, v8, 1
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmv.x.s a0, v8
+; RV32-NEXT: slli a0, a0, 1
+; RV32-NEXT: srai a0, a0, 1
+; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore s0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: vreduce_mul_nxv8i31_exact_vlen:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset s0, -8
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v12, v11, 3
+; RV64-NEXT: vslidedown.vi v13, v11, 2
+; RV64-NEXT: vslidedown.vi v14, v11, 1
+; RV64-NEXT: vmv.x.s a0, v11
+; RV64-NEXT: vslidedown.vi v11, v10, 3
+; RV64-NEXT: vslidedown.vi v15, v10, 2
+; RV64-NEXT: vslidedown.vi v16, v10, 1
+; RV64-NEXT: vmv.x.s a1, v10
+; RV64-NEXT: vslidedown.vi v10, v9, 3
+; RV64-NEXT: vslidedown.vi v17, v9, 2
+; RV64-NEXT: vslidedown.vi v18, v9, 1
+; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 3
+; RV64-NEXT: vslidedown.vi v19, v8, 2
+; RV64-NEXT: vmv.x.s a3, v8
+; RV64-NEXT: vslidedown.vi v8, v8, 1
+; RV64-NEXT: vmv.x.s a4, v12
+; RV64-NEXT: vmv.x.s a5, v13
+; RV64-NEXT: vmv.x.s a6, v14
+; RV64-NEXT: vmv.x.s a7, v11
+; RV64-NEXT: vmv.x.s t0, v15
+; RV64-NEXT: vmv.x.s t1, v16
+; RV64-NEXT: vmv.x.s t2, v10
+; RV64-NEXT: vmv.x.s t3, v17
+; RV64-NEXT: vmv.x.s t4, v18
+; RV64-NEXT: vmv.x.s t5, v9
+; RV64-NEXT: vmv.x.s t6, v19
+; RV64-NEXT: vmv.x.s s0, v8
+; RV64-NEXT: vmv.v.x v8, a3
+; RV64-NEXT: vmv.v.x v9, a2
+; RV64-NEXT: vmv.v.x v10, a1
+; RV64-NEXT: vmv.v.x v11, a0
+; RV64-NEXT: vslide1down.vx v8, v8, s0
+; RV64-NEXT: vslide1down.vx v9, v9, t4
+; RV64-NEXT: vslide1down.vx v10, v10, t1
+; RV64-NEXT: vslide1down.vx v11, v11, a6
+; RV64-NEXT: vslide1down.vx v8, v8, t6
+; RV64-NEXT: vslide1down.vx v9, v9, t3
+; RV64-NEXT: vslide1down.vx v10, v10, t0
+; RV64-NEXT: vslide1down.vx v11, v11, a5
+; RV64-NEXT: vslide1down.vx v8, v8, t5
+; RV64-NEXT: vslide1down.vx v9, v9, t2
+; RV64-NEXT: vslide1down.vx v10, v10, a7
+; RV64-NEXT: vslide1down.vx v11, v11, a4
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vslidedown.vi v9, v8, 2
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vrgather.vi v9, v8, 1
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmv.x.s a0, v8
+; RV64-NEXT: slli a0, a0, 33
+; RV64-NEXT: srai a0, a0, 33
+; RV64-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore s0
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_def_cfa_offset 0
+; RV64-NEXT: ret
+ %red = call i31 @llvm.vector.reduce.mul.nxv8i31(<vscale x 8 x i31> %v)
+ ret i31 %red
+}
+
+define signext i32 @vreduce_mul_nxv1i32_exact_vlen(<vscale x 1 x i32> %v) vscale_range(2,2) {
+; CHECK-LABEL: vreduce_mul_nxv1i32_exact_vlen:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT: vrgather.vi v9, v8, 1
+; CHECK-NEXT: vmul.vv v8, v8, v9
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: ret
+ %red = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> %v)
+ ret i32 %red
+}
More information about the llvm-commits
mailing list