[llvm] [RISCV] Further improved exact VLEN lowering for mul reductions (PR #192688)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 08:54:52 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Philip Reames (preames)
<details>
<summary>Changes</summary>
This is a follow up to 973a05ed. When we have a horizontal multiply reduction at high LMUL and we have exact knowledge of VLEN, we can extract the individual m1 sub-vectors and perform the entire reduction tree at m1. This reduces the work performed (by not performing high LMUL operations on a vectors with empty tails), and decreases register pressure. Interestingly, we don't even increase dynamic instruction count as the register alignment of the original LMUL forced the use of whole register moves in the tree reduction anyways. (In the non-exact case, these are vslidedown instructions, and are required.)
Originally written by Claude Code, heavily revised by me.
---
Full diff: https://github.com/llvm/llvm-project/pull/192688.diff
2 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp (+36-15)
- (modified) llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll (+16-26)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
index 3880d0e68a764..061910b93c797 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp
@@ -288,25 +288,46 @@ bool RISCVCodeGenPrepare::expandMulReduction(IntrinsicInst &II) {
return false;
IRBuilder<> Builder(&II);
+ auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
- // Shuffle-reduce at the original vector width. This just duplicates the
- // default lowering down to m1.
- SmallVector<int, 32> ShuffleMask(VF);
- for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
- unsigned Half = LiveElts / 2;
- std::iota(ShuffleMask.begin(), ShuffleMask.begin() + Half, Half);
- std::fill(ShuffleMask.begin() + Half, ShuffleMask.end(), -1);
- Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
- TmpVec = Builder.CreateMul(TmpVec, Shuf, "bin.rdx");
+ // When VLEN is exactly known, extract m1 pieces and build a mul tree.
+ // This greatly reduces register pressure during the reduction, and
+ // avoids all but one vsetvli (the one from original LMUL to m1).
+ if (MinVLen == ST->getRealMaxVLen()) {
+ unsigned NumM1 = VF / M1VF;
+ assert(isPowerOf2_32(NumM1) && NumM1 <= 8);
+ SmallVector<Value *, 8> Pieces(NumM1);
+ for (unsigned i = 0; i < NumM1; i++)
+ Pieces[i] = Builder.CreateExtractVector(M1Ty, TmpVec,
+ (uint64_t)(i * M1VF));
+
+ while (Pieces.size() > 1) {
+ for (unsigned i = 0; i < Pieces.size() / 2; i++)
+ Pieces[i] = Builder.CreateMul(Pieces[i * 2], Pieces[i * 2 + 1],
+ "bin.rdx");
+ Pieces.truncate(Pieces.size() / 2);
+ }
+ TmpVec = Pieces[0];
+ } else {
+ // For non-exact VLEN, shuffle-reduce at the original vector width down to
+ // m1, then extract. This prioritizes reducing the number of vsetvli
+ // over maximual reduction of LMUL for the intermediate states.
+ SmallVector<int, 32> ShuffleMask(VF);
+ for (unsigned LiveElts = VF; LiveElts > M1VF; LiveElts /= 2) {
+ unsigned Half = LiveElts / 2;
+ std::iota(ShuffleMask.begin(), ShuffleMask.begin() + Half, Half);
+ std::fill(ShuffleMask.begin() + Half, ShuffleMask.end(), -1);
+ Value *Shuf =
+ Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
+ TmpVec = Builder.CreateMul(TmpVec, Shuf, "bin.rdx");
+ }
+ // Extract the M1-sized subvector and emit the final reduction intrinsic.
+ // This is the reason we're here - to force a vsetvli toggle once at m1.
+ TmpVec = Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)0, "rdx.sub");
}
- // Extract the M1-sized subvector and emit the final reduction intrinsic.
- // This is the reason we're here - to force a vsetvli toggle once at m1.
- auto *M1Ty = FixedVectorType::get(VecTy->getElementType(), M1VF);
- Value *Sub =
- Builder.CreateExtractVector(M1Ty, TmpVec, (uint64_t)0, "rdx.sub");
Value *Rdx =
- Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {Sub});
+ Builder.CreateIntrinsic(Intrinsic::vector_reduce_mul, {M1Ty}, {TmpVec});
II.replaceAllUsesWith(Rdx);
II.eraseFromParent();
return true;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
index d63e7023d2e50..c2b1733d9e048 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int.ll
@@ -6232,18 +6232,15 @@ define i64 @vreduce_mul_v32i64_vscale4_exact(ptr %x) nounwind vscale_range(4,4)
; RV32-LABEL: vreduce_mul_v32i64_vscale4_exact:
; RV32: # %bb.0:
; RV32-NEXT: vl8re64.v v8, (a0)
-; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, ma
-; RV32-NEXT: vmv1r.v v16, v12
-; RV32-NEXT: vmv1r.v v17, v13
-; RV32-NEXT: vmv1r.v v18, v14
-; RV32-NEXT: vmv1r.v v19, v15
-; RV32-NEXT: vmul.vv v8, v8, v16
-; RV32-NEXT: vmv1r.v v16, v10
; RV32-NEXT: li a1, 32
-; RV32-NEXT: vmv1r.v v17, v11
-; RV32-NEXT: vmul.vv v8, v8, v16
; RV32-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v10, v12, v13
+; RV32-NEXT: vmul.vv v11, v14, v15
+; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vslidedown.vi v9, v8, 2
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vrgather.vi v9, v8, 1
@@ -6257,17 +6254,14 @@ define i64 @vreduce_mul_v32i64_vscale4_exact(ptr %x) nounwind vscale_range(4,4)
; RV64-LABEL: vreduce_mul_v32i64_vscale4_exact:
; RV64: # %bb.0:
; RV64-NEXT: vl8re64.v v8, (a0)
-; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, ma
-; RV64-NEXT: vmv1r.v v16, v12
-; RV64-NEXT: vmv1r.v v17, v13
-; RV64-NEXT: vmv1r.v v18, v14
-; RV64-NEXT: vmv1r.v v19, v15
-; RV64-NEXT: vmul.vv v8, v8, v16
-; RV64-NEXT: vmv1r.v v16, v10
-; RV64-NEXT: vmv1r.v v17, v11
-; RV64-NEXT: vmul.vv v8, v8, v16
; RV64-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v10, v12, v13
+; RV64-NEXT: vmul.vv v11, v14, v15
+; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vslidedown.vi v9, v8, 2
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vrgather.vi v9, v8, 1
@@ -6283,13 +6277,11 @@ define i64 @vreduce_mul_v32i64_vscale8_exact(ptr %x) nounwind vscale_range(8,8)
; RV32-LABEL: vreduce_mul_v32i64_vscale8_exact:
; RV32: # %bb.0:
; RV32-NEXT: vl4re64.v v8, (a0)
-; RV32-NEXT: vsetvli a0, zero, e64, m4, ta, ma
-; RV32-NEXT: vmv1r.v v12, v10
; RV32-NEXT: li a1, 32
-; RV32-NEXT: vmv1r.v v13, v11
-; RV32-NEXT: vmul.vv v8, v8, v12
; RV32-NEXT: vsetivli zero, 8, e64, m1, ta, ma
; RV32-NEXT: vmul.vv v8, v8, v9
+; RV32-NEXT: vmul.vv v9, v10, v11
+; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vslidedown.vi v9, v8, 4
; RV32-NEXT: vmul.vv v8, v8, v9
; RV32-NEXT: vslidedown.vi v9, v8, 2
@@ -6305,12 +6297,10 @@ define i64 @vreduce_mul_v32i64_vscale8_exact(ptr %x) nounwind vscale_range(8,8)
; RV64-LABEL: vreduce_mul_v32i64_vscale8_exact:
; RV64: # %bb.0:
; RV64-NEXT: vl4re64.v v8, (a0)
-; RV64-NEXT: vsetvli a0, zero, e64, m4, ta, ma
-; RV64-NEXT: vmv1r.v v12, v10
-; RV64-NEXT: vmv1r.v v13, v11
-; RV64-NEXT: vmul.vv v8, v8, v12
; RV64-NEXT: vsetivli zero, 8, e64, m1, ta, ma
; RV64-NEXT: vmul.vv v8, v8, v9
+; RV64-NEXT: vmul.vv v9, v10, v11
+; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vslidedown.vi v9, v8, 4
; RV64-NEXT: vmul.vv v8, v8, v9
; RV64-NEXT: vslidedown.vi v9, v8, 2
``````````
</details>
https://github.com/llvm/llvm-project/pull/192688
More information about the llvm-commits
mailing list