[llvm] [LV] Reduce register pressure of RISC-V outer loop reduction. (PR #208621)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 22:12:49 PDT 2026
https://github.com/ElvisWang123 created https://github.com/llvm/llvm-project/pull/208621
For the outer loop reduction in RISC-V, LV will generate a `vp.merge`
after the inloop operation to prevent poison value in the tail.
But in after codegen, this kind of `vp.merge` can be folded to the `vsetvl`
with tail un-disturb policy. So it won't genetate any `vp.merge` for
this case.
Currently, when analysis the register pressure, this `vp.merge` make the
live range of the reduction-phi overlap the inloop operation, and make
the register pressure too high.
This patch removes the usage of this kind of `vp.merge` and updates the usage for the in-loop operand to make the register pressure smaller and close to real assembly.
The correct live range of the outer loop reduction should be
```
REDUCTION-PHI -> in-loop operation, in-loop operation -> REDUCTION-PHI.
```
Not
```
REDUCTION-PHI -> in-loop operation, in-loop operation -> vp.merge, vp.merge -> REDUCTION-PHI.
REDUCTION-PHI -> vp.merge.
```
The ASM output of the (vscale x 4) test case is https://godbolt.org/z/4ns9Pr5r3.
```
.LBB0_1: # %vector.body
add a5, a0, a3
vsetivli zero, 0, e64, m4, ta, ma
vlse64.v v24, (a5), a4
add a5, a1, a3
vlse64.v v28, (a5), a4
vsetvli zero, zero, e64, m4, tu, ma
vmin.vv v8, v24, v8
vmax.vv v16, v24, v16
vmin.vv v12, v28, v12
vmax.vv v20, v28, v20
vsetvli a5, a2, e8, mf2, ta, ma
sub a2, a2, a5
add a3, a5, a3
bnez a2, .LBB0_1
ret
```
>From d3e2c93fd9034d1bfb8cc7dd1fb49e853b365c2c Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 6 Jul 2026 23:25:43 -0700
Subject: [PATCH 1/2] Precommit tests.
---
.../LoopVectorize/RISCV/reg-usage.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index ccf989d80efde..226470b217f26 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -128,3 +128,54 @@ for.body:
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
}
+
+
+define void @red(ptr %base.0, ptr %base.1, i64 %end) {
+; CHECK-SCALAR-LABEL: red
+; CHECK-SCALAR: LV(REG): VF = 1
+; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-LMUL1-LABEL: 'red'
+; CHECK-LMUL1: LV(REG): VF = vscale x 1
+; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 9 registers
+; CHECK-LMUL2-LABEL: 'red'
+; CHECK-LMUL2: LV(REG): VF = vscale x 2
+; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 18 registers
+; CHECK-LMUL4-LABEL: 'red'
+; CHECK-LMUL4: LV(REG): VF = vscale x 4
+; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 36 registers
+; CHECK-LMUL8-LABEL: 'red'
+; CHECK-LMUL8: LV(REG): VF = vscale x 8
+; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 72 registers
+entry:
+ br label %loop.body
+
+loop.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]
+ %red.0 = phi i64 [ 0, %entry ], [ %red.0.next, %loop.body ]
+ %red.1 = phi i64 [ 0, %entry ], [ %red.1.next, %loop.body ]
+ %red.2 = phi i64 [ 0, %entry ], [ %red.2.next, %loop.body ]
+ %red.3 = phi i64 [ 0, %entry ], [ %red.3.next, %loop.body ]
+ %ptr.0 = getelementptr i8, ptr %base.0, i64 %iv
+ %0 = load i64, ptr %ptr.0, align 8
+ %ptr.1 = getelementptr i8, ptr %base.1, i64 %iv
+ %1 = load i64, ptr %ptr.1, align 8
+ %red.0.next = tail call i64 @llvm.smin.i64(i64 %0, i64 %red.0)
+ %red.2.next = tail call i64 @llvm.smax.i64(i64 %0, i64 %red.2)
+ %red.1.next = tail call i64 @llvm.smin.i64(i64 %1, i64 %red.1)
+ %red.3.next = tail call i64 @llvm.smax.i64(i64 %1, i64 %red.3)
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv, %end
+ br i1 %exitcond, label %loop.exit, label %loop.body
+
+loop.exit:
+ ret void
+}
>From 31be83ac6a346220c452b98c91aaaaf309288638 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 9 Jul 2026 21:11:15 -0700
Subject: [PATCH 2/2] [LV] Reduce register pressure of RISC-V outer loop
reduction.
For the outer loop reduction in RISC-V, it will generate a `vp.merge`
after the inloop operation to prevent poison value in the tail.
But in the codegen, this kind of `vp.merge` can be folded to the `vsetvl`
with tail un-disturb policy. So it won't genetate any `vp.merge` for
this case.
But when analysis the register pressure, this `vp.merge` make the
live range of the reduction-phi overlap the inloop operation, and make
the register pressure too high.
The correct live range of the outer loop reduction should be
REDUCTION-PHI <-> inloop operation, inloop operation <-> REDUCTION-PHI.
Not
REDUCTION-PHI <-> inloop operation, inloop operation <-> vp.merge, vp.merge <-> REDUCTION-PHI.
REDUCTION-PHI <-> vp.merge.
The ASM output of the test case is https://godbolt.org/z/4ns9Pr5r3.
```
.LBB0_1: # %vector.body
add a5, a0, a3
vsetivli zero, 0, e64, m4, ta, ma
vlse64.v v24, (a5), a4
add a5, a1, a3
vlse64.v v28, (a5), a4
vsetvli zero, zero, e64, m4, tu, ma
vmin.vv v8, v24, v8
vmax.vv v16, v24, v16
vmin.vv v12, v28, v12
vmax.vv v20, v28, v20
vsetvli a5, a2, e8, mf2, ta, ma
sub a2, a2, a5
add a3, a5, a3
bnez a2, .LBB0_1
ret
```
---
.../Transforms/Vectorize/VPlanAnalysis.cpp | 31 +++++++++++++++++++
.../LoopVectorize/RISCV/reg-usage.ll | 18 +++++------
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 77a33339eb5f9..66246d568524d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -127,6 +127,23 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
if (!Plan.getVectorTripCount().user_empty())
LoopInvariants.insert(&Plan.getVectorTripCount());
+ // For the vp_merge dealing with the tail poison for the outer loop
+ // reduction, it won't generate any assembly. Instead, it will be handled by
+ // the `vsetvl` with tail-undisturb policy.
+ auto MatchAndGetReductionTailPoisonMerge = [](VPRecipeBase *R) -> VPValue * {
+ auto *SingleR = dyn_cast<VPSingleDefRecipe>(R);
+ VPValue *InLoopOp, *RedPhiVal;
+ if (!SingleR || !match(SingleR, m_Intrinsic<Intrinsic::vp_merge>(
+ m_True(), m_VPValue(InLoopOp),
+ m_VPValue(RedPhiVal), m_VPValue())))
+ return nullptr;
+ auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(RedPhiVal);
+ // Only outer loop reduction will generate vp_merge to clean up tail poison.
+ if (!RedPhi || RedPhi->isInLoop() || RedPhi->getBackedgeValue() != SingleR)
+ return nullptr;
+ return InLoopOp;
+ };
+
// We scan the loop in a topological order in order and assign a number to
// each recipe. We use RPO to ensure that defs are met before their users. We
// assume that each recipe that has in-loop users starts an interval. We
@@ -141,8 +158,22 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
for (VPRecipeBase &R : *VPBB) {
Idx2Recipe.push_back(&R);
+ // For the outer loop reduction, don't add the live-range between
+ // ReductionPHI and the vp_merge since the vp_merge won't be generated in
+ // assembly. Also, not recording an end-of-use point for the in-loop
+ // operand lets its live range extend to the end of the block, matching
+ // the PHI backedge's behavior.
+ if (MatchAndGetReductionTailPoisonMerge(&R))
+ continue;
+
// Save the end location of each USE.
for (VPValue *U : R.operands()) {
+ // Since the vp_merge for tail poison won't be generated,
+ // update the Def of the user of the vp_merge to its true operand.
+ if (VPRecipeBase *Def = U->getDefiningRecipe())
+ if (VPValue *InLoopOp = MatchAndGetReductionTailPoisonMerge(Def))
+ U = InLoopOp;
+
if (isa<VPRecipeValue>(U)) {
// Overwrite previous end points.
EndPoint[U] = Idx2Recipe.size();
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 226470b217f26..76c4f7c902625 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -134,27 +134,27 @@ define void @red(ptr %base.0, ptr %base.1, i64 %end) {
; CHECK-SCALAR-LABEL: red
; CHECK-SCALAR: LV(REG): VF = 1
; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item
-; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 7 registers
; CHECK-LMUL1-LABEL: 'red'
; CHECK-LMUL1: LV(REG): VF = vscale x 1
; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
-; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 9 registers
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
; CHECK-LMUL2-LABEL: 'red'
; CHECK-LMUL2: LV(REG): VF = vscale x 2
; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
-; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 18 registers
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
; CHECK-LMUL4-LABEL: 'red'
; CHECK-LMUL4: LV(REG): VF = vscale x 4
; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
-; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 36 registers
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
; CHECK-LMUL8-LABEL: 'red'
; CHECK-LMUL8: LV(REG): VF = vscale x 8
; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
-; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 72 registers
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 48 registers
entry:
br label %loop.body
More information about the llvm-commits
mailing list