[llvm] [LV] Reduce register pressure of RISC-V outer loop reduction. (PR #208621)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 22:13:24 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Elvis Wang (ElvisWang123)
<details>
<summary>Changes</summary>
For the outer loop reduction in RISC-V, LV will generate a `vp.merge`
after the inloop operation to prevent poison value in the tail.
But in after codegen, this kind of `vp.merge` can be folded to the `vsetvl`
with tail un-disturb policy. So it won't genetate any `vp.merge` for
this case.
Currently, when analysis the register pressure, this `vp.merge` make the
live range of the reduction-phi overlap the inloop operation, and make
the register pressure too high.
This patch removes the usage of this kind of `vp.merge` and updates the usage for the in-loop operand to make the register pressure smaller and close to real assembly.
The correct live range of the outer loop reduction should be
```
REDUCTION-PHI -> in-loop operation, in-loop operation -> REDUCTION-PHI.
```
Not
```
REDUCTION-PHI -> in-loop operation, in-loop operation -> vp.merge, vp.merge -> REDUCTION-PHI.
REDUCTION-PHI -> vp.merge.
```
The ASM output of the (vscale x 4) test case is https://godbolt.org/z/4ns9Pr5r3.
```
.LBB0_1: # %vector.body
add a5, a0, a3
vsetivli zero, 0, e64, m4, ta, ma
vlse64.v v24, (a5), a4
add a5, a1, a3
vlse64.v v28, (a5), a4
vsetvli zero, zero, e64, m4, tu, ma
vmin.vv v8, v24, v8
vmax.vv v16, v24, v16
vmin.vv v12, v28, v12
vmax.vv v20, v28, v20
vsetvli a5, a2, e8, mf2, ta, ma
sub a2, a2, a5
add a3, a5, a3
bnez a2, .LBB0_1
ret
```
---
Full diff: https://github.com/llvm/llvm-project/pull/208621.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp (+31)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll (+51)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 77a33339eb5f9..66246d568524d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -127,6 +127,23 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
if (!Plan.getVectorTripCount().user_empty())
LoopInvariants.insert(&Plan.getVectorTripCount());
+ // For the vp_merge dealing with the tail poison for the outer loop
+ // reduction, it won't generate any assembly. Instead, it will be handled by
+ // the `vsetvl` with tail-undisturb policy.
+ auto MatchAndGetReductionTailPoisonMerge = [](VPRecipeBase *R) -> VPValue * {
+ auto *SingleR = dyn_cast<VPSingleDefRecipe>(R);
+ VPValue *InLoopOp, *RedPhiVal;
+ if (!SingleR || !match(SingleR, m_Intrinsic<Intrinsic::vp_merge>(
+ m_True(), m_VPValue(InLoopOp),
+ m_VPValue(RedPhiVal), m_VPValue())))
+ return nullptr;
+ auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(RedPhiVal);
+ // Only outer loop reduction will generate vp_merge to clean up tail poison.
+ if (!RedPhi || RedPhi->isInLoop() || RedPhi->getBackedgeValue() != SingleR)
+ return nullptr;
+ return InLoopOp;
+ };
+
// We scan the loop in a topological order in order and assign a number to
// each recipe. We use RPO to ensure that defs are met before their users. We
// assume that each recipe that has in-loop users starts an interval. We
@@ -141,8 +158,22 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
for (VPRecipeBase &R : *VPBB) {
Idx2Recipe.push_back(&R);
+ // For the outer loop reduction, don't add the live-range between
+ // ReductionPHI and the vp_merge since the vp_merge won't be generated in
+ // assembly. Also, not recording an end-of-use point for the in-loop
+ // operand lets its live range extend to the end of the block, matching
+ // the PHI backedge's behavior.
+ if (MatchAndGetReductionTailPoisonMerge(&R))
+ continue;
+
// Save the end location of each USE.
for (VPValue *U : R.operands()) {
+ // Since the vp_merge for tail poison won't be generated,
+ // update the Def of the user of the vp_merge to its true operand.
+ if (VPRecipeBase *Def = U->getDefiningRecipe())
+ if (VPValue *InLoopOp = MatchAndGetReductionTailPoisonMerge(Def))
+ U = InLoopOp;
+
if (isa<VPRecipeValue>(U)) {
// Overwrite previous end points.
EndPoint[U] = Idx2Recipe.size();
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index ccf989d80efde..76c4f7c902625 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -128,3 +128,54 @@ for.body:
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
}
+
+
+define void @red(ptr %base.0, ptr %base.1, i64 %end) {
+; CHECK-SCALAR-LABEL: red
+; CHECK-SCALAR: LV(REG): VF = 1
+; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 7 registers
+; CHECK-LMUL1-LABEL: 'red'
+; CHECK-LMUL1: LV(REG): VF = vscale x 1
+; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; CHECK-LMUL2-LABEL: 'red'
+; CHECK-LMUL2: LV(REG): VF = vscale x 2
+; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; CHECK-LMUL4-LABEL: 'red'
+; CHECK-LMUL4: LV(REG): VF = vscale x 4
+; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; CHECK-LMUL8-LABEL: 'red'
+; CHECK-LMUL8: LV(REG): VF = vscale x 8
+; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 48 registers
+entry:
+ br label %loop.body
+
+loop.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]
+ %red.0 = phi i64 [ 0, %entry ], [ %red.0.next, %loop.body ]
+ %red.1 = phi i64 [ 0, %entry ], [ %red.1.next, %loop.body ]
+ %red.2 = phi i64 [ 0, %entry ], [ %red.2.next, %loop.body ]
+ %red.3 = phi i64 [ 0, %entry ], [ %red.3.next, %loop.body ]
+ %ptr.0 = getelementptr i8, ptr %base.0, i64 %iv
+ %0 = load i64, ptr %ptr.0, align 8
+ %ptr.1 = getelementptr i8, ptr %base.1, i64 %iv
+ %1 = load i64, ptr %ptr.1, align 8
+ %red.0.next = tail call i64 @llvm.smin.i64(i64 %0, i64 %red.0)
+ %red.2.next = tail call i64 @llvm.smax.i64(i64 %0, i64 %red.2)
+ %red.1.next = tail call i64 @llvm.smin.i64(i64 %1, i64 %red.1)
+ %red.3.next = tail call i64 @llvm.smax.i64(i64 %1, i64 %red.3)
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv, %end
+ br i1 %exitcond, label %loop.exit, label %loop.body
+
+loop.exit:
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/208621
More information about the llvm-commits
mailing list