[llvm] 9aebd4b - [RISCV] Run EarlyMachineLICM before VLOptimizer to hoist constant splats (#210028)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 23:33:21 PDT 2026
Author: Luke Lau
Date: 2026-07-21T06:33:16Z
New Revision: 9aebd4b405d7299337d4561de8d35c579ab5d88b
URL: https://github.com/llvm/llvm-project/commit/9aebd4b405d7299337d4561de8d35c579ab5d88b
DIFF: https://github.com/llvm/llvm-project/commit/9aebd4b405d7299337d4561de8d35c579ab5d88b.diff
LOG: [RISCV] Run EarlyMachineLICM before VLOptimizer to hoist constant splats (#210028)
RISCVVLOptimizer runs before the machine SSA optimization passes. This
includes EarlyMachineLICM, which causes previously loop invariant
vmv.v.i/vmv.v.x/vfmv.v.f splats to use a loop variant VL defined by a
PseudoVSETVLI. This prevents splats from being hoisted out:
https://godbolt.org/z/Wf49roo7s
This fixes it by adding another pass of EarlyMachineLICM before
RISCVVLOptimizer.
As measured on llvm-test-suite w/ `-march=rva23u64 -O3` this increases
the number of vector splat instructions hoisted by MachineLICM by 4.4x,
from 2140 to 9327.
Compile time wise, the number of instructions executed increases geomean
+0.2% on CTMark for -O3. -O0 builds are unaffected.
Two other approaches were considered:
- Use MachineLoopInfo in RISCVVLOptimizer to avoid changing the VL of
any loop invariant pseudos. But that results in more vsetvli toggles,
and still involves running the same analysis that EarlyMachineLICM
depends on anyway
- Hoist loop invariant pseudos ourselves in RISCVVLOptimizer. But in
order to catch all cases that EarlyMachineLICM handles, we need to split
critical edges and modify the CFG (which would invalidate the CFG
analyses), and also hoist scalar non-pseudo instructions. In some cases
it also led to more spilling due to increased register pressure, which
EarlyMachineLICM has heuristics for avoiding.
Added:
llvm/test/CodeGen/RISCV/rvv/vl-opt-licm.ll
Modified:
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
llvm/test/CodeGen/RISCV/O3-pipeline.ll
llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 88bc7ae9c1994..8cf29ca900faa 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -623,8 +623,16 @@ void RISCVPassConfig::addPreEmitPass2() {
void RISCVPassConfig::addMachineSSAOptimization() {
// It's beneficial to reduce the VL to enable more
// Machine SSA optimizations.
- if (TM->getOptLevel() != CodeGenOptLevel::None)
+ if (TM->getOptLevel() != CodeGenOptLevel::None) {
+ // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i
+ // loop variant by propagating a VL defined inside the loop. Run LICM and
+ // hoist them early. Don't do this at -O0 to avoid the compile-time
+ // overhead. Not reducing the VL of loop invariant pseudos results in more
+ // vsetvli toggles, and still requires the MachineLoopInfo analysis to be
+ // run.
+ addPass(&EarlyMachineLICMID);
addPass(createRISCVVLOptimizerPass());
+ }
addPass(createRISCVVectorPeepholePass());
addPass(createRISCVFoldMemOffsetPass());
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 12b0895e0f158..c963236e5bb4c 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -106,6 +106,10 @@
; CHECK-NEXT: RISC-V DAG->DAG Pattern Instruction Selection
; CHECK-NEXT: Finalize ISel and expand pseudo-instructions
; CHECK-NEXT: MachineDominator Tree Construction
+; CHECK-NEXT: Machine Natural Loop Construction
+; CHECK-NEXT: Machine Block Frequency Analysis
+; CHECK-NEXT: Early Machine Loop Invariant Code Motion
+; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: RISC-V VL Optimizer
; CHECK-NEXT: RISC-V Vector Peephole Optimization
; CHECK-NEXT: RISC-V Fold Memory Offset
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-licm.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-licm.ll
new file mode 100644
index 0000000000000..9d78449b84539
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-licm.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple riscv64 -mattr=+v,+d | FileCheck %s
+
+define void @hoist_const_splats(ptr %p, i32 zeroext %n) {
+; CHECK-LABEL: hoist_const_splats:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vsetvli a2, zero, e32, m2, ta, ma
+; CHECK-NEXT: vmv.v.i v8, 1
+; CHECK-NEXT: li a2, 0
+; CHECK-NEXT: .LBB0_1: # %loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vsetvli a3, a1, e8, mf2, ta, ma
+; CHECK-NEXT: slli a4, a2, 2
+; CHECK-NEXT: add a4, a0, a4
+; CHECK-NEXT: sub a1, a1, a3
+; CHECK-NEXT: addw a2, a2, a3
+; CHECK-NEXT: vse32.v v8, (a4)
+; CHECK-NEXT: bnez a1, .LBB0_1
+; CHECK-NEXT: # %bb.2: # %exit
+; CHECK-NEXT: ret
+entry:
+ %n.zext = zext i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %avl = phi i64 [%n.zext, %entry], [%avl.next, %loop]
+ %evl = call i32 @llvm.experimental.get.vector.length(i64 %avl, i32 4, i1 true)
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ call void @llvm.vp.store(<vscale x 4 x i32> splat (i32 1), ptr %gep, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+ %iv.next = add i32 %iv, %evl
+ %evl.zext = zext i32 %evl to i64
+ %avl.next = sub i64 %avl, %evl.zext
+ %ec = icmp eq i64
+ %avl.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @hoist_const_vmv_v_x(ptr %p, i32 zeroext %n) {
+; CHECK-LABEL: hoist_const_vmv_v_x:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: li a2, 16
+; CHECK-NEXT: vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: li a2, 0
+; CHECK-NEXT: .LBB1_1: # %loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vsetvli a3, a1, e8, mf4, ta, ma
+; CHECK-NEXT: slli a4, a2, 2
+; CHECK-NEXT: add a4, a0, a4
+; CHECK-NEXT: sub a1, a1, a3
+; CHECK-NEXT: vse32.v v8, (a4)
+; CHECK-NEXT: addw a2, a2, a3
+; CHECK-NEXT: bnez a1, .LBB1_1
+; CHECK-NEXT: # %bb.2: # %exit
+; CHECK-NEXT: ret
+entry:
+ %n.zext = zext i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %avl = phi i64 [%n.zext, %entry], [%avl.next, %loop]
+ %evl = call i32 @llvm.experimental.get.vector.length(i64 %avl, i32 2, i1 true)
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ call void @llvm.vp.store(<vscale x 2 x i32> splat (i32 16), ptr %gep, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %iv.next = add i32 %iv, %evl
+ %evl.zext = zext i32 %evl to i64
+ %avl.next = sub i64 %avl, %evl.zext
+ %ec = icmp eq i64
+ %avl.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @hoist_const_vfmv_v_f(ptr %p, i32 zeroext %n) {
+; CHECK-LABEL: hoist_const_vfmv_v_f:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: lui a2, %hi(.LCPI2_0)
+; CHECK-NEXT: fld fa5, %lo(.LCPI2_0)(a2)
+; CHECK-NEXT: vsetvli a2, zero, e64, m1, ta, ma
+; CHECK-NEXT: vfmv.v.f v8, fa5
+; CHECK-NEXT: li a2, 0
+; CHECK-NEXT: .LBB2_1: # %loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vsetvli a3, a1, e8, mf8, ta, ma
+; CHECK-NEXT: slli a4, a2, 3
+; CHECK-NEXT: add a4, a0, a4
+; CHECK-NEXT: sub a1, a1, a3
+; CHECK-NEXT: vse64.v v8, (a4)
+; CHECK-NEXT: addw a2, a2, a3
+; CHECK-NEXT: bnez a1, .LBB2_1
+; CHECK-NEXT: # %bb.2: # %exit
+; CHECK-NEXT: ret
+entry:
+ %n.zext = zext i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %avl = phi i64 [%n.zext, %entry], [%avl.next, %loop]
+ %evl = call i32 @llvm.experimental.get.vector.length(i64 %avl, i32 1, i1 true)
+ %gep = getelementptr double, ptr %p, i32 %iv
+ call void @llvm.vp.store(<vscale x 1 x double> splat (double 1.23), ptr %gep, <vscale x 1 x i1> splat (i1 true), i32 %evl)
+ %iv.next = add i32 %iv, %evl
+ %evl.zext = zext i32 %evl to i64
+ %avl.next = sub i64 %avl, %evl.zext
+ %ec = icmp eq i64
+ %avl.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 0e767cff76fe9..c1a7435849221 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -821,20 +821,21 @@ define void @coalesce_vl_clobber(ptr %p) {
; CHECK-NEXT: vmv.v.i v9, 0
; CHECK-NEXT: vmv1r.v v0, v8
; CHECK-NEXT: vmerge.vim v9, v9, 1, v0
+; CHECK-NEXT: vsetvli a2, zero, e32, m2, ta, ma
+; CHECK-NEXT: vmv.v.i v10, 0
; CHECK-NEXT: li a2, 0
; CHECK-NEXT: .LBB43_1: # %vector.body
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vsetivli zero, 0, e8, mf2, ta, mu
; CHECK-NEXT: vmv1r.v v0, v8
-; CHECK-NEXT: vmv1r.v v10, v9
+; CHECK-NEXT: vmv1r.v v12, v9
; CHECK-NEXT: slli a2, a2, 32
; CHECK-NEXT: srli a2, a2, 32
-; CHECK-NEXT: vslideup.vx v10, v9, a2, v0.t
+; CHECK-NEXT: vslideup.vx v12, v9, a2, v0.t
; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma
-; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t
+; CHECK-NEXT: vmsne.vi v0, v12, 0, v0.t
; CHECK-NEXT: vsetvli a2, a1, e8, mf8, ta, ma
; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma
-; CHECK-NEXT: vmv.v.i v10, 0
; CHECK-NEXT: vse32.v v10, (a0), v0.t
; CHECK-NEXT: li a1, 1
; CHECK-NEXT: j .LBB43_1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
index d164861aa3e0d..cba670e62a79a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
@@ -196,11 +196,11 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64P670-NEXT: or t4, t4, t5
; RV64P670-NEXT: li t5, 32
; RV64P670-NEXT: maxu t5, t3, t5
-; RV64P670-NEXT: or t6, t6, t4
-; RV64P670-NEXT: add t4, a0, a6
+; RV64P670-NEXT: or t4, t6, t4
; RV64P670-NEXT: sltu t5, a6, t5
-; RV64P670-NEXT: or t5, t5, t6
-; RV64P670-NEXT: andi t5, t5, 1
+; RV64P670-NEXT: or t4, t5, t4
+; RV64P670-NEXT: add t5, a0, a6
+; RV64P670-NEXT: andi t4, t4, 1
; RV64P670-NEXT: j .LBB0_4
; RV64P670-NEXT: .LBB0_3: # %for.cond1.for.cond.cleanup3_crit_edge.us
; RV64P670-NEXT: # in Loop: Header=BB0_4 Depth=1
@@ -214,7 +214,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64P670-NEXT: # =>This Loop Header: Depth=1
; RV64P670-NEXT: # Child Loop BB0_7 Depth 2
; RV64P670-NEXT: # Child Loop BB0_10 Depth 2
-; RV64P670-NEXT: beqz t5, .LBB0_6
+; RV64P670-NEXT: beqz t4, .LBB0_6
; RV64P670-NEXT: # %bb.5: # in Loop: Header=BB0_4 Depth=1
; RV64P670-NEXT: li t6, 0
; RV64P670-NEXT: j .LBB0_9
@@ -249,7 +249,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64P670-NEXT: add s1, a0, t6
; RV64P670-NEXT: add s2, a4, t6
; RV64P670-NEXT: add t6, t6, a2
-; RV64P670-NEXT: add s0, s0, t4
+; RV64P670-NEXT: add s0, s0, t5
; RV64P670-NEXT: .LBB0_10: # %for.body4.us
; RV64P670-NEXT: # Parent Loop BB0_4 Depth=1
; RV64P670-NEXT: # => This Inner Loop Header: Depth=2
@@ -330,9 +330,9 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64X60-NEXT: maxu t6, t3, s0
; RV64X60-NEXT: or t4, t4, t5
; RV64X60-NEXT: sltu t5, a6, t6
-; RV64X60-NEXT: or t5, t5, t4
-; RV64X60-NEXT: add t4, a0, a6
-; RV64X60-NEXT: andi t5, t5, 1
+; RV64X60-NEXT: or t4, t5, t4
+; RV64X60-NEXT: andi t4, t4, 1
+; RV64X60-NEXT: add t5, a0, a6
; RV64X60-NEXT: j .LBB0_4
; RV64X60-NEXT: .LBB0_3: # %for.cond1.for.cond.cleanup3_crit_edge.us
; RV64X60-NEXT: # in Loop: Header=BB0_4 Depth=1
@@ -346,7 +346,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64X60-NEXT: # =>This Loop Header: Depth=1
; RV64X60-NEXT: # Child Loop BB0_7 Depth 2
; RV64X60-NEXT: # Child Loop BB0_10 Depth 2
-; RV64X60-NEXT: beqz t5, .LBB0_6
+; RV64X60-NEXT: beqz t4, .LBB0_6
; RV64X60-NEXT: # %bb.5: # in Loop: Header=BB0_4 Depth=1
; RV64X60-NEXT: li t6, 0
; RV64X60-NEXT: j .LBB0_9
@@ -379,7 +379,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64X60-NEXT: # in Loop: Header=BB0_4 Depth=1
; RV64X60-NEXT: mul s1, a1, t0
; RV64X60-NEXT: add s0, a0, t6
-; RV64X60-NEXT: add s1, s1, t4
+; RV64X60-NEXT: add s1, s1, t5
; RV64X60-NEXT: add s2, a4, t6
; RV64X60-NEXT: add t6, t6, a2
; RV64X60-NEXT: .LBB0_10: # %for.body4.us
@@ -466,9 +466,9 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64-NEXT: or t5, t5, t6
; RV64-NEXT: or t5, s0, t5
; RV64-NEXT: sltu t4, a6, t4
-; RV64-NEXT: or t5, t4, t5
-; RV64-NEXT: add t4, a0, a6
-; RV64-NEXT: andi t5, t5, 1
+; RV64-NEXT: or t4, t4, t5
+; RV64-NEXT: andi t4, t4, 1
+; RV64-NEXT: add t5, a0, a6
; RV64-NEXT: csrwi vxrm, 0
; RV64-NEXT: j .LBB0_6
; RV64-NEXT: .LBB0_5: # %for.cond1.for.cond.cleanup3_crit_edge.us
@@ -483,7 +483,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64-NEXT: # =>This Loop Header: Depth=1
; RV64-NEXT: # Child Loop BB0_9 Depth 2
; RV64-NEXT: # Child Loop BB0_12 Depth 2
-; RV64-NEXT: beqz t5, .LBB0_8
+; RV64-NEXT: beqz t4, .LBB0_8
; RV64-NEXT: # %bb.7: # in Loop: Header=BB0_6 Depth=1
; RV64-NEXT: li t6, 0
; RV64-NEXT: j .LBB0_11
@@ -516,7 +516,7 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64-NEXT: # in Loop: Header=BB0_6 Depth=1
; RV64-NEXT: mul s1, a1, t2
; RV64-NEXT: add s0, a0, t6
-; RV64-NEXT: add s1, t4, s1
+; RV64-NEXT: add s1, t5, s1
; RV64-NEXT: add s2, a4, t6
; RV64-NEXT: add t6, a2, t6
; RV64-NEXT: .LBB0_12: # %for.body4.us
More information about the llvm-commits
mailing list