[llvm] [PassBuilder] Remove LoopFullUnrollPass from before LoopVectorize in the FullLTO pipeline (PR #197176)
Momchil Velikov via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 9 02:40:29 PDT 2026
https://github.com/momchil-velikov updated https://github.com/llvm/llvm-project/pull/197176
>From d29583e92ad6d25fefa56e849df00a7a09b0f9b5 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Mon, 11 May 2026 13:02:03 +0000
Subject: [PATCH 1/3] [PassBuilder] Move LoopFullUnrollPass to after
vectorization in the full LTO pipeline
This patch is in preparation for disabling the pre-LTO vectorization
in the FullLTO pipeline.
It moves `LoopFullUnrollPass` after `LoopVectorizePass` in
the full LTO pipeline, replacing the `LoopUnrollPass` that was
previously there.
Running the `LoopFullUnrollPass` before `LoopVectorizePass` causes
functions to be unrolled before the vectoriser has had a chance
to vectorize them (in the modified pipeline where
vectorisation does not happen in the pre-link stage).
---
llvm/lib/Passes/PassBuilderPipelines.cpp | 21 ++++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 37a429138c2cf2..75789d96e590c3 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1376,6 +1376,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(InferAlignmentPass());
if (isFullLTOPostLink(LTOPhase)) {
+ LoopPassManager LPM;
// The vectorizer may have significantly shortened a loop body; unroll
// again. Unroll small loops to hide loop backedge latency and saturate any
// parallel execution resources of an out-of-order processor. We also then
@@ -1385,11 +1386,15 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// across the loop nests.
// We do UnrollAndJam in a separate LPM to ensure it happens before unroll
if (EnableUnrollAndJam && PTO.LoopUnrolling)
- FPM.addPass(createFunctionToLoopPassAdaptor(
- LoopUnrollAndJamPass(static_cast<int>(Level))));
- FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
- static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
- PTO.ForgetAllSCEVInLoopUnroll)));
+ LPM.addPass(LoopUnrollAndJamPass(static_cast<int>(Level)));
+
+ // Unroll small loops and perform peeling.
+ LPM.addPass(LoopFullUnrollPass(static_cast<int>(Level),
+ /* OnlyWhenForced= */ !PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll));
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM),
+ /*UseMemorySSA=*/false));
+
FPM.addPass(WarnMissedTransformationsPass());
// Now that we are done with loop unrolling, be it either by LoopVectorizer,
// or LoopUnroll passes, some variable-offset GEP's into alloca's could have
@@ -2317,12 +2322,10 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
LPM.addPass(LoopDeletionPass());
// FIXME: Add loop interchange.
- // Unroll small loops and perform peeling.
- LPM.addPass(LoopFullUnrollPass(static_cast<int>(Level),
- /* OnlyWhenForced= */ !PTO.LoopUnrolling,
- PTO.ForgetAllSCEVInLoopUnroll));
// The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
// *All* loop passes must preserve it, in order to be able to use it.
+ // TODO: Explore enabling MemorySSA for loop passes after we remove the
+ // LoopFullUnrollPass.
MainFPM.addPass(
createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false));
>From 6560444c6a7204b141e7d928f9d8b7beddb86ec8 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 12 May 2026 13:02:42 +0100
Subject: [PATCH 2/3] [fixup] Update tests for new LTO defaults
---
llvm/test/Other/new-pm-lto-defaults.ll | 5 ++-
.../PhaseOrdering/loop-vectorize-bfi.ll | 38 ++++++++++---------
2 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 97bc3e5ce758af..af771698221b43 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -125,14 +125,15 @@
; CHECK-O23-NEXT: Running pass: LCSSAPass on foo
; CHECK-O23-NEXT: Running pass: IndVarSimplifyPass on loop
; CHECK-O23-NEXT: Running pass: LoopDeletionPass on loop
-; CHECK-O23-NEXT: Running pass: LoopFullUnrollPass on loop
; CHECK-O23-NEXT: Running pass: LoopDistributePass on foo
; CHECK-O23-NEXT: Running analysis: LoopAccessAnalysis on foo
; CHECK-O23-NEXT: Running pass: LoopVectorizePass on foo
; CHECK-O23-NEXT: Running analysis: DemandedBitsAnalysis on foo
; CHECK-O23-NEXT: Running pass: DropUnnecessaryAssumesPass on foo
; CHECK-O23-NEXT: Running pass: InferAlignmentPass on foo
-; CHECK-O23-NEXT: Running pass: LoopUnrollPass on foo
+; CHECK-O23-NEXT: Running pass: LoopSimplifyPass on foo
+; CHECK-O23-NEXT: Running pass: LCSSAPass on foo
+; CHECK-O23-NEXT: Running pass: LoopFullUnrollPass on loop
; CHECK-O23-NEXT: WarnMissedTransformationsPass on foo
; CHECK-O23-NEXT: Running pass: SROAPass on foo
; CHECK-O23-NEXT: Running pass: InstCombinePass on foo
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
index 1693b232b4e2aa..c459c9dbe52c1d 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; REQUIRES: riscv-registered-target
-; RUN: opt -p 'lto<O3>' -mtriple riscv64 -mattr=+v -S < %s | FileCheck %s
+; RUN: opt -p 'loop-unroll,loop-vectorize' -mtriple riscv64 -mattr=+v -S < %s | FileCheck %s
; Test that BlockFrequencyInfo is invalidated after loop passes, so it's
; accurate whenever LoopVectorize uses it. LoopVectorizer requires that
@@ -9,25 +9,29 @@
define void @f(i1 %x) vscale_range(2, 1024) !prof !0 {
; CHECK-LABEL: define void @f(
-; CHECK-SAME: i1 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {{.*}}{
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[DOTSCALAR:%.*]] = xor i1 [[X]], true
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[DOTSCALAR]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR0:[0-9]+]] {{.*}}{
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 65, %[[ENTRY]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [8 x i8], ptr null, i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = zext nneg i32 [[TMP3]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[EVL_BASED_IV]], [[TMP5]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw nsw i64 [[AVL]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 65, %[[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr null, i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> zeroinitializer, ptr align 8 [[TMP2]], <vscale x 2 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> zeroinitializer, ptr align 8 [[TMP2]], <vscale x 2 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[EXIT1:.*]], label %[[EXIT]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: br label %[[EXIT2:.*]]
+; CHECK: [[EXIT2]]:
; CHECK-NEXT: ret void
;
entry:
>From a0653917822524d24fe03f603ff74eca1f6e3ca8 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Fri, 15 May 2026 15:49:51 +0100
Subject: [PATCH 3/3] [fixup] Keep the LoopUnrollPass in the full LTO pipeline
after vectorization
---
llvm/lib/Passes/PassBuilderPipelines.cpp | 15 +++++----------
llvm/test/Other/new-pm-lto-defaults.ll | 4 +---
2 files changed, 6 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 75789d96e590c3..f04532ab8c5858 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1376,7 +1376,6 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(InferAlignmentPass());
if (isFullLTOPostLink(LTOPhase)) {
- LoopPassManager LPM;
// The vectorizer may have significantly shortened a loop body; unroll
// again. Unroll small loops to hide loop backedge latency and saturate any
// parallel execution resources of an out-of-order processor. We also then
@@ -1386,15 +1385,11 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// across the loop nests.
// We do UnrollAndJam in a separate LPM to ensure it happens before unroll
if (EnableUnrollAndJam && PTO.LoopUnrolling)
- LPM.addPass(LoopUnrollAndJamPass(static_cast<int>(Level)));
-
- // Unroll small loops and perform peeling.
- LPM.addPass(LoopFullUnrollPass(static_cast<int>(Level),
- /* OnlyWhenForced= */ !PTO.LoopUnrolling,
- PTO.ForgetAllSCEVInLoopUnroll));
- FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM),
- /*UseMemorySSA=*/false));
-
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ LoopUnrollAndJamPass(static_cast<int>(Level))));
+ FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
+ static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll)));
FPM.addPass(WarnMissedTransformationsPass());
// Now that we are done with loop unrolling, be it either by LoopVectorizer,
// or LoopUnroll passes, some variable-offset GEP's into alloca's could have
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index af771698221b43..bf7fbe208efbdd 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -131,9 +131,7 @@
; CHECK-O23-NEXT: Running analysis: DemandedBitsAnalysis on foo
; CHECK-O23-NEXT: Running pass: DropUnnecessaryAssumesPass on foo
; CHECK-O23-NEXT: Running pass: InferAlignmentPass on foo
-; CHECK-O23-NEXT: Running pass: LoopSimplifyPass on foo
-; CHECK-O23-NEXT: Running pass: LCSSAPass on foo
-; CHECK-O23-NEXT: Running pass: LoopFullUnrollPass on loop
+; CHECK-O23-NEXT: Running pass: LoopUnrollPass on foo
; CHECK-O23-NEXT: WarnMissedTransformationsPass on foo
; CHECK-O23-NEXT: Running pass: SROAPass on foo
; CHECK-O23-NEXT: Running pass: InstCombinePass on foo
More information about the llvm-commits
mailing list