[llvm] [PassBuilder] VectorizerEnd Extension Points (PR #123494)
Axel Sorenson via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 18 18:59:27 PST 2025
https://github.com/axelcool1234 created https://github.com/llvm/llvm-project/pull/123494
Added an extension point after vectorizer passes in the PassBuilder. Additionally, added extension points before and after vectorizer passes in `buildLTODefaultPipeline`. Credit goes to @mshockwave for guiding me through my first LLVM contribution (and my first open source contribution in general!) :)
- Implemented `registerVectorizerEndEPCallback`
- Implemented `invokeVectorizerEndEPCallbacks`
- Added `VectorizerEndEPCallbacks` SmallVector
- Added a command line option `passes-ep-vectorizer-end` to `NewPMDriver.cpp`
- `buildModuleOptimizationPipeline` now calls `invokeVectorizerEndEPCallbacks`
- `buildO0DefaultPipeline` now calls `invokeVectorizerEndEPCallbacks`
- `buildLTODefaultPipeline` now calls BOTH `invokeVectorizerStartEPCallbacks` and `invokeVectorizerEndEPCallbacks`
- Added LIT tests to `new-pm-defaults.ll` and `new-pm-lto-defaults.ll`
- I'm not sure if a LIT test is required for `new-pm-O0-defaults.ll`. Let me know if I need to add a test there.
- Renamed `CHECK-EP-Peephole` to `CHECK-EP-PEEPHOLE` in `new-pm-lto-defaults.ll` for consistency.
>From 14bf48e4bac2f766ce4e2b989bf7c7a614e0e8f0 Mon Sep 17 00:00:00 2001
From: Axel Sorenson <AxelPSorenson at gmail.com>
Date: Tue, 31 Dec 2024 17:38:27 -0800
Subject: [PATCH] [PassBuilder] VectorizerEnd Extension Points Added an
extension point after vectorizer passes in the PassBuilder. Additionally,
added extension points before and after vectorizer passes in
`buildLTODefaultPipeline`. Credit goes to @mshockwave for guiding me through
my first LLVM contribution (and my first open source contribution in
general!) :) - Implemented `registerVectorizerEndEPCallback` - Implemented
`invokeVectorizerEndEPCallbacks` - Added `VectorizerEndEPCallbacks`
SmallVector - Added a command line option `passes-ep-vectorizer-end` to
`NewPMDriver.cpp` - `buildModuleOptimizationPipeline` now calls
`invokeVectorizerEndEPCallbacks` - `buildO0DefaultPipeline` now calls
`invokeVectorizerEndEPCallbacks` - `buildLTODefaultPipeline` now calls BOTH
`invokeVectorizerStartEPCallbacks` and `invokeVectorizerEndEPCallbacks` -
Added LIT tests to `new-pm-defaults.ll` and `new-pm-lto-defaults.ll` - I'm
not sure if a LIT test is required for `new-pm-O0-defaults.ll`. Let me know
if I need to add a test there. - Renamed `CHECK-EP-Peephole` to
`CHECK-EP-PEEPHOLE` in `new-pm-lto-defaults.ll` for consistency.
---
llvm/include/llvm/Passes/PassBuilder.h | 15 +++++++++++++++
llvm/lib/Passes/PassBuilderPipelines.cpp | 18 ++++++++++++++++++
llvm/test/Other/new-pm-defaults.ll | 5 +++++
llvm/test/Other/new-pm-lto-defaults.ll | 16 ++++++++++++----
llvm/tools/opt/NewPMDriver.cpp | 11 +++++++++++
5 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e7bc3a58f414f1..1b54855a5c6f44 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -465,6 +465,17 @@ class PassBuilder {
VectorizerStartEPCallbacks.push_back(C);
}
+ /// Register a callback for a default optimizer pipeline extension
+ /// point
+ ///
+ /// This extension point allows adding optimization passes after the
+ /// vectorizer and other highly target specific optimization passes are
+ /// executed.
+ void registerVectorizerEndEPCallback(
+ const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+ VectorizerEndEPCallbacks.push_back(C);
+ }
+
/// Register a callback for a default optimizer pipeline extension point.
///
/// This extension point allows adding optimization once at the start of the
@@ -631,6 +642,8 @@ class PassBuilder {
OptimizationLevel Level);
void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
OptimizationLevel Level);
+ void invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+ OptimizationLevel Level);
void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase Phase);
@@ -759,6 +772,8 @@ class PassBuilder {
CGSCCOptimizerLateEPCallbacks;
SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
VectorizerStartEPCallbacks;
+ SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+ VectorizerEndEPCallbacks;
// Module callbacks
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase)>,
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4ec0fb8fc81ea4..f5a82a243a07d7 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -360,6 +360,11 @@ void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
for (auto &C : VectorizerStartEPCallbacks)
C(FPM, Level);
}
+void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+ OptimizationLevel Level) {
+ for (auto &C : VectorizerEndEPCallbacks)
+ C(FPM, Level);
+}
void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
@@ -1534,6 +1539,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
+ invokeVectorizerEndEPCallbacks(OptimizePM, Level);
+
// LoopSink pass sinks instructions hoisted by LICM, which serves as a
// canonicalization pass that enables other optimizations. As a result,
// LoopSink pass needs to be a very late IR pass to avoid undoing LICM
@@ -2048,6 +2055,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MainFPM.addPass(MoveAutoInitPass());
MainFPM.addPass(MergedLoadStoreMotionPass());
+ invokeVectorizerStartEPCallbacks(MainFPM, Level);
+
LoopPassManager LPM;
if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
LPM.addPass(LoopFlattenPass());
@@ -2068,6 +2077,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
+ invokeVectorizerEndEPCallbacks(MainFPM, Level);
+
// Run the OpenMPOpt CGSCC pass again late.
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
@@ -2231,6 +2242,13 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
+ if (!VectorizerEndEPCallbacks.empty()) {
+ FunctionPassManager FPM;
+ invokeVectorizerEndEPCallbacks(FPM, Level);
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+
ModulePassManager CoroPM;
CoroPM.addPass(CoroEarlyPass());
CGSCCPassManager CGPM;
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 7cf035b0c6f376..c554fdbf4c7991 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -51,6 +51,10 @@
; RUN: -passes='default<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-START,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes-ep-vectorizer-end='no-op-function' \
+; RUN: -passes='default<O3>' -S %s 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-END,CHECK-O23SZ
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes-ep-pipeline-start='no-op-module' \
; RUN: -passes='default<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-START,CHECK-O23SZ
@@ -274,6 +278,7 @@
; CHECK-O-NEXT: Running pass: LCSSAPass
; CHECK-O-NEXT: Running pass: LICMPass
; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass
; CHECK-O-NEXT: Running pass: LoopSinkPass
; CHECK-O-NEXT: Running pass: InstSimplifyPass
; CHECK-O-NEXT: Running pass: DivRemPairsPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index f788db1e338a1e..0b0202d4a14b5a 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -18,6 +18,12 @@
; RUN: -passes='lto<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes='lto<O3>' -S %s -passes-ep-vectorizer-start='no-op-function' 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-START
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes='lto<O3>' -S %s -passes-ep-vectorizer-end='no-op-function' 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-END
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes='lto<Os>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-OS,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
@@ -25,7 +31,7 @@
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes='lto<O3>' -S %s -passes-ep-peephole='no-op-function' 2>&1 \
-; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-Peephole
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-PEEPHOLE
; CHECK-EP: Running pass: NoOpModulePass
; CHECK-O: Running pass: CrossDSOCFIPass
@@ -69,7 +75,7 @@
; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
; CHECK-O23SZ-NEXT: Running analysis: LastRunTrackingAnalysis
; CHECK-O23SZ-NEXT: Running pass: AggressiveInstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O23SZ-NEXT: Running pass: ExpandVariadicsPass
; CHECK-O23SZ-NEXT: Running pass: ModuleInlinerWrapperPass
; CHECK-O23SZ-NEXT: Running analysis: InlineAdvisorAnalysis
@@ -81,7 +87,7 @@
; CHECK-O23SZ-NEXT: Running pass: GlobalDCEPass
; CHECK-O23SZ-NEXT: Running pass: ArgumentPromotionPass
; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O23SZ-NEXT: Running pass: ConstraintEliminationPass
; CHECK-O23SZ-NEXT: Running analysis: LoopAnalysis on foo
; CHECK-O23SZ-NEXT: Running analysis: ScalarEvolutionAnalysis on foo
@@ -109,6 +115,7 @@
; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo
; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo
+; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass on foo
; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo
; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo
; CHECK-O23SZ-NEXT: Running pass: IndVarSimplifyPass on loop
@@ -137,7 +144,8 @@
; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
; CHECK-O23SZ-NEXT: Running pass: LICMPass
; CHECK-O23SZ-NEXT: Running pass: AlignmentFromAssumptionsPass on foo
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass on foo
; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass on foo
; CHECK-O23SZ-NEXT: Running pass: LowerTypeTestsPass
; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index ea300fb3dbeb13..5ed42b054316be 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -122,6 +122,11 @@ static cl::opt<std::string> VectorizerStartEPPipeline(
cl::desc("A textual description of the function pass pipeline inserted at "
"the VectorizerStart extension point into default pipelines"),
cl::Hidden);
+static cl::opt<std::string> VectorizerEndEPPipeline(
+ "passes-ep-vectorizer-end",
+ cl::desc("A textual description of the function pass pipeline inserted at "
+ "the VectorizerEnd extension point into default pipelines"),
+ cl::Hidden);
static cl::opt<std::string> PipelineStartEPPipeline(
"passes-ep-pipeline-start",
cl::desc("A textual description of the module pass pipeline inserted at "
@@ -285,6 +290,12 @@ static void registerEPCallbacks(PassBuilder &PB) {
ExitOnError Err("Unable to parse VectorizerStartEP pipeline: ");
Err(PB.parsePassPipeline(PM, VectorizerStartEPPipeline));
});
+ if (tryParsePipelineText<FunctionPassManager>(PB, VectorizerEndEPPipeline))
+ PB.registerVectorizerEndEPCallback(
+ [&PB](FunctionPassManager &PM, OptimizationLevel Level) {
+ ExitOnError Err("Unable to parse VectorizerEndEP pipeline: ");
+ Err(PB.parsePassPipeline(PM, VectorizerEndEPPipeline));
+ });
if (tryParsePipelineText<ModulePassManager>(PB, PipelineStartEPPipeline))
PB.registerPipelineStartEPCallback(
[&PB](ModulePassManager &PM, OptimizationLevel) {
More information about the llvm-commits
mailing list