[llvm] d3161de - [PassBuilder] VectorizerEnd Extension Points (#123494)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 29 11:24:06 PST 2025
Author: Axel Sorenson
Date: 2025-01-29T11:24:03-08:00
New Revision: d3161defd6b717241a85b6ca528754c747060735
URL: https://github.com/llvm/llvm-project/commit/d3161defd6b717241a85b6ca528754c747060735
DIFF: https://github.com/llvm/llvm-project/commit/d3161defd6b717241a85b6ca528754c747060735.diff
LOG: [PassBuilder] VectorizerEnd Extension Points (#123494)
Added an extension point after vectorizer passes in the PassBuilder.
Additionally, added extension points before and after vectorizer passes
in `buildLTODefaultPipeline`. Credit goes to @mshockwave for guiding me
through my first LLVM contribution (and my first open source
contribution in general!) :)
- Implemented `registerVectorizerEndEPCallback`
- Implemented `invokeVectorizerEndEPCallbacks`
- Added `VectorizerEndEPCallbacks` SmallVector
- Added a command line option `passes-ep-vectorizer-end` to
`NewPMDriver.cpp`
- `buildModuleOptimizationPipeline` now calls
`invokeVectorizerEndEPCallbacks`
- `buildO0DefaultPipeline` now calls `invokeVectorizerEndEPCallbacks`
- `buildLTODefaultPipeline` now calls BOTH
`invokeVectorizerStartEPCallbacks` and `invokeVectorizerEndEPCallbacks`
- Added LIT tests to `new-pm-defaults.ll`, `new-pm-lto-defaults.ll`,
`new-pm-O0-ep-callbacks.ll`, and `pass-pipeline-parsing.ll`
- Renamed `CHECK-EP-Peephole` to `CHECK-EP-PEEPHOLE` in
`new-pm-lto-defaults.ll` for consistency.
This code is intended for developers that wish to implement and run
custom passes after the vectorizer passes in the PassBuilder pipeline.
For example, in #91796, a pass was created that changed the induction
variables of vectorized code. This is right after the vectorization
passes.
Added:
Modified:
llvm/include/llvm/Passes/PassBuilder.h
llvm/lib/Passes/PassBuilderPipelines.cpp
llvm/test/Other/new-pm-O0-ep-callbacks.ll
llvm/test/Other/new-pm-defaults.ll
llvm/test/Other/new-pm-lto-defaults.ll
llvm/test/Other/pass-pipeline-parsing.ll
llvm/tools/opt/NewPMDriver.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e7bc3a58f414f1..1b54855a5c6f44 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -465,6 +465,17 @@ class PassBuilder {
VectorizerStartEPCallbacks.push_back(C);
}
+ /// Register a callback for a default optimizer pipeline extension
+ /// point
+ ///
+ /// This extension point allows adding optimization passes after the
+ /// vectorizer and other highly target specific optimization passes are
+ /// executed.
+ void registerVectorizerEndEPCallback(
+ const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+ VectorizerEndEPCallbacks.push_back(C);
+ }
+
/// Register a callback for a default optimizer pipeline extension point.
///
/// This extension point allows adding optimization once at the start of the
@@ -631,6 +642,8 @@ class PassBuilder {
OptimizationLevel Level);
void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
OptimizationLevel Level);
+ void invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+ OptimizationLevel Level);
void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase Phase);
@@ -759,6 +772,8 @@ class PassBuilder {
CGSCCOptimizerLateEPCallbacks;
SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
VectorizerStartEPCallbacks;
+ SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+ VectorizerEndEPCallbacks;
// Module callbacks
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase)>,
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 17ff3bd37884b7..c6cf6cdbe9390f 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -360,6 +360,11 @@ void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
for (auto &C : VectorizerStartEPCallbacks)
C(FPM, Level);
}
+void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
+ OptimizationLevel Level) {
+ for (auto &C : VectorizerEndEPCallbacks)
+ C(FPM, Level);
+}
void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
@@ -1534,6 +1539,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
+ invokeVectorizerEndEPCallbacks(OptimizePM, Level);
+
// LoopSink pass sinks instructions hoisted by LICM, which serves as a
// canonicalization pass that enables other optimizations. As a result,
// LoopSink pass needs to be a very late IR pass to avoid undoing LICM
@@ -2046,6 +2053,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
MainFPM.addPass(MoveAutoInitPass());
MainFPM.addPass(MergedLoadStoreMotionPass());
+ invokeVectorizerStartEPCallbacks(MainFPM, Level);
+
LoopPassManager LPM;
if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
LPM.addPass(LoopFlattenPass());
@@ -2066,6 +2075,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
+ invokeVectorizerEndEPCallbacks(MainFPM, Level);
+
// Run the OpenMPOpt CGSCC pass again late.
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
@@ -2231,6 +2242,13 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
+ if (!VectorizerEndEPCallbacks.empty()) {
+ FunctionPassManager FPM;
+ invokeVectorizerEndEPCallbacks(FPM, Level);
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+
ModulePassManager CoroPM;
CoroPM.addPass(CoroEarlyPass());
CGSCCPassManager CGPM;
diff --git a/llvm/test/Other/new-pm-O0-ep-callbacks.ll b/llvm/test/Other/new-pm-O0-ep-callbacks.ll
index 2184a2ded5920d..77ccddba56a623 100644
--- a/llvm/test/Other/new-pm-O0-ep-callbacks.ll
+++ b/llvm/test/Other/new-pm-O0-ep-callbacks.ll
@@ -3,6 +3,7 @@
; RUN: opt -disable-output -debug-pass-manager -passes-ep-scalar-optimizer-late=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
; RUN: opt -disable-output -debug-pass-manager -passes-ep-cgscc-optimizer-late=no-op-cgscc -passes='default<O0>' 2>&1 < %s | FileCheck %s
; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-start=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-end=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-start=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-early-simplification=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
; RUN: opt -disable-output -debug-pass-manager -passes-ep-optimizer-early=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 7cf035b0c6f376..c554fdbf4c7991 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -51,6 +51,10 @@
; RUN: -passes='default<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-START,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes-ep-vectorizer-end='no-op-function' \
+; RUN: -passes='default<O3>' -S %s 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-END,CHECK-O23SZ
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes-ep-pipeline-start='no-op-module' \
; RUN: -passes='default<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-START,CHECK-O23SZ
@@ -274,6 +278,7 @@
; CHECK-O-NEXT: Running pass: LCSSAPass
; CHECK-O-NEXT: Running pass: LICMPass
; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass
; CHECK-O-NEXT: Running pass: LoopSinkPass
; CHECK-O-NEXT: Running pass: InstSimplifyPass
; CHECK-O-NEXT: Running pass: DivRemPairsPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 59d9ee24527b64..86480c5115748d 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -18,6 +18,12 @@
; RUN: -passes='lto<O3>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes='lto<O3>' -S %s -passes-ep-vectorizer-start='no-op-function' 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-START
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
+; RUN: -passes='lto<O3>' -S %s -passes-ep-vectorizer-end='no-op-function' 2>&1 \
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-END
+; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes='lto<Os>' -S %s 2>&1 \
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-OS,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
@@ -25,7 +31,7 @@
; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ
; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \
; RUN: -passes='lto<O3>' -S %s -passes-ep-peephole='no-op-function' 2>&1 \
-; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-Peephole
+; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-PEEPHOLE
; CHECK-EP: Running pass: NoOpModulePass
; CHECK-O: Running pass: CrossDSOCFIPass
@@ -69,7 +75,7 @@
; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
; CHECK-O23SZ-NEXT: Running analysis: LastRunTrackingAnalysis
; CHECK-O23SZ-NEXT: Running pass: AggressiveInstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O23SZ-NEXT: Running pass: ExpandVariadicsPass
; CHECK-O23SZ-NEXT: Running pass: ModuleInlinerWrapperPass
; CHECK-O23SZ-NEXT: Running analysis: InlineAdvisorAnalysis
@@ -81,7 +87,7 @@
; CHECK-O23SZ-NEXT: Running pass: GlobalDCEPass
; CHECK-O23SZ-NEXT: Running pass: ArgumentPromotionPass
; CHECK-O23SZ-NEXT: Running pass: InstCombinePass
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O23SZ-NEXT: Running pass: ConstraintEliminationPass
; CHECK-O23SZ-NEXT: Running analysis: LoopAnalysis on foo
; CHECK-O23SZ-NEXT: Running analysis: ScalarEvolutionAnalysis on foo
@@ -109,6 +115,7 @@
; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo
; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo
+; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass on foo
; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo
; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo
; CHECK-O23SZ-NEXT: Running pass: IndVarSimplifyPass on loop
@@ -137,7 +144,8 @@
; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
; CHECK-O23SZ-NEXT: Running pass: LICMPass
; CHECK-O23SZ-NEXT: Running pass: AlignmentFromAssumptionsPass on foo
-; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass on foo
+; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass on foo
; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass on foo
; CHECK-O23SZ-NEXT: Running pass: LowerTypeTestsPass
; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll
index fd65224b1ef0d8..4253b646b69b05 100644
--- a/llvm/test/Other/pass-pipeline-parsing.ll
+++ b/llvm/test/Other/pass-pipeline-parsing.ll
@@ -264,6 +264,9 @@
; RUN: opt -passes-ep-vectorizer-start=bad -passes=no-op-function \
; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZERSTART-ERR
; PASSES-EP-VECTORIZERSTART-ERR: Could not parse -passes-ep-vectorizer-start pipeline: unknown function pass 'bad'
+; RUN: opt -passes-ep-vectorizer-end=bad -passes=no-op-function \
+; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZEREND-ERR
+; PASSES-EP-VECTORIZEREND-ERR: Could not parse -passes-ep-vectorizer-end pipeline: unknown function pass 'bad'
; RUN: opt -passes-ep-pipeline-start=bad -passes=no-op-function \
; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-PIPELINESTART-ERR
; PASSES-EP-PIPELINESTART-ERR: Could not parse -passes-ep-pipeline-start pipeline: unknown pass name 'bad'
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index ea300fb3dbeb13..5ed42b054316be 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -122,6 +122,11 @@ static cl::opt<std::string> VectorizerStartEPPipeline(
cl::desc("A textual description of the function pass pipeline inserted at "
"the VectorizerStart extension point into default pipelines"),
cl::Hidden);
+static cl::opt<std::string> VectorizerEndEPPipeline(
+ "passes-ep-vectorizer-end",
+ cl::desc("A textual description of the function pass pipeline inserted at "
+ "the VectorizerEnd extension point into default pipelines"),
+ cl::Hidden);
static cl::opt<std::string> PipelineStartEPPipeline(
"passes-ep-pipeline-start",
cl::desc("A textual description of the module pass pipeline inserted at "
@@ -285,6 +290,12 @@ static void registerEPCallbacks(PassBuilder &PB) {
ExitOnError Err("Unable to parse VectorizerStartEP pipeline: ");
Err(PB.parsePassPipeline(PM, VectorizerStartEPPipeline));
});
+ if (tryParsePipelineText<FunctionPassManager>(PB, VectorizerEndEPPipeline))
+ PB.registerVectorizerEndEPCallback(
+ [&PB](FunctionPassManager &PM, OptimizationLevel Level) {
+ ExitOnError Err("Unable to parse VectorizerEndEP pipeline: ");
+ Err(PB.parsePassPipeline(PM, VectorizerEndEPPipeline));
+ });
if (tryParsePipelineText<ModulePassManager>(PB, PipelineStartEPPipeline))
PB.registerPipelineStartEPCallback(
[&PB](ModulePassManager &PM, OptimizationLevel) {
More information about the llvm-commits
mailing list