[llvm] [CodeGen][Pass] Add `TargetPassBuilder` (PR #137290)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 25 00:23:46 PDT 2025
https://github.com/paperchalice created https://github.com/llvm/llvm-project/pull/137290
This implementation supports `--start/stop-before/after` options by hijacking original pass managers. With injected-class-name, user should not feel any difference from original pass manager in most cases.
Except virtual functions to add necessary passes (e.g. instruction select pass), the only method to extend the pipeline is `injectBefore`, which accept a callback to extend pipeline at the specified point. The return type of the callback is depend on the selected pass
```c++
injectBefore<SomeFunctionPass>([](){
FunctionPassManager FPM;
// Add passes you want.
return FPM;
});
// If you really want to add module pass between machine function passes...
injectBefore<SomeMachineFunctionPass, ModulePassManager>([](){
ModulePassManager MPM;
MachineFunctionPass MFPM;
// Add passes you want.
MPM.createModuleToFunctionPassAdaptor(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
return MPM;
});
```
>From 22d7f27fd78b3b0a9d0378f7dc81788a351807cf Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 24 Mar 2025 19:34:01 +0800
Subject: [PATCH] Add `TargetPassBuilder`.
---
.../llvm/Passes/MachinePassRegistry.def | 4 +
llvm/include/llvm/Passes/PassBuilder.h | 2 +
llvm/include/llvm/Passes/TargetPassBuilder.h | 238 ++++++
.../include/llvm/Target/CGPassBuilderOption.h | 5 +
llvm/lib/CodeGen/TargetPassConfig.cpp | 3 +
llvm/lib/Passes/CMakeLists.txt | 1 +
llvm/lib/Passes/TargetPassBuilder.cpp | 791 ++++++++++++++++++
llvm/unittests/CodeGen/CMakeLists.txt | 1 +
.../CodeGen/TargetPassBuilderTest.cpp | 176 ++++
9 files changed, 1221 insertions(+)
create mode 100644 llvm/include/llvm/Passes/TargetPassBuilder.h
create mode 100644 llvm/lib/Passes/TargetPassBuilder.cpp
create mode 100644 llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 3e9e788662900..a2b182a8ea39b 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -269,11 +269,13 @@ DUMMY_MACHINE_MODULE_PASS("pseudo-probe-inserter", PseudoProbeInserterPass)
DUMMY_MACHINE_MODULE_PASS("mir-debugify", DebugifyMachineModule)
DUMMY_MACHINE_MODULE_PASS("mir-check-debugify", CheckDebugMachineModulePass)
DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
+DUMMY_MACHINE_MODULE_PASS("static-data-annotator", StaticDataAnnotatorPass)
#undef DUMMY_MACHINE_MODULE_PASS
#ifndef DUMMY_MACHINE_FUNCTION_PASS
#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
#endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
DUMMY_MACHINE_FUNCTION_PASS("block-placement-stats", MachineBlockPlacementStatsPass)
@@ -314,6 +316,8 @@ DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass)
DUMMY_MACHINE_FUNCTION_PASS("shrink-wrap", ShrinkWrapPass)
DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
+DUMMY_MACHINE_FUNCTION_PASS("static-data-splitter", StaticDataSplitterPass)
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
+DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination", UnreachableMachineBlockElim)
DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 51ccaa53447d7..6fca52e5062e5 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -112,6 +112,8 @@ class PassBuilder {
std::optional<PGOOptions> PGOOpt;
PassInstrumentationCallbacks *PIC;
+ friend class TargetPassBuilder;
+
public:
/// A struct to capture parsed pass pipeline names.
///
diff --git a/llvm/include/llvm/Passes/TargetPassBuilder.h b/llvm/include/llvm/Passes/TargetPassBuilder.h
new file mode 100644
index 0000000000000..561f6c58c3d2c
--- /dev/null
+++ b/llvm/include/llvm/Passes/TargetPassBuilder.h
@@ -0,0 +1,238 @@
+//===- Parsing, selection, and construction of pass pipelines --*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Interfaces for registering analysis passes, producing common pass manager
+/// configurations, and parsing of pass pipelines.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PASSES_TARGETPASSBUILDER_H
+#define LLVM_PASSES_TARGETPASSBUILDER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/identity.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/Target/CGPassBuilderOption.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include <list>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace llvm {
+
+class PassBuilder;
+class TargetMachine;
+class SelectionDAGISel;
+
+class TargetPassBuilder {
+public:
+ TargetPassBuilder(PassBuilder &PB);
+
+ virtual ~TargetPassBuilder() = default;
+
+ // TODO: Add necessary parameters once AsmPrinter is ported to new pass
+ // manager.
+ llvm::ModulePassManager buildPipeline(CodeGenFileType FileType);
+
+private:
+ using PassList = std::list<std::pair<
+ StringRef,
+ std::variant<llvm::ModulePassManager, llvm::FunctionPassManager,
+ llvm::LoopPassManager, llvm::MachineFunctionPassManager>>>;
+
+ template <typename InternalPassT> struct AdaptorWrapper : InternalPassT {
+ using InternalPassT::Passes;
+ };
+
+ template <typename PassManagerT, typename InternalPassT = void>
+ class PassManagerWrapper {
+ friend class TargetPassBuilder;
+
+ public:
+ bool isEmpty() const { return Passes.empty(); }
+
+ template <typename PassT> void addPass(PassT &&P) {
+ PassManagerT PM;
+ PM.addPass(std::forward<PassT>(P));
+ Passes.emplace_back(PassT::name(), std::move(PM));
+ }
+
+ void addPass(PassManagerWrapper &&PM) {
+ for (auto &P : PM.Passes)
+ Passes.push_back(std::move(P));
+ }
+
+ void addPass(AdaptorWrapper<InternalPassT> &&Adaptor) {
+ for (auto &P : Adaptor.Passes)
+ Passes.push_back(std::move(P));
+ }
+
+ void addPass(llvm::ModulePassManager &&) = delete;
+ void addPass(llvm::FunctionPassManager &&) = delete;
+ void addPass(llvm::LoopPassManager &&) = delete;
+ void addPass(llvm::MachineFunctionPassManager &&) = delete;
+
+ private:
+ PassList Passes;
+ };
+
+ template <typename NestedPassManagerT, typename PassT>
+ AdaptorWrapper<NestedPassManagerT> createPassAdaptor(PassT &&P) {
+ AdaptorWrapper<NestedPassManagerT> Adaptor;
+ Adaptor.addPass(std::forward<PassT>(P));
+ return Adaptor;
+ }
+
+private:
+ template <typename PassT, typename IRUnitT>
+ using HasRunOnIRUnit = decltype(std::declval<PassT>().run(
+ std::declval<IRUnitT &>(), std::declval<AnalysisManager<IRUnitT> &>()));
+ template <typename PassT>
+ static constexpr bool isModulePass =
+ is_detected<HasRunOnIRUnit, PassT, Module>::value;
+ template <typename PassT>
+ static constexpr bool isFunctionPass =
+ is_detected<HasRunOnIRUnit, PassT, Function>::value;
+ template <typename PassT>
+ static constexpr bool isMachineFunctionPass =
+ is_detected<HasRunOnIRUnit, PassT, MachineFunction>::value;
+
+protected:
+ // Hijack real pass managers intentionally.
+ using MachineFunctionPassManager =
+ PassManagerWrapper<llvm::MachineFunctionPassManager>;
+ using FunctionPassManager =
+ PassManagerWrapper<llvm::FunctionPassManager, MachineFunctionPassManager>;
+ using ModulePassManager =
+ PassManagerWrapper<llvm::ModulePassManager, FunctionPassManager>;
+
+protected:
+ template <typename FunctionPassT>
+ AdaptorWrapper<FunctionPassManager>
+ createModuleToFunctionPassAdaptor(FunctionPassT &&P) {
+ return createPassAdaptor<FunctionPassManager>(
+ std::forward<FunctionPassT>(P));
+ }
+
+ template <typename FunctionPassT>
+ AdaptorWrapper<FunctionPassManager>
+ createModuleToPostOrderCGSCCPassAdaptor(FunctionPassManager &&PM) {
+ assert(CGPBO.RequiresCodeGenSCCOrder &&
+ "Target must set RequiresCodeGenSCCOrder!");
+ return createPassAdaptor<FunctionPassManager>(
+ std::forward<FunctionPassT>(PM));
+ }
+
+ template <typename MachineFunctionPassT>
+ AdaptorWrapper<MachineFunctionPassManager>
+ createFunctionToMachineFunctionPassAdaptor(MachineFunctionPassT &&P) {
+ return createPassAdaptor<MachineFunctionPassManager>(
+ std::forward<MachineFunctionPassManager>(P));
+ }
+
+protected:
+ PassBuilder &PB;
+ TargetMachine *TM;
+ CodeGenOptLevel OptLevel;
+ CGPassBuilderOption CGPBO = getCGPassBuilderOption();
+
+ template <typename PassT,
+ typename PassManagerT = std::conditional_t<
+ isModulePass<PassT>, ModulePassManager,
+ std::conditional_t<isFunctionPass<PassT>, FunctionPassManager,
+ MachineFunctionPassManager>>>
+ void injectBefore(
+ typename llvm::identity<std::function<PassManagerT()>>::argument_type F) {
+ InjectionCallbacks.push_back(
+ [Accessed = false, F](PassList &Passes, PassList::iterator I) mutable {
+ if (Accessed)
+ return I;
+ if (PassT::name() != I->first)
+ return I;
+ Accessed = true;
+ auto PMPasses = F().Passes;
+ return Passes.insert(I, std::make_move_iterator(PMPasses.begin()),
+ std::make_move_iterator(PMPasses.end()));
+ });
+ }
+
+ template <typename PassTs> void disablePass() {
+ DisabedPasses.insert(PassTs::name());
+ }
+
+ void disablePass(StringRef Name) { DisabedPasses.insert(Name); }
+
+ template <typename PassT> bool isPassDisabled() const {
+ return DisabedPasses.contains(PassT::name());
+ }
+
+ bool isPassDisabled(StringRef Name) const {
+ return DisabedPasses.contains(Name);
+ }
+
+ template <typename PassT> bool isPassEnabled() const {
+ return !isPassDisabled<PassT>();
+ }
+
+ bool isPassEnabled(StringRef Name) const { return !isPassDisabled(Name); }
+
+protected:
+ /// \defgroup target specific hooks
+ /// @{
+ // FIXME: Find a way that each target can return sliced object.
+ virtual std::unique_ptr<SelectionDAGISel> getSelectionDAGISelPass() = 0;
+ /// @}
+
+private:
+ void buildCoreCodeGenPipeline(ModulePassManager &MPM);
+
+ ModulePassManager buildCodeGenIRPipeline();
+
+ /// Add passes that optimize machine instructions in SSA form.
+ void addISelPasses(MachineFunctionPassManager &MFPM);
+ void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM);
+ void addRegAllocPipeline(MachineFunctionPassManager &MFPM);
+ void addRegAllocPass(MachineFunctionPassManager &MFPM, bool Optimized);
+ ModulePassManager buildCodeGenMIRPipeline();
+
+ void addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+ void filtPassList(ModulePassManager &MPM) const;
+ static void addMIRPrinterPasses(ModulePassManager &MPM);
+
+ llvm::ModulePassManager constructRealPassManager(ModulePassManager &&MPMW);
+
+private:
+ virtual void anchor();
+
+ StringSet<> DisabedPasses;
+ std::vector<std::function<PassList::iterator(PassList &, PassList::iterator)>>
+ InjectionCallbacks;
+
+ void invokeInjectionCallbacks(ModulePassManager &MPM) const;
+
+ // Only Loop Strength Reduction need this, shadow LoopPassManager
+ // in future if it is necessary.
+ template <typename PassT>
+ void addLoopPass(FunctionPassManager &FPM, PassT &&P) {
+ LoopPassManager LPM;
+ LPM.addPass(std::forward<PassT>(P));
+ FPM.Passes.push_back(std::pair(PassT::name(), std::move(LPM)));
+ }
+};
+
+template <> struct TargetPassBuilder::AdaptorWrapper<void> {};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h
index 51f25c1360b87..2cc2afc0b24b1 100644
--- a/llvm/include/llvm/Target/CGPassBuilderOption.h
+++ b/llvm/include/llvm/Target/CGPassBuilderOption.h
@@ -51,9 +51,11 @@ struct CGPassBuilderOption {
bool EnableMachineFunctionSplitter = false;
bool EnableSinkAndFold = false;
bool EnableTailMerge = true;
+ bool EnableLoopTermFold = false;
bool MISchedPostRA = false;
bool EarlyLiveIntervals = false;
bool GCEmptyBlocks = false;
+ bool SplitStaticData = false;
bool DisableLSR = false;
bool DisableCGP = false;
@@ -65,6 +67,9 @@ struct CGPassBuilderOption {
bool DisableExpandReductions = false;
bool DisableRAFSProfileLoader = false;
bool DisableCFIFixup = false;
+ bool DisableReplaceWithVecLib = false;
+ bool DisableLayoutFSProfileLoader = false;
+ bool DisablePrologEpilogInserterPass = false;
bool PrintAfterISel = false;
bool PrintISelInput = false;
bool RequiresCodeGenSCCOrder = false;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index fa1bb84ec5319..70e1190b88418 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -506,12 +506,15 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
SET_BOOLEAN_OPTION(DisableCGP)
SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
SET_BOOLEAN_OPTION(DisableSelectOptimize)
+ SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+ SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
SET_BOOLEAN_OPTION(PrintISelInput)
SET_BOOLEAN_OPTION(DebugifyAndStripAll)
SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)
SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)
SET_BOOLEAN_OPTION(DisableCFIFixup)
SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)
+ SET_BOOLEAN_OPTION(SplitStaticData)
return Opt;
}
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 6425f4934b210..77f23b2273af1 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMPasses
PassBuilderPipelines.cpp
PassPlugin.cpp
StandardInstrumentations.cpp
+ TargetPassBuilder.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm
diff --git a/llvm/lib/Passes/TargetPassBuilder.cpp b/llvm/lib/Passes/TargetPassBuilder.cpp
new file mode 100644
index 0000000000000..879cbab6e03e7
--- /dev/null
+++ b/llvm/lib/Passes/TargetPassBuilder.cpp
@@ -0,0 +1,791 @@
+//===- Construction of CodeGen pass pipelines -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/TargetPassBuilder.h"
+#include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
+#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandFp.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/FinalizeISel.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/GlobalMergeFunctions.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LocalStackSlotAllocation.h"
+#include "llvm/CodeGen/LowerEmuTLS.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
+#include "llvm/CodeGen/SafeStack.h"
+#include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
+#include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/CodeGen/WasmEHPrepare.h"
+#include "llvm/CodeGen/WinEHPrepare.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only!
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+#include <stack>
+
+using namespace llvm;
+
+TargetPassBuilder::TargetPassBuilder(PassBuilder &PB)
+ : PB(PB), TM(PB.TM), OptLevel(TM->getOptLevel()) {}
+
+ModulePassManager TargetPassBuilder::buildPipeline(CodeGenFileType FileType) {
+ ModulePassManager MPM;
+ buildCoreCodeGenPipeline(MPM);
+ invokeInjectionCallbacks(MPM);
+ filtPassList(MPM);
+ // TODO: Add file printer passes.
+ if (TargetPassConfig::willCompleteCodeGenPipeline()) {
+ // TODO: Add ASM printer.
+ } else {
+ if (FileType != CodeGenFileType::Null)
+ addMIRPrinterPasses(MPM);
+ }
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ InvalidateAnalysisPass<MachineFunctionAnalysis>()));
+ return constructRealPassManager(std::move(MPM));
+}
+
+void TargetPassBuilder::buildCoreCodeGenPipeline(ModulePassManager &MPM) {
+ MPM.addPass(buildCodeGenIRPipeline());
+ MPM.addPass(buildCodeGenMIRPipeline());
+}
+
+TargetPassBuilder::ModulePassManager
+TargetPassBuilder::buildCodeGenIRPipeline() {
+ ModulePassManager MPM;
+ if (TM->useEmulatedTLS())
+ MPM.addPass(LowerEmuTLSPass());
+ MPM.addPass(PreISelIntrinsicLoweringPass(TM));
+ FunctionPassManager FPM;
+ FPM.addPass(ExpandLargeDivRemPass(TM));
+ FPM.addPass(ExpandFpPass(TM));
+
+ // Run loop strength reduction before anything else.
+ if (TM->getOptLevel() == CodeGenOptLevel::None) {
+ // Basic AliasAnalysis support.
+ // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
+ // BasicAliasAnalysis wins if they disagree. This is intended to help
+ // support "obvious" type-punning idioms.
+ FPM.addPass(RequireAnalysisPass<TypeBasedAA, Function>());
+ FPM.addPass(RequireAnalysisPass<ScopedNoAliasAA, Function>());
+ FPM.addPass(RequireAnalysisPass<BasicAA, Function>());
+
+ if (!CGPBO.DisableLSR) {
+ addLoopPass(FPM, CanonicalizeFreezeInLoopsPass());
+ addLoopPass(FPM, LoopStrengthReducePass());
+ if (CGPBO.EnableLoopTermFold)
+ addLoopPass(FPM, LoopTermFoldPass());
+ }
+
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences
+ // of loads and compares. ExpandMemCmpPass then tries to expand those
+ // calls into optimally-sized loads and compares. The transforms are
+ // enabled by a target lowering hook.
+ if (!CGPBO.DisableMergeICmps)
+ FPM.addPass(MergeICmpsPass());
+ FPM.addPass(ExpandMemCmpPass(TM));
+ }
+
+ // Run GC lowering passes for builtin collectors
+ FPM.addPass(GCLoweringPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = FunctionPassManager();
+ MPM.addPass(ShadowStackGCLoweringPass());
+ // PB.invokeGCLoweringEPCallbacks();
+
+ if (TM->getTargetTriple().isOSBinFormatMachO() &&
+ !CGPBO.DisableAtExitBasedGlobalDtorLowering) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = FunctionPassManager();
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(LowerGlobalDtorsPass());
+ }
+
+ // Make sure that no unreachable blocks are instruction selected.
+ FPM.addPass(UnreachableBlockElimPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableConstantHoisting)
+ FPM.addPass(ConstantHoistingPass());
+ if (!CGPBO.DisableReplaceWithVecLib)
+ FPM.addPass(ReplaceWithVeclib());
+ if (!CGPBO.DisablePartialLibcallInlining)
+ FPM.addPass(PartiallyInlineLibCallsPass());
+ }
+
+ // Instrument function entry after all inlining.
+ FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ FPM.addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants
+ // to. Allow disabling it for testing purposes.
+ if (!CGPBO.DisableExpandReductions)
+ FPM.addPass(ExpandReductionsPass());
+
+ // Convert conditional moves to conditional jumps when profitable.
+ if (OptLevel != CodeGenOptLevel::None && !CGPBO.DisableSelectOptimize)
+ FPM.addPass(SelectOptimizePass(TM));
+
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ if (CGPBO.EnableGlobalMergeFunc)
+ MPM.addPass(GlobalMergeFuncPass());
+
+ FPM.addPass(CodeGenPreparePass(PB.TM));
+ addExceptionHandlingPasses(FPM);
+
+ // Add common passes that perform LLVM IR to IR transforms in preparation for
+ // instruction selection.
+
+ if (OptLevel != CodeGenOptLevel::None)
+ FPM.addPass(ObjCARCContractPass());
+
+ FPM.addPass(CallBrPreparePass());
+
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ FPM.addPass(SafeStackPass(TM));
+ FPM.addPass(StackProtectorPass(TM));
+
+ // All passes which modify the LLVM IR are now complete; run the verifier
+ // to ensure that the IR is valid.
+ if (!CGPBO.DisableVerify)
+ FPM.addPass(VerifierPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ return MPM;
+}
+
+void TargetPassBuilder::addISelPasses(MachineFunctionPassManager &MFPM) {
+ // Add core instruction selection passes.
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true));
+
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ if (CGPBO.EnableFastISelOption.value_or(false))
+ Selector = SelectorType::FastISel;
+ else if (CGPBO.EnableGlobalISelOption.value_or(false) ||
+ (TM->Options.EnableGlobalISel &&
+ CGPBO.EnableGlobalISelOption.value_or(true)))
+ Selector = SelectorType::GlobalISel;
+ else if (TM->getOptLevel() == CodeGenOptLevel::None &&
+ TM->getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM->setFastISel(true);
+ TM->setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM->setFastISel(false);
+ TM->setGlobalISel(true);
+ }
+
+ // FIXME: Currently debugify is not support in new pass manager,
+ // because it inserts module passes between each pass.
+
+ // Add instruction selector passes for global isel if enabled.
+ if (Selector == SelectorType::GlobalISel) {
+ MFPM.addPass(IRTranslatorPass());
+ MFPM.addPass(RegBankSelectPass());
+ MFPM.addPass(InstructionSelectPass(OptLevel));
+ }
+
+ // Pass to reset the MachineFunction if the ISel failed. Outside of the
+ // above if so that the verifier is not added to it.
+ MFPM.addPass(ResetMachineFunctionPass());
+
+ // Run the SDAG InstSelector, providing a fallback path when we do not want
+ // to abort on not-yet-supported input.
+ if (Selector != SelectorType::GlobalISel ||
+ TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) {
+ // TODO: MFPM.addPass(getSelectionDAGISelPass());
+ }
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier
+ // before FinalizeISel.
+ MFPM.addPass(FinalizeISelPass());
+}
+
+void TargetPassBuilder::addRegAllocPipeline(MachineFunctionPassManager &MFPM) {
+ if (CGPBO.OptimizeRegAlloc.value_or(OptLevel != CodeGenOptLevel::None)) {
+ MFPM.addPass(DetectDeadLanesPass());
+
+ MFPM.addPass(InitUndefPass());
+
+ MFPM.addPass(ProcessImplicitDefsPass());
+
+ // LiveVariables currently requires pure SSA form.
+ //
+ // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+ // LiveVariables can be removed completely, and LiveIntervals can be
+ // directly computed. (We still either need to regenerate kill flags after
+ // regalloc, or preferably fix the scavenger to not depend on them).
+ // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+ // When LiveVariables is removed this has to be removed/moved either.
+ // Explicit addition of UnreachableMachineBlockElim allows stopping before
+ // or after it with -stop-before/-stop-after.
+ MFPM.addPass(UnreachableMachineBlockElim());
+ MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>());
+
+ // Edge splitting is smarter with machine loop info.
+ MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>());
+ MFPM.addPass(PHIEliminationPass());
+
+ // Eventually, we want to run LiveIntervals before PHI elimination.
+ if (CGPBO.EarlyLiveIntervals)
+ MFPM.addPass(
+ RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
+
+ MFPM.addPass(TwoAddressInstructionPass());
+ MFPM.addPass(RegisterCoalescerPass());
+
+ // The machine scheduler may accidentally create disconnected components
+ // when moving subregister definitions around, avoid this by splitting them
+ // to separate vregs before. Splitting can also improve reg. allocation
+ // quality.
+ MFPM.addPass(RenameIndependentSubregsPass());
+
+ // PreRA instruction scheduling.
+ MFPM.addPass(MachineSchedulerPass(TM));
+
+ /// Add register assign and rewrite passes.
+ // Add the selected register allocation pass.
+ addRegAllocPass(MFPM, true);
+
+ // Finally rewrite virtual registers.
+ MFPM.addPass(VirtRegRewriterPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ MFPM.addPass(RegAllocScoringPass());
+
+ // Perform stack slot coloring and post-ra machine LICM.
+ MFPM.addPass(StackSlotColoringPass());
+
+ // Allow targets to expand pseudo instructions depending on the choice of
+ // registers before MachineCopyPropagation.
+
+ // Copy propagate to forward register uses and try to eliminate COPYs that
+ // were not coalesced.
+ MFPM.addPass(MachineCopyPropagationPass());
+
+ // Run post-ra machine LICM to hoist reloads / remats.
+ //
+ // FIXME: can this move into MachineLateOptimization?
+ MFPM.addPass(MachineLICMPass());
+ } else {
+ MFPM.addPass(PHIEliminationPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+
+ /// Add register assign and rewrite passes.
+ if (CGPBO.RegAlloc != RegAllocType::Default &&
+ CGPBO.RegAlloc != RegAllocType::Fast)
+ report_fatal_error("Must use fast (default) register allocator for "
+ "unoptimized regalloc.");
+
+ addRegAllocPass(MFPM, false);
+ }
+}
+
+void TargetPassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM,
+ bool Optimized) {
+ // TODO: Handle register allocator
+}
+
+void TargetPassBuilder::addMachineSSAOptimizationPasses(
+ MachineFunctionPassManager &MFPM) {
+ // Pre-ra tail duplication.
+ MFPM.addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ MFPM.addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a different pass
+ // which merges spill slots.
+ MFPM.addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ MFPM.addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level
+ // parallelism, like if-conversion. Such passes will typically need
+ // dominator trees and loop info, just like LICM and CSE below.
+ // MFPM.addILPOpts();
+
+ MFPM.addPass(EarlyMachineLICMPass());
+ MFPM.addPass(MachineCSEPass());
+
+ MFPM.addPass(MachineSinkingPass(CGPBO.EnableSinkAndFold));
+
+ MFPM.addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ MFPM.addPass(DeadMachineInstructionElimPass());
+}
+
+// Find the FSProfile file name. The internal option takes the precedence
+// before getting from TargetMachine.
+static std::string getFSProfileFile(const CGPassBuilderOption &CGPBO,
+ const TargetMachine *TM) {
+ if (!CGPBO.FSProfileFile.empty())
+ return CGPBO.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (!PGOOpt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+}
+
+// Find the Profile remapping file name. The internal option takes the
+// precedence before getting from TargetMachine.
+static std::string getFSRemappingFile(const CGPassBuilderOption &CGPBO,
+ const TargetMachine *TM) {
+ if (!CGPBO.FSRemappingFile.empty())
+ return CGPBO.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (!PGOOpt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+}
+
+TargetPassBuilder::ModulePassManager
+TargetPassBuilder::buildCodeGenMIRPipeline() {
+ ModulePassManager MPM;
+ MachineFunctionPassManager MFPM;
+ if (OptLevel != CodeGenOptLevel::None) {
+ addMachineSSAOptimizationPasses(MFPM);
+ } else {
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+ }
+
+ if (TM->Options.EnableIPRA)
+ MFPM.addPass(RegUsageInfoPropagationPass());
+
+ // Add a FSDiscriminator pass right before RA, so that we could get
+ // more precise SampleFDO profile for RA.
+
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::Pass1, nullptr));
+ }
+
+ // Run register allocation and passes that are tightly coupled with it,
+ // including phi elimination and scheduling.
+ addRegAllocPipeline(MFPM);
+
+ MFPM.addPass(RemoveRedundantDebugValuesPass());
+
+ MFPM.addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (OptLevel != CodeGenOptLevel::None) {
+ MFPM.addPass(PostRAMachineSinkingPass());
+ MFPM.addPass(ShrinkWrapPass());
+ }
+
+ // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
+ // do so if it hasn't been disabled, substituted, or overridden.
+ if (!CGPBO.DisablePrologEpilogInserterPass)
+ MFPM.addPass(PrologEpilogCodeInserterPass());
+
+ /// Add passes that optimize machine instructions after register
+ /// allocation.
+ if (OptLevel != CodeGenOptLevel::None) {
+ // Cleanup of redundant immediate/address loads.
+ MFPM.addPass(MachineLateInstrsCleanupPass());
+
+ // Branch folding must be run after regalloc and prolog/epilog insertion.
+ MFPM.addPass(BranchFolderPass(CGPBO.EnableTailMerge));
+
+ // Tail duplication.
+ // Note that duplicating tail just increases code size and degrades
+ // performance for targets that require Structured Control Flow.
+ // In addition it can also make CFG irreducible. Thus we disable it.
+ if (!TM->requiresStructuredCFG())
+ MFPM.addPass(TailDuplicatePass());
+
+ // Copy propagation.
+ MFPM.addPass(MachineCopyPropagationPass());
+ }
+
+ // Expand pseudo instructions before second scheduling pass.
+ MFPM.addPass(ExpandPostRAPseudosPass());
+
+ if (CGPBO.EnableImplicitNullChecks)
+ MFPM.addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other
+ // point.
+ if (OptLevel != CodeGenOptLevel::None &&
+ !TM->targetSchedulesPostRAScheduling()) {
+ if (CGPBO.MISchedPostRA)
+ MFPM.addPass(PostMachineSchedulerPass(TM));
+ else
+ MFPM.addPass(PostRASchedulerPass(TM));
+ }
+
+ // GC
+ // MFPM.addPass(GCMachineCodeAnalysis());
+
+ // Basic block placement.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2));
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::Pass2, nullptr));
+ MFPM.addPass(MachineBlockPlacementPass(CGPBO.EnableTailMerge));
+ // Run a separate pass to collect block placement statistics.
+ if (CGPBO.EnableBlockPlacementStats)
+ MFPM.addPass(MachineBlockPlacementStatsPass());
+ }
+ }
+
+ // Insert before XRay Instrumentation.
+ MFPM.addPass(FEntryInserterPass());
+
+ MFPM.addPass(XRayInstrumentationPass());
+ MFPM.addPass(PatchableFunctionPass());
+
+ if (TM->Options.EnableIPRA)
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ MFPM.addPass(RegUsageInfoCollectorPass());
+
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
+ MFPM.addPass(FuncletLayoutPass());
+
+ MFPM.addPass(RemoveLoadsIntoFakeUsesPass());
+ MFPM.addPass(StackMapLivenessPass());
+ MFPM.addPass(LiveDebugValuesPass(TM->Options.ShouldEmitDebugEntryValues()));
+ MFPM.addPass(MachineSanitizerBinaryMetadata());
+
+ if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None &&
+ CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
+ if (AddOutliner) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ MPM.addPass(MachineOutlinerPass(RunOnAllFunctions));
+ MFPM = MachineFunctionPassManager();
+ }
+ }
+
+ if (CGPBO.GCEmptyBlocks)
+ MFPM.addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ // Machine function splitter uses the basic block sections feature.
+ // When used along with `-basic-block-sections=`, the basic-block-sections
+ // feature takes precedence. This means functions eligible for
+ // basic-block-sections optimizations (`=all`, or `=list=` with function
+ // included in the list profile) will get that optimization instead.
+ if (TM->Options.EnableMachineFunctionSplitter ||
+ CGPBO.EnableMachineFunctionSplitter) {
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.\n";
+ }
+ }
+ MFPM.addPass(MachineFunctionSplitterPass());
+ if (CGPBO.SplitStaticData || TM->Options.EnableStaticDataPartitioning) {
+ // The static data splitter pass is a machine function pass. and
+ // static data annotator pass is a module-wide pass. See the file comment
+ // in StaticDataAnnotator.cpp for the motivation.
+ MFPM.addPass(StaticDataSplitterPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ MFPM = MachineFunctionPassManager();
+ MPM.addPass(StaticDataAnnotatorPass());
+ }
+ }
+ // We run the BasicBlockSections pass if either we need BB sections or BB
+ // address map (or both).
+ if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
+ TM->Options.BBAddrMap) {
+ if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
+ MFPM.addPass(
+ BasicBlockSectionsProfileReaderPass(TM->getBBSectionsFuncListBuf()));
+ MFPM.addPass(BasicBlockPathCloningPass());
+ }
+ MFPM.addPass(BasicBlockSectionsPass());
+ }
+
+ if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup)
+ MFPM.addPass(CFIFixupPass());
+
+ MFPM.addPass(StackFrameLayoutAnalysisPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ return MPM;
+}
+
+void TargetPassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) {
+ const MCAsmInfo *MCAI = TM->getMCAsmInfo();
+
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ FPM.addPass(SjLjEHPreparePass(TM));
+ [[fallthrough]];
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ case ExceptionHandling::AIX:
+ case ExceptionHandling::ZOS:
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ FPM.addPass(WinEHPreparePass());
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true));
+ FPM.addPass(WasmEHPreparePass());
+ break;
+ case ExceptionHandling::None:
+ FPM.addPass(LowerInvokePass());
+ // The lower invoke pass may create unreachable code. Remove it.
+ FPM.addPass(UnreachableBlockElimPass());
+ break;
+ }
+}
+
+void TargetPassBuilder::filtPassList(ModulePassManager &MPM) const {
+ PassList &Passes = MPM.Passes;
+ auto *PIC = PB.getPassInstrumentationCallbacks();
+ auto ESSI = TargetPassConfig::getStartStopInfo(*PIC);
+ if (!ESSI)
+ report_fatal_error(ESSI.takeError(), "invalid start stop flag!");
+ auto SSI = *ESSI;
+ size_t StartCnt = 0, StopCnt = 0;
+ bool HandledStartAfter = !SSI.StartAfter, HandledStopAfter = !SSI.StopAfter;
+ bool ShouldRemove = !SSI.StartPass.empty();
+ for (auto I = Passes.begin(), E = Passes.end(); I != E;) {
+ // Handle disabled pass firstly.
+ if (isPassDisabled(I->first)) {
+ I = Passes.erase(I);
+ continue;
+ }
+
+ auto PassName = PIC->getPassNameForClassName(I->first);
+ if (!SSI.StartPass.empty() && PassName == SSI.StartPass) {
+ ++StartCnt;
+ if (StartCnt == SSI.StartInstanceNum)
+ ShouldRemove = SSI.StartAfter;
+ }
+ if (!SSI.StopPass.empty() && PassName == SSI.StopPass) {
+ ++StopCnt;
+ if (StopCnt == SSI.StopInstanceNum)
+ ShouldRemove = !SSI.StopAfter;
+ }
+ bool Inc = true;
+ if (ShouldRemove && !PassName.starts_with("RequireAnalysisPass")) {
+ Inc = false;
+ I = Passes.erase(I);
+ }
+ if (!HandledStartAfter && !SSI.StartPass.empty() &&
+ PassName == SSI.StartPass && SSI.StartAfter &&
+ StartCnt == SSI.StartInstanceNum) {
+ HandledStartAfter = true;
+ ShouldRemove = false;
+ }
+ if (!HandledStopAfter && !SSI.StopPass.empty() &&
+ PassName == SSI.StopPass && SSI.StopAfter &&
+ StopCnt == SSI.StopInstanceNum) {
+ HandledStopAfter = true;
+ ShouldRemove = true;
+ }
+ if (Inc)
+ ++I;
+ }
+}
+
+void TargetPassBuilder::addMIRPrinterPasses(ModulePassManager &MPM) {
+ llvm::MachineFunctionPassManager MFPM;
+ MFPM.addPass(PrintMIRPass());
+ MPM.Passes.push_back(std::pair(PrintMIRPass::name(), std::move(MFPM)));
+ PassList &Passes = MPM.Passes;
+ for (auto I = Passes.rbegin(), E = Passes.rend(); I != E; ++I) {
+ // Insert PrintMIRPreparePass after the last module pass.
+ if (I->second.index() == 0) {
+ llvm::ModulePassManager PM;
+ PM.addPass(PrintMIRPreparePass());
+ Passes.insert(I.base(),
+ std::pair(PrintMIRPreparePass::name(), std::move(PM)));
+ break;
+ }
+ }
+}
+
+ModulePassManager
+TargetPassBuilder::constructRealPassManager(ModulePassManager &&MPMW) {
+ using llvm::createFunctionToLoopPassAdaptor,
+ llvm::createFunctionToMachineFunctionPassAdaptor,
+ llvm::createModuleToFunctionPassAdaptor;
+ using llvm::ModulePassManager, llvm::FunctionPassManager,
+ llvm::LoopPassManager, llvm::MachineFunctionPassManager;
+
+ ModulePassManager MPM;
+ FunctionPassManager FPM;
+ LoopPassManager LPM;
+ MachineFunctionPassManager MFPM;
+
+ std::stack<size_t> S({0});
+ for (auto &P : MPMW.Passes) {
+ auto &PMVar = P.second;
+ std::visit(
+ [&](auto &&PM) {
+ size_t VarIdx = PMVar.index();
+ while (VarIdx < S.top()) {
+ switch (S.top()) {
+ case 3:
+ if (!MFPM.isEmpty())
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(
+ std::move(MFPM)));
+ MFPM = llvm::MachineFunctionPassManager();
+ S.pop();
+ break;
+ case 2:
+ if (!LPM.isEmpty())
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ std::move(LPM), /*UseMemorySSA=*/true));
+ LPM = llvm::LoopPassManager();
+ S.pop();
+ break;
+ case 1:
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = llvm::FunctionPassManager();
+ S.pop();
+ break;
+ case 0:
+ break;
+ default:
+ llvm_unreachable("");
+ }
+ }
+ while (VarIdx > S.top())
+ S.push(S.top() + 1);
+
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ ModulePassManager>)
+ MPM.addPass(std::move(PM));
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ FunctionPassManager>)
+ FPM.addPass(std::move(PM));
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ LoopPassManager>)
+ LPM.addPass(std::move(PM));
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ MachineFunctionPassManager>)
+ MFPM.addPass(std::move(PM));
+ },
+ PMVar);
+ }
+
+ if (!MFPM.isEmpty())
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ if (!LPM.isEmpty())
+ FPM.addPass(
+ createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true));
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+ return MPM;
+}
+
+void TargetPassBuilder::invokeInjectionCallbacks(ModulePassManager &MPM) const {
+ PassList &Passes = MPM.Passes;
+ for (auto I = Passes.begin(), E = Passes.end(); I != E; ++I)
+ for (auto &C : InjectionCallbacks)
+ I = C(Passes, I);
+}
+
+void TargetPassBuilder::anchor() {}
diff --git a/llvm/unittests/CodeGen/CMakeLists.txt b/llvm/unittests/CodeGen/CMakeLists.txt
index 4f580e7539f4d..f7eacfcb2647e 100644
--- a/llvm/unittests/CodeGen/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_unittest(CodeGenTests
SelectionDAGPatternMatchTest.cpp
TypeTraitsTest.cpp
TargetOptionsTest.cpp
+ TargetPassBuilderTest.cpp
TestAsmPrinter.cpp
MLRegAllocDevelopmentFeatures.cpp
)
diff --git a/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp b/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
new file mode 100644
index 0000000000000..8ec07c37b663c
--- /dev/null
+++ b/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
@@ -0,0 +1,176 @@
+//===- TargetPassBuilderTest.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/TargetPassBuilder.h"
+#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
+#include "llvm/CodeGen/ExpandFp.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/TargetFrameLowering.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Target/TargetMachine.h"
+#include "gtest/gtest.h"
+#include <cassert>
+
+using namespace llvm;
+
+namespace {
+
+class TestTargetMachine : public CodeGenTargetMachineImpl {
+public:
+ TestTargetMachine()
+ : CodeGenTargetMachineImpl(Target(), "", Triple(""), "", "",
+ TargetOptions(), Reloc::Static,
+ CodeModel::Small, CodeGenOptLevel::Default) {
+ AsmInfo.reset(new MCAsmInfo());
+ }
+
+ ~TestTargetMachine() override = default;
+};
+
+TestTargetMachine *createTargetMachine() {
+ static TestTargetMachine TestTM;
+ return &TestTM;
+}
+
+template <size_t Tag, typename MemberPtrT> struct PrivateVisitor {
+ inline static MemberPtrT Ptr;
+};
+template <size_t Tag, auto MemberPtrV> struct PrivateVisitorHelper {
+ struct Assigner {
+ Assigner() { PrivateVisitor<Tag, decltype(MemberPtrV)>::Ptr = MemberPtrV; }
+ };
+ inline static Assigner A;
+};
+
+template <size_t Tag, typename MemberPtrT>
+MemberPtrT PrivatePtr = PrivateVisitor<Tag, MemberPtrT>::Ptr;
+
+class TestPassBuilder : public TargetPassBuilder {
+public:
+ using TargetPassBuilder::ModulePassManager;
+
+ TestPassBuilder(PassBuilder &PB) : TargetPassBuilder(PB) {
+ injectBefore<PreISelIntrinsicLoweringPass>([]() {
+ ModulePassManager MPM;
+ MPM.addPass(NoOpModulePass());
+ return MPM;
+ });
+
+ injectBefore<ExpandFpPass, ModulePassManager>([] {
+ ModulePassManager MPM;
+ MPM.addPass(NoOpModulePass());
+ return MPM;
+ });
+ }
+
+ std::unique_ptr<SelectionDAGISel> getSelectionDAGISelPass() override {
+ return nullptr;
+ }
+};
+
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::buildCodeGenIRPipeline>;
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::invokeInjectionCallbacks>;
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::ModulePassManager::Passes>;
+
+TEST(TargetPassBuilder, Basic) {
+ TargetMachine *TM = createTargetMachine();
+ PassInstrumentationCallbacks PIC;
+ PassBuilder PB(TM, PipelineTuningOptions(), std::nullopt, &PIC);
+ ModulePassManager PM;
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager AM;
+
+ /// Register builtin analyses and cross-register the analysis proxies
+ PB.registerModuleAnalyses(AM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerLoopAnalyses(LAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, AM);
+
+ using PassList = std::list<std::pair<
+ StringRef,
+ std::variant<llvm::ModulePassManager, llvm::FunctionPassManager,
+ llvm::LoopPassManager, llvm::MachineFunctionPassManager>>>;
+
+ TestPassBuilder TPB(PB);
+ auto MPM = (TPB.*PrivatePtr<0, TestPassBuilder::ModulePassManager (
+ TargetPassBuilder::*)()>)();
+ auto &Passes =
+ MPM.*PrivatePtr<0, PassList TestPassBuilder::ModulePassManager::*>;
+ (TPB.*PrivatePtr<0, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // invokeInjectionCallbacks
+ auto B = Passes.begin();
+ EXPECT_EQ(B->first, NoOpModulePass::name());
+ ++B, ++B, ++B;
+ EXPECT_EQ(B->first, NoOpModulePass::name());
+}
+
+template struct PrivateVisitorHelper<1, &TargetPassBuilder::filtPassList>;
+
+TEST(TargetPassBuilder, StartStop) {
+ const char *Argv[] = {"CodeGenTests",
+ "--start-after=pre-isel-intrinsic-lowering",
+ "--stop-before=no-op-module,2"};
+ cl::ParseCommandLineOptions(std::size(Argv), Argv);
+
+ TargetMachine *TM = createTargetMachine();
+ PassInstrumentationCallbacks PIC;
+ PassBuilder PB(TM, PipelineTuningOptions(), std::nullopt, &PIC);
+ ModulePassManager PM;
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager AM;
+
+ /// Register builtin analyses and cross-register the analysis proxies
+ PB.registerModuleAnalyses(AM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerLoopAnalyses(LAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, AM);
+
+ using PassList = std::list<std::pair<
+ StringRef,
+ std::variant<llvm::ModulePassManager, llvm::FunctionPassManager,
+ llvm::LoopPassManager, llvm::MachineFunctionPassManager>>>;
+
+ TestPassBuilder TPB(PB);
+ auto MPM = (TPB.*PrivatePtr<0, TestPassBuilder::ModulePassManager (
+ TargetPassBuilder::*)()>)();
+ auto &Passes =
+ MPM.*PrivatePtr<0, PassList TestPassBuilder::ModulePassManager::*>;
+ (TPB.*PrivatePtr<0, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // invokeInjectionCallbacks
+ (TPB.*PrivatePtr<1, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // filtPassList
+ EXPECT_EQ(Passes.size(), 1u);
+ EXPECT_EQ(Passes.begin()->first, ExpandLargeDivRemPass::name());
+}
+} // namespace
More information about the llvm-commits
mailing list