[llvm] [CodeGen][Pass] Add `TargetPassBuilder` (PR #137290)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 6 21:44:02 PDT 2025
https://github.com/paperchalice updated https://github.com/llvm/llvm-project/pull/137290
>From d30808d5e602d7e5561b33a787998a72503c2cf7 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 24 Mar 2025 19:34:01 +0800
Subject: [PATCH] Add `TargetPassBuilder`.
---
.../llvm/Passes/MachinePassRegistry.def | 3 +
llvm/include/llvm/Passes/PassBuilder.h | 2 +
llvm/include/llvm/Passes/TargetPassBuilder.h | 314 +++++++
.../include/llvm/Target/CGPassBuilderOption.h | 5 +
llvm/lib/CodeGen/TargetPassConfig.cpp | 3 +
llvm/lib/Passes/CMakeLists.txt | 1 +
llvm/lib/Passes/TargetPassBuilder.cpp | 852 ++++++++++++++++++
llvm/unittests/CodeGen/CMakeLists.txt | 1 +
.../CodeGen/TargetPassBuilderTest.cpp | 180 ++++
9 files changed, 1361 insertions(+)
create mode 100644 llvm/include/llvm/Passes/TargetPassBuilder.h
create mode 100644 llvm/lib/Passes/TargetPassBuilder.cpp
create mode 100644 llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index c69573ee3ed97..b71f3eafa7f02 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -288,11 +288,13 @@ DUMMY_MACHINE_MODULE_PASS("pseudo-probe-inserter", PseudoProbeInserterPass)
DUMMY_MACHINE_MODULE_PASS("mir-debugify", DebugifyMachineModule)
DUMMY_MACHINE_MODULE_PASS("mir-check-debugify", CheckDebugMachineModulePass)
DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
+DUMMY_MACHINE_MODULE_PASS("static-data-annotator", StaticDataAnnotatorPass)
#undef DUMMY_MACHINE_MODULE_PASS
#ifndef DUMMY_MACHINE_FUNCTION_PASS
#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
#endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
DUMMY_MACHINE_FUNCTION_PASS("break-false-deps", BreakFalseDepsPass)
@@ -325,5 +327,6 @@ DUMMY_MACHINE_FUNCTION_PASS("regallocscoringpass", RegAllocScoringPass)
DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass)
DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass)
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
+DUMMY_MACHINE_FUNCTION_PASS("static-data-splitter", StaticDataSplitterPass)
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 51ccaa53447d7..6fca52e5062e5 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -112,6 +112,8 @@ class PassBuilder {
std::optional<PGOOptions> PGOOpt;
PassInstrumentationCallbacks *PIC;
+ friend class TargetPassBuilder;
+
public:
/// A struct to capture parsed pass pipeline names.
///
diff --git a/llvm/include/llvm/Passes/TargetPassBuilder.h b/llvm/include/llvm/Passes/TargetPassBuilder.h
new file mode 100644
index 0000000000000..3ad472827b01b
--- /dev/null
+++ b/llvm/include/llvm/Passes/TargetPassBuilder.h
@@ -0,0 +1,314 @@
+//===- Parsing, selection, and construction of pass pipelines --*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Interfaces for registering analysis passes, producing common pass manager
+/// configurations, and parsing of pass pipelines.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PASSES_TARGETPASSBUILDER_H
+#define LLVM_PASSES_TARGETPASSBUILDER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/identity.h"
+#include "llvm/Analysis/CGSCCPassManager.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/Target/CGPassBuilderOption.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include <list>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace llvm {
+
+class PassBuilder;
+class TargetMachine;
+class SelectionDAGISel;
+
+class TargetPassBuilder {
+public:
+ TargetPassBuilder(PassBuilder &PB);
+
+ virtual ~TargetPassBuilder() = default;
+
+ // TODO: Add necessary parameters once AsmPrinter is ported to new pass
+ // manager.
+ llvm::ModulePassManager buildPipeline(raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx);
+
+private:
+ struct PassWrapper {
+ StringRef Name;
+ std::variant<llvm::ModulePassManager, llvm::FunctionPassManager,
+ llvm::LoopPassManager, llvm::MachineFunctionPassManager>
+ InternalPass;
+ bool InCGSCC = false;
+
+ template <typename PassManagerT>
+ PassWrapper(StringRef Name, PassManagerT &&PM)
+ : Name(Name), InternalPass(std::forward<PassManagerT>(PM)) {}
+
+ template <typename PassT> PassWrapper(PassT &&P) : Name(PassT::name()) {
+ if constexpr (isModulePass<PassT>) {
+ llvm::ModulePassManager MPM;
+ MPM.addPass(std::forward<PassT>(P));
+ InternalPass.emplace<llvm::ModulePassManager>(std::move(MPM));
+ } else if constexpr (isFunctionPass<PassT>) {
+ llvm::FunctionPassManager FPM;
+ FPM.addPass(std::forward<PassT>(P));
+ InternalPass.emplace<llvm::FunctionPassManager>(std::move(FPM));
+ } else {
+ static_assert(isMachineFunctionPass<PassT>, "Invalid pass type!");
+ llvm::MachineFunctionPassManager MFPM;
+ MFPM.addPass(std::forward<PassT>(P));
+ InternalPass.emplace<llvm::MachineFunctionPassManager>(std::move(MFPM));
+ }
+ }
+ };
+
+public:
+ using PassList = std::list<PassWrapper>;
+
+private:
+ template <typename InternalPassT> struct AdaptorWrapper : InternalPassT {
+ using InternalPassT::Passes;
+ };
+
+ template <typename PassManagerT, typename InternalPassT = void>
+ class PassManagerWrapper {
+ friend class TargetPassBuilder;
+
+ public:
+ bool isEmpty() const { return Passes.empty(); }
+
+ template <typename PassT> void addPass(PassT &&P) {
+ Passes.emplace_back(std::forward<PassT>(P));
+ }
+
+ void addPass(PassManagerWrapper &&PM) {
+ for (auto &P : PM.Passes)
+ Passes.push_back(std::move(P));
+ }
+
+ void addPass(AdaptorWrapper<InternalPassT> &&Adaptor) {
+ for (auto &P : Adaptor.Passes)
+ Passes.push_back(std::move(P));
+ }
+
+ void addPass(llvm::ModulePassManager &&) = delete;
+ void addPass(llvm::FunctionPassManager &&) = delete;
+ void addPass(llvm::LoopPassManager &&) = delete;
+ void addPass(llvm::MachineFunctionPassManager &&) = delete;
+
+ private:
+ PassList Passes;
+ };
+
+ template <typename NestedPassManagerT, typename PassT>
+ AdaptorWrapper<NestedPassManagerT> createPassAdaptor(PassT &&P) {
+ AdaptorWrapper<NestedPassManagerT> Adaptor;
+ Adaptor.addPass(std::forward<PassT>(P));
+ return Adaptor;
+ }
+
+private:
+ template <typename PassT, typename IRUnitT>
+ using HasRunOnIRUnit = decltype(std::declval<PassT>().run(
+ std::declval<IRUnitT &>(), std::declval<AnalysisManager<IRUnitT> &>()));
+ template <typename PassT>
+ static constexpr bool isModulePass =
+ is_detected<HasRunOnIRUnit, PassT, Module>::value;
+ template <typename PassT>
+ static constexpr bool isFunctionPass =
+ is_detected<HasRunOnIRUnit, PassT, Function>::value;
+ template <typename PassT>
+ static constexpr bool isMachineFunctionPass =
+ is_detected<HasRunOnIRUnit, PassT, MachineFunction>::value;
+
+protected:
+ // Hijack real pass managers intentionally.
+ using MachineFunctionPassManager =
+ PassManagerWrapper<llvm::MachineFunctionPassManager>;
+ using FunctionPassManager =
+ PassManagerWrapper<llvm::FunctionPassManager, MachineFunctionPassManager>;
+ using ModulePassManager =
+ PassManagerWrapper<llvm::ModulePassManager, FunctionPassManager>;
+
+ struct CGSCCAdaptorWrapper : AdaptorWrapper<FunctionPassManager> {};
+
+protected:
+ template <typename FunctionPassT>
+ AdaptorWrapper<FunctionPassManager>
+ createModuleToFunctionPassAdaptor(FunctionPassT &&P) {
+ return createPassAdaptor<FunctionPassManager>(
+ std::forward<FunctionPassT>(P));
+ }
+
+ AdaptorWrapper<FunctionPassManager>
+ createModuleToPostOrderCGSCCPassAdaptor(CGSCCAdaptorWrapper &&PM) {
+ AdaptorWrapper<FunctionPassManager> AW;
+ AW.Passes = std::move(PM.Passes);
+ return AW;
+ }
+
+ template <typename FunctionPassT>
+ CGSCCAdaptorWrapper createCGSCCToFunctionPassAdaptor(FunctionPassT &&PM) {
+ for (auto &P : PM.Passes)
+ P.InCGSCC = true;
+ CGSCCAdaptorWrapper AW;
+ AW.Passes = std::move(PM.Passes);
+ return AW;
+ }
+
+ template <typename MachineFunctionPassT>
+ AdaptorWrapper<MachineFunctionPassManager>
+ createFunctionToMachineFunctionPassAdaptor(MachineFunctionPassT &&P) {
+ return createPassAdaptor<MachineFunctionPassManager>(
+ std::forward<MachineFunctionPassT>(P));
+ }
+
+protected:
+ PassBuilder &PB;
+ TargetMachine *TM;
+ CodeGenOptLevel OptLevel;
+ CGPassBuilderOption CGPBO = getCGPassBuilderOption();
+
+ template <typename PassT,
+ typename PassManagerT = std::conditional_t<
+ isModulePass<PassT>, ModulePassManager,
+ std::conditional_t<isFunctionPass<PassT>, FunctionPassManager,
+ MachineFunctionPassManager>>>
+ void injectBefore(
+ typename llvm::identity<std::function<PassManagerT()>>::argument_type F) {
+ InjectionCallbacks.push_back(
+ [Accessed = false, F](PassList &Passes, PassList::iterator I) mutable {
+ if (Accessed)
+ return I;
+ if (PassT::name() != I->Name)
+ return I;
+ Accessed = true;
+ auto PMPasses = F().Passes;
+ return Passes.insert(I, std::make_move_iterator(PMPasses.begin()),
+ std::make_move_iterator(PMPasses.end()));
+ });
+ }
+
+ template <typename PassTs> void disablePass() {
+ DisabedPasses.insert(PassTs::name());
+ }
+
+ void disablePass(StringRef Name) { DisabedPasses.insert(Name); }
+
+ template <typename PassT> bool isPassDisabled() const {
+ return DisabedPasses.contains(PassT::name());
+ }
+
+ bool isPassDisabled(StringRef Name) const {
+ return DisabedPasses.contains(Name);
+ }
+
+ template <typename PassT> bool isPassEnabled() const {
+ return !isPassDisabled<PassT>();
+ }
+
+ bool isPassEnabled(StringRef Name) const { return !isPassDisabled(Name); }
+
+protected:
+ /// @brief Set target pass hook.
+ /// @param Hook An instance provide pass getters
+ /// It must implement getSelectionDAGISelPass
+ /// TODO: Add ASM printer related hooks.
+ template <typename HookT> void setTargetHook(HookT &&Hook) {
+ TH = std::make_unique<TargetHookModel<HookT>>(std::forward<HookT>(Hook));
+ }
+
+private:
+ class TargetHookConcept {
+ virtual void anchor();
+
+ public:
+ virtual void addSelectionDAGISelPass(MachineFunctionPassManager &MFPM) = 0;
+ virtual void addAsmPrinterPass(ModulePassManager &MPM) = 0;
+ virtual ~TargetHookConcept() = default;
+ };
+
+ template <typename HookT> class TargetHookModel : public TargetHookConcept {
+ HookT Hook;
+
+ public:
+ TargetHookModel(HookT &&H) : Hook(H) {}
+
+ void addSelectionDAGISelPass(MachineFunctionPassManager &MFPM) override {
+ using SelectionDAGISelPassT = decltype(Hook.getSelectionDAGISelPass());
+ static_assert(isMachineFunctionPass<SelectionDAGISelPassT>,
+ "Add machine function pass here.");
+ static_assert(!std::is_same_v<llvm::MachineFunctionPassManager,
+ SelectionDAGISelPassT>,
+ "MachineFunctionPassManager is not allowed here.");
+ MFPM.addPass(Hook.getSelectionDAGISelPass());
+ }
+
+ void addAsmPrinterPass(ModulePassManager &MPM) override {
+ // TODO: get AsmPrinterPass here.
+ }
+ };
+
+ std::unique_ptr<TargetHookConcept> TH;
+
+ void buildCoreCodeGenPipeline(ModulePassManager &MPM);
+
+ ModulePassManager buildCodeGenIRPipeline();
+
+ /// Add passes that optimize machine instructions in SSA form.
+ void addISelPasses(MachineFunctionPassManager &MFPM);
+ void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM);
+ void addRegAllocPipeline(MachineFunctionPassManager &MFPM);
+ void addRegAllocPass(MachineFunctionPassManager &MFPM, bool Optimized);
+ ModulePassManager buildCodeGenMIRPipeline();
+
+ void addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+ void filtPassList(ModulePassManager &MPM) const;
+
+ void addPrinterPasses(ModulePassManager &MPM, raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+ MCContext &Ctx);
+
+ llvm::ModulePassManager constructRealPassManager(ModulePassManager &&MPMW);
+
+private:
+ virtual void anchor();
+
+ StringSet<> DisabedPasses;
+ std::vector<std::function<PassList::iterator(PassList &, PassList::iterator)>>
+ InjectionCallbacks;
+
+ void invokeInjectionCallbacks(ModulePassManager &MPM) const;
+
+ // Only Loop Strength Reduction need this, shadow LoopPassManager
+ // in future if it is necessary.
+ template <typename PassT>
+ void addLoopPass(FunctionPassManager &FPM, PassT &&P) {
+ LoopPassManager LPM;
+ LPM.addPass(std::forward<PassT>(P));
+ FPM.Passes.push_back(PassWrapper(PassT::name(), std::move(LPM)));
+ }
+};
+
+template <> struct TargetPassBuilder::AdaptorWrapper<void> {};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h
index 51f25c1360b87..2cc2afc0b24b1 100644
--- a/llvm/include/llvm/Target/CGPassBuilderOption.h
+++ b/llvm/include/llvm/Target/CGPassBuilderOption.h
@@ -51,9 +51,11 @@ struct CGPassBuilderOption {
bool EnableMachineFunctionSplitter = false;
bool EnableSinkAndFold = false;
bool EnableTailMerge = true;
+ bool EnableLoopTermFold = false;
bool MISchedPostRA = false;
bool EarlyLiveIntervals = false;
bool GCEmptyBlocks = false;
+ bool SplitStaticData = false;
bool DisableLSR = false;
bool DisableCGP = false;
@@ -65,6 +67,9 @@ struct CGPassBuilderOption {
bool DisableExpandReductions = false;
bool DisableRAFSProfileLoader = false;
bool DisableCFIFixup = false;
+ bool DisableReplaceWithVecLib = false;
+ bool DisableLayoutFSProfileLoader = false;
+ bool DisablePrologEpilogInserterPass = false;
bool PrintAfterISel = false;
bool PrintISelInput = false;
bool RequiresCodeGenSCCOrder = false;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 0095ce3d96277..03d0889a47aca 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -506,12 +506,15 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
SET_BOOLEAN_OPTION(DisableCGP)
SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
SET_BOOLEAN_OPTION(DisableSelectOptimize)
+ SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+ SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
SET_BOOLEAN_OPTION(PrintISelInput)
SET_BOOLEAN_OPTION(DebugifyAndStripAll)
SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)
SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)
SET_BOOLEAN_OPTION(DisableCFIFixup)
SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)
+ SET_BOOLEAN_OPTION(SplitStaticData)
return Opt;
}
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 6425f4934b210..77f23b2273af1 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMPasses
PassBuilderPipelines.cpp
PassPlugin.cpp
StandardInstrumentations.cpp
+ TargetPassBuilder.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm
diff --git a/llvm/lib/Passes/TargetPassBuilder.cpp b/llvm/lib/Passes/TargetPassBuilder.cpp
new file mode 100644
index 0000000000000..f15638c162cc4
--- /dev/null
+++ b/llvm/lib/Passes/TargetPassBuilder.cpp
@@ -0,0 +1,852 @@
+//===- Construction of CodeGen pass pipelines -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/TargetPassBuilder.h"
+#include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
+#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandFp.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/FinalizeISel.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/GlobalMergeFunctions.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LocalStackSlotAllocation.h"
+#include "llvm/CodeGen/LowerEmuTLS.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
+#include "llvm/CodeGen/SafeStack.h"
+#include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
+#include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/CodeGen/WasmEHPrepare.h"
+#include "llvm/CodeGen/WinEHPrepare.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only!
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+#include <stack>
+
+using namespace llvm;
+
+TargetPassBuilder::TargetPassBuilder(PassBuilder &PB)
+ : PB(PB), TM(PB.TM), OptLevel(TM->getOptLevel()) {}
+
+ModulePassManager TargetPassBuilder::buildPipeline(raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx) {
+ ModulePassManager MPM;
+ buildCoreCodeGenPipeline(MPM);
+ invokeInjectionCallbacks(MPM);
+ filtPassList(MPM);
+ addPrinterPasses(MPM, Out, DwoOut, FileType, Ctx);
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ InvalidateAnalysisPass<MachineFunctionAnalysis>()));
+ return constructRealPassManager(std::move(MPM));
+}
+
+void TargetPassBuilder::buildCoreCodeGenPipeline(ModulePassManager &MPM) {
+ MPM.addPass(buildCodeGenIRPipeline());
+ MachineFunctionPassManager MFPM;
+ addISelPasses(MFPM);
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ MPM.addPass(buildCodeGenMIRPipeline());
+}
+
+TargetPassBuilder::ModulePassManager
+TargetPassBuilder::buildCodeGenIRPipeline() {
+ ModulePassManager MPM;
+ if (TM->useEmulatedTLS())
+ MPM.addPass(LowerEmuTLSPass());
+ MPM.addPass(PreISelIntrinsicLoweringPass(TM));
+ FunctionPassManager FPM;
+ FPM.addPass(ExpandLargeDivRemPass(TM));
+ FPM.addPass(ExpandFpPass(TM));
+
+ // Run loop strength reduction before anything else.
+ if (TM->getOptLevel() == CodeGenOptLevel::None) {
+ // Basic AliasAnalysis support.
+ // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
+ // BasicAliasAnalysis wins if they disagree. This is intended to help
+ // support "obvious" type-punning idioms.
+ FPM.addPass(RequireAnalysisPass<TypeBasedAA, Function>());
+ FPM.addPass(RequireAnalysisPass<ScopedNoAliasAA, Function>());
+ FPM.addPass(RequireAnalysisPass<BasicAA, Function>());
+
+ if (!CGPBO.DisableLSR) {
+ addLoopPass(FPM, CanonicalizeFreezeInLoopsPass());
+ addLoopPass(FPM, LoopStrengthReducePass());
+ if (CGPBO.EnableLoopTermFold)
+ addLoopPass(FPM, LoopTermFoldPass());
+ }
+
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences
+ // of loads and compares. ExpandMemCmpPass then tries to expand those
+ // calls into optimally-sized loads and compares. The transforms are
+ // enabled by a target lowering hook.
+ if (!CGPBO.DisableMergeICmps)
+ FPM.addPass(MergeICmpsPass());
+ FPM.addPass(ExpandMemCmpPass(TM));
+ }
+
+ // Run GC lowering passes for builtin collectors
+ FPM.addPass(GCLoweringPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = FunctionPassManager();
+ MPM.addPass(ShadowStackGCLoweringPass());
+ // PB.invokeGCLoweringEPCallbacks();
+
+ if (TM->getTargetTriple().isOSBinFormatMachO() &&
+ !CGPBO.DisableAtExitBasedGlobalDtorLowering) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = FunctionPassManager();
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(LowerGlobalDtorsPass());
+ }
+
+ // Make sure that no unreachable blocks are instruction selected.
+ FPM.addPass(UnreachableBlockElimPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableConstantHoisting)
+ FPM.addPass(ConstantHoistingPass());
+ if (!CGPBO.DisableReplaceWithVecLib)
+ FPM.addPass(ReplaceWithVeclib());
+ if (!CGPBO.DisablePartialLibcallInlining)
+ FPM.addPass(PartiallyInlineLibCallsPass());
+ }
+
+ // Instrument function entry after all inlining.
+ FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ FPM.addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants
+ // to. Allow disabling it for testing purposes.
+ if (!CGPBO.DisableExpandReductions)
+ FPM.addPass(ExpandReductionsPass());
+
+ // Convert conditional moves to conditional jumps when profitable.
+ if (OptLevel != CodeGenOptLevel::None && !CGPBO.DisableSelectOptimize)
+ FPM.addPass(SelectOptimizePass(TM));
+
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ if (CGPBO.EnableGlobalMergeFunc)
+ MPM.addPass(GlobalMergeFuncPass());
+
+ FPM.addPass(CodeGenPreparePass(PB.TM));
+ addExceptionHandlingPasses(FPM);
+
+ // Add common passes that perform LLVM IR to IR transforms in preparation for
+ // instruction selection.
+ if (CGPBO.RequiresCodeGenSCCOrder) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ FPM = FunctionPassManager();
+ }
+
+ // Now we need to force codegen to run according to the callgraph if target
+ // requires it.
+ if (OptLevel != CodeGenOptLevel::None)
+ FPM.addPass(ObjCARCContractPass());
+
+ FPM.addPass(CallBrPreparePass());
+
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ FPM.addPass(SafeStackPass(TM));
+ FPM.addPass(StackProtectorPass(TM));
+
+ // All passes which modify the LLVM IR are now complete; run the verifier
+ // to ensure that the IR is valid.
+ if (!CGPBO.DisableVerify)
+ FPM.addPass(VerifierPass());
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ return MPM;
+}
+
+void TargetPassBuilder::addISelPasses(MachineFunctionPassManager &MFPM) {
+ // Add core instruction selection passes.
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true));
+
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ if (CGPBO.EnableFastISelOption.value_or(false))
+ Selector = SelectorType::FastISel;
+ else if (CGPBO.EnableGlobalISelOption.value_or(false) ||
+ (TM->Options.EnableGlobalISel &&
+ CGPBO.EnableGlobalISelOption.value_or(true)))
+ Selector = SelectorType::GlobalISel;
+ else if (TM->getOptLevel() == CodeGenOptLevel::None &&
+ TM->getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM->setFastISel(true);
+ TM->setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM->setFastISel(false);
+ TM->setGlobalISel(true);
+ }
+
+ // FIXME: Currently debugify is not support in new pass manager,
+ // because it inserts module passes between each pass.
+
+ // Add instruction selector passes for global isel if enabled.
+ if (Selector == SelectorType::GlobalISel) {
+ MFPM.addPass(IRTranslatorPass());
+ MFPM.addPass(RegBankSelectPass());
+ MFPM.addPass(InstructionSelectPass(OptLevel));
+ }
+
+ // Pass to reset the MachineFunction if the ISel failed. Outside of the
+ // above if so that the verifier is not added to it.
+ MFPM.addPass(ResetMachineFunctionPass());
+
+ // Run the SDAG InstSelector, providing a fallback path when we do not want
+ // to abort on not-yet-supported input.
+ if (Selector != SelectorType::GlobalISel ||
+ TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) {
+ TH->addSelectionDAGISelPass(MFPM);
+ }
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier
+ // before FinalizeISel.
+ MFPM.addPass(FinalizeISelPass());
+}
+
+void TargetPassBuilder::addRegAllocPipeline(MachineFunctionPassManager &MFPM) {
+ if (CGPBO.OptimizeRegAlloc.value_or(OptLevel != CodeGenOptLevel::None)) {
+ MFPM.addPass(DetectDeadLanesPass());
+
+ MFPM.addPass(InitUndefPass());
+
+ MFPM.addPass(ProcessImplicitDefsPass());
+
+ // LiveVariables currently requires pure SSA form.
+ //
+ // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+ // LiveVariables can be removed completely, and LiveIntervals can be
+ // directly computed. (We still either need to regenerate kill flags after
+ // regalloc, or preferably fix the scavenger to not depend on them).
+ // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+ // When LiveVariables is removed this has to be removed/moved either.
+ // Explicit addition of UnreachableMachineBlockElim allows stopping before
+ // or after it with -stop-before/-stop-after.
+ MFPM.addPass(UnreachableMachineBlockElimPass());
+ MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>());
+
+ // Edge splitting is smarter with machine loop info.
+ MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>());
+ MFPM.addPass(PHIEliminationPass());
+
+ // Eventually, we want to run LiveIntervals before PHI elimination.
+ if (CGPBO.EarlyLiveIntervals)
+ MFPM.addPass(
+ RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
+
+ MFPM.addPass(TwoAddressInstructionPass());
+ MFPM.addPass(RegisterCoalescerPass());
+
+ // The machine scheduler may accidentally create disconnected components
+ // when moving subregister definitions around, avoid this by splitting them
+ // to separate vregs before. Splitting can also improve reg. allocation
+ // quality.
+ MFPM.addPass(RenameIndependentSubregsPass());
+
+ // PreRA instruction scheduling.
+ MFPM.addPass(MachineSchedulerPass(TM));
+
+ /// Add register assign and rewrite passes.
+ // Add the selected register allocation pass.
+ addRegAllocPass(MFPM, true);
+
+ // Finally rewrite virtual registers.
+ MFPM.addPass(VirtRegRewriterPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ MFPM.addPass(RegAllocScoringPass());
+
+ // Perform stack slot coloring and post-ra machine LICM.
+ MFPM.addPass(StackSlotColoringPass());
+
+ // Allow targets to expand pseudo instructions depending on the choice of
+ // registers before MachineCopyPropagation.
+
+ // Copy propagate to forward register uses and try to eliminate COPYs that
+ // were not coalesced.
+ MFPM.addPass(MachineCopyPropagationPass());
+
+ // Run post-ra machine LICM to hoist reloads / remats.
+ //
+ // FIXME: can this move into MachineLateOptimization?
+ MFPM.addPass(MachineLICMPass());
+ } else {
+ MFPM.addPass(PHIEliminationPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+
+ /// Add register assign and rewrite passes.
+ if (CGPBO.RegAlloc != RegAllocType::Default &&
+ CGPBO.RegAlloc != RegAllocType::Fast)
+ report_fatal_error("Must use fast (default) register allocator for "
+ "unoptimized regalloc.");
+
+ addRegAllocPass(MFPM, false);
+ }
+}
+
+void TargetPassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM,
+ bool Optimized) {
+ // TODO: Handle register allocator
+}
+
+void TargetPassBuilder::addMachineSSAOptimizationPasses(
+ MachineFunctionPassManager &MFPM) {
+ // Pre-ra tail duplication.
+ MFPM.addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ MFPM.addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a different pass
+ // which merges spill slots.
+ MFPM.addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ MFPM.addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level
+ // parallelism, like if-conversion. Such passes will typically need
+ // dominator trees and loop info, just like LICM and CSE below.
+ // MFPM.addILPOpts();
+
+ MFPM.addPass(EarlyMachineLICMPass());
+ MFPM.addPass(MachineCSEPass());
+
+ MFPM.addPass(MachineSinkingPass(CGPBO.EnableSinkAndFold));
+
+ MFPM.addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ MFPM.addPass(DeadMachineInstructionElimPass());
+}
+
+// Find the FSProfile file name. The internal option takes the precedence
+// before getting from TargetMachine.
+static std::string getFSProfileFile(const CGPassBuilderOption &CGPBO,
+ const TargetMachine *TM) {
+ if (!CGPBO.FSProfileFile.empty())
+ return CGPBO.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (!PGOOpt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+}
+
+// Find the Profile remapping file name. The internal option takes the
+// precedence before getting from TargetMachine.
+static std::string getFSRemappingFile(const CGPassBuilderOption &CGPBO,
+ const TargetMachine *TM) {
+ if (!CGPBO.FSRemappingFile.empty())
+ return CGPBO.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (!PGOOpt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+}
+
+TargetPassBuilder::ModulePassManager
+TargetPassBuilder::buildCodeGenMIRPipeline() {
+ ModulePassManager MPM;
+ MachineFunctionPassManager MFPM;
+ if (OptLevel != CodeGenOptLevel::None) {
+ addMachineSSAOptimizationPasses(MFPM);
+ } else {
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+ }
+
+ if (TM->Options.EnableIPRA)
+ MFPM.addPass(RegUsageInfoPropagationPass());
+
+ // Add a FSDiscriminator pass right before RA, so that we could get
+ // more precise SampleFDO profile for RA.
+
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::Pass1, nullptr));
+ }
+
+ // Run register allocation and passes that are tightly coupled with it,
+ // including phi elimination and scheduling.
+ addRegAllocPipeline(MFPM);
+
+ MFPM.addPass(RemoveRedundantDebugValuesPass());
+
+ MFPM.addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (OptLevel != CodeGenOptLevel::None) {
+ MFPM.addPass(PostRAMachineSinkingPass());
+ MFPM.addPass(ShrinkWrapPass());
+ }
+
+ // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
+ // do so if it hasn't been disabled, substituted, or overridden.
+ if (!CGPBO.DisablePrologEpilogInserterPass)
+ MFPM.addPass(PrologEpilogCodeInserterPass());
+
+ /// Add passes that optimize machine instructions after register
+ /// allocation.
+ if (OptLevel != CodeGenOptLevel::None) {
+ // Cleanup of redundant immediate/address loads.
+ MFPM.addPass(MachineLateInstrsCleanupPass());
+
+ // Branch folding must be run after regalloc and prolog/epilog insertion.
+ MFPM.addPass(BranchFolderPass(CGPBO.EnableTailMerge));
+
+ // Tail duplication.
+ // Note that duplicating tail just increases code size and degrades
+ // performance for targets that require Structured Control Flow.
+ // In addition it can also make CFG irreducible. Thus we disable it.
+ if (!TM->requiresStructuredCFG())
+ MFPM.addPass(TailDuplicatePass());
+
+ // Copy propagation.
+ MFPM.addPass(MachineCopyPropagationPass());
+ }
+
+ // Expand pseudo instructions before second scheduling pass.
+ MFPM.addPass(ExpandPostRAPseudosPass());
+
+ if (CGPBO.EnableImplicitNullChecks)
+ MFPM.addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other
+ // point.
+ if (OptLevel != CodeGenOptLevel::None &&
+ !TM->targetSchedulesPostRAScheduling()) {
+ if (CGPBO.MISchedPostRA)
+ MFPM.addPass(PostMachineSchedulerPass(TM));
+ else
+ MFPM.addPass(PostRASchedulerPass(TM));
+ }
+
+ // GC
+ // MFPM.addPass(GCMachineCodeAnalysis());
+
+ // Basic block placement.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2));
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::Pass2, nullptr));
+ MFPM.addPass(MachineBlockPlacementPass(CGPBO.EnableTailMerge));
+ // Run a separate pass to collect block placement statistics.
+ if (CGPBO.EnableBlockPlacementStats)
+ MFPM.addPass(MachineBlockPlacementStatsPass());
+ }
+ }
+
+ // Insert before XRay Instrumentation.
+ MFPM.addPass(FEntryInserterPass());
+
+ MFPM.addPass(XRayInstrumentationPass());
+ MFPM.addPass(PatchableFunctionPass());
+
+ if (TM->Options.EnableIPRA)
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ MFPM.addPass(RegUsageInfoCollectorPass());
+
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
+ MFPM.addPass(FuncletLayoutPass());
+
+ MFPM.addPass(RemoveLoadsIntoFakeUsesPass());
+ MFPM.addPass(StackMapLivenessPass());
+ MFPM.addPass(LiveDebugValuesPass(TM->Options.ShouldEmitDebugEntryValues()));
+ MFPM.addPass(MachineSanitizerBinaryMetadataPass());
+
+ if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None &&
+ CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
+ if (AddOutliner) {
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ MPM.addPass(MachineOutlinerPass(RunOnAllFunctions));
+ MFPM = MachineFunctionPassManager();
+ }
+ }
+
+ if (CGPBO.GCEmptyBlocks)
+ MFPM.addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ // Machine function splitter uses the basic block sections feature.
+ // When used along with `-basic-block-sections=`, the basic-block-sections
+ // feature takes precedence. This means functions eligible for
+ // basic-block-sections optimizations (`=all`, or `=list=` with function
+ // included in the list profile) will get that optimization instead.
+ if (TM->Options.EnableMachineFunctionSplitter ||
+ CGPBO.EnableMachineFunctionSplitter) {
+ const std::string ProfileFile = getFSProfileFile(CGPBO, TM);
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(CGPBO, TM),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.\n";
+ }
+ }
+ MFPM.addPass(MachineFunctionSplitterPass());
+ if (CGPBO.SplitStaticData || TM->Options.EnableStaticDataPartitioning) {
+ // The static data splitter pass is a machine function pass. and
+ // static data annotator pass is a module-wide pass. See the file comment
+ // in StaticDataAnnotator.cpp for the motivation.
+ MFPM.addPass(StaticDataSplitterPass());
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ MFPM = MachineFunctionPassManager();
+ MPM.addPass(StaticDataAnnotatorPass());
+ }
+ }
+ // We run the BasicBlockSections pass if either we need BB sections or BB
+ // address map (or both).
+ if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
+ TM->Options.BBAddrMap) {
+ if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
+ MFPM.addPass(
+ BasicBlockSectionsProfileReaderPass(TM->getBBSectionsFuncListBuf()));
+ MFPM.addPass(BasicBlockPathCloningPass());
+ }
+ MFPM.addPass(BasicBlockSectionsPass());
+ }
+
+ if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup)
+ MFPM.addPass(CFIFixupPass());
+
+ MFPM.addPass(StackFrameLayoutAnalysisPass());
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
+ return MPM;
+}
+
+void TargetPassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) {
+ const MCAsmInfo *MCAI = TM->getMCAsmInfo();
+
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ FPM.addPass(SjLjEHPreparePass(TM));
+ [[fallthrough]];
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ case ExceptionHandling::AIX:
+ case ExceptionHandling::ZOS:
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ FPM.addPass(WinEHPreparePass());
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true));
+ FPM.addPass(WasmEHPreparePass());
+ break;
+ case ExceptionHandling::None:
+ FPM.addPass(LowerInvokePass());
+ // The lower invoke pass may create unreachable code. Remove it.
+ FPM.addPass(UnreachableBlockElimPass());
+ break;
+ }
+}
+
+void TargetPassBuilder::filtPassList(ModulePassManager &MPM) const {
+ PassList &Passes = MPM.Passes;
+ auto *PIC = PB.getPassInstrumentationCallbacks();
+ auto ESSI = TargetPassConfig::getStartStopInfo(*PIC);
+ if (!ESSI)
+ report_fatal_error(ESSI.takeError(), "invalid start stop flag!");
+ auto SSI = *ESSI;
+ size_t StartCnt = 0, StopCnt = 0;
+ bool HandledStartAfter = !SSI.StartAfter, HandledStopAfter = !SSI.StopAfter;
+ bool ShouldRemove = !SSI.StartPass.empty();
+ for (auto I = Passes.begin(), E = Passes.end(); I != E;) {
+ // Handle disabled pass firstly.
+ if (isPassDisabled(I->Name)) {
+ I = Passes.erase(I);
+ continue;
+ }
+
+ auto PassName = PIC->getPassNameForClassName(I->Name);
+ if (!SSI.StartPass.empty() && PassName == SSI.StartPass) {
+ ++StartCnt;
+ if (StartCnt == SSI.StartInstanceNum)
+ ShouldRemove = SSI.StartAfter;
+ }
+ if (!SSI.StopPass.empty() && PassName == SSI.StopPass) {
+ ++StopCnt;
+ if (StopCnt == SSI.StopInstanceNum)
+ ShouldRemove = !SSI.StopAfter;
+ }
+ bool Inc = true;
+ if (ShouldRemove && !PassName.starts_with("RequireAnalysisPass")) {
+ Inc = false;
+ I = Passes.erase(I);
+ }
+ if (!HandledStartAfter && !SSI.StartPass.empty() &&
+ PassName == SSI.StartPass && SSI.StartAfter &&
+ StartCnt == SSI.StartInstanceNum) {
+ HandledStartAfter = true;
+ ShouldRemove = false;
+ }
+ if (!HandledStopAfter && !SSI.StopPass.empty() &&
+ PassName == SSI.StopPass && SSI.StopAfter &&
+ StopCnt == SSI.StopInstanceNum) {
+ HandledStopAfter = true;
+ ShouldRemove = true;
+ }
+ if (Inc)
+ ++I;
+ }
+}
+
+void TargetPassBuilder::addPrinterPasses(ModulePassManager &MPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx) {
+
+ PassList &Passes = MPM.Passes;
+ PassList::iterator LastModulePassInsertPoint =
+ llvm::find_if(llvm::reverse(Passes), [](PassWrapper &PW) {
+ return PW.InternalPass.index() == 0; // The last module pass
+ }).base();
+
+ // FIXME: CodeGenFileType here is not enough, we need an output type for MC
+ if (TargetPassConfig::willCompleteCodeGenPipeline()) {
+ auto MCStreamerOrErr = TM->createMCStreamer(Out, DwoOut, FileType, Ctx);
+ if (!MCStreamerOrErr)
+ report_fatal_error("Couldn't create MCStreamer for target!");
+ [[maybe_unused]] auto &Streamer = **MCStreamerOrErr;
+ // TODO: Passes.insert(LastModulePassInsertPoint,
+ // PassWrapper(AsmPrinterInitializePass(Out)));
+ Passes.emplace_back(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+ // TODO: Passes.emplace_back(PassWrapper(AsmPrinterFinalizePass(Out)));
+ report_fatal_error("not implemented.");
+ } else {
+ Passes.insert(LastModulePassInsertPoint,
+ PassWrapper(PrintMIRPreparePass(Out)));
+ Passes.emplace_back(PrintMIRPass(Out));
+ Passes.emplace_back(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+ }
+}
+
+llvm::ModulePassManager
+TargetPassBuilder::constructRealPassManager(ModulePassManager &&MPMW) {
+ llvm::ModulePassManager MPM;
+ llvm::FunctionPassManager FPM;
+ llvm::LoopPassManager LPM;
+ llvm::MachineFunctionPassManager MFPM;
+
+ std::stack<size_t> S({0});
+ for (auto &P : MPMW.Passes) {
+ auto &PMVar = P.InternalPass;
+ bool InCGSCC = false, LastInCGSCC = false;
+ std::visit(
+ [&](auto &&PM) {
+ if (P.InCGSCC)
+ InCGSCC = true;
+ size_t VarIdx = PMVar.index();
+ while (VarIdx < S.top()) {
+ switch (S.top()) {
+ case 3:
+ if (!MFPM.isEmpty())
+ FPM.addPass(llvm::createFunctionToMachineFunctionPassAdaptor(
+ std::move(MFPM)));
+ MFPM = llvm::MachineFunctionPassManager();
+ S.pop();
+ break;
+ case 2:
+ if (!LPM.isEmpty())
+ FPM.addPass(llvm::createFunctionToLoopPassAdaptor(
+ std::move(LPM), /*UseMemorySSA=*/true));
+ LPM = llvm::LoopPassManager();
+ S.pop();
+ break;
+ case 1:
+ if (!FPM.isEmpty()) {
+ if (CGPBO.RequiresCodeGenSCCOrder && InCGSCC) {
+ InCGSCC = false;
+ MPM.addPass(llvm::createModuleToPostOrderCGSCCPassAdaptor(
+ llvm::createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ } else {
+ MPM.addPass(
+ llvm::createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+ }
+ FPM = llvm::FunctionPassManager();
+ S.pop();
+ break;
+ case 0:
+ break;
+ default:
+ llvm_unreachable("Invalid pass manager type!");
+ }
+ }
+ while (VarIdx > S.top())
+ S.push(S.top() + 1);
+
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ llvm::ModulePassManager>)
+ MPM.addPass(std::move(PM));
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ llvm::FunctionPassManager>) {
+ if (!LastInCGSCC && P.InCGSCC) {
+ MPM.addPass(llvm::createModuleToPostOrderCGSCCPassAdaptor(
+ llvm::createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ FPM = llvm::FunctionPassManager();
+ }
+ FPM.addPass(std::move(PM));
+ }
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ llvm::LoopPassManager>)
+ LPM.addPass(std::move(PM));
+ if constexpr (std::is_same_v<std::remove_reference_t<decltype(PM)>,
+ llvm::MachineFunctionPassManager>)
+ MFPM.addPass(std::move(PM));
+
+ LastInCGSCC = P.InCGSCC;
+ },
+ PMVar);
+ }
+
+ if (!MFPM.isEmpty())
+ FPM.addPass(
+ llvm::createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ if (!LPM.isEmpty())
+ FPM.addPass(llvm::createFunctionToLoopPassAdaptor(std::move(LPM),
+ /*UseMemorySSA=*/true));
+ if (!FPM.isEmpty()) {
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(llvm::createModuleToPostOrderCGSCCPassAdaptor(
+ llvm::createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+
+ return MPM;
+}
+
+void TargetPassBuilder::invokeInjectionCallbacks(ModulePassManager &MPM) const {
+ PassList &Passes = MPM.Passes;
+ for (auto I = Passes.begin(), E = Passes.end(); I != E; ++I)
+ for (auto &C : InjectionCallbacks)
+ I = C(Passes, I);
+}
+
+void TargetPassBuilder::anchor() {}
+void TargetPassBuilder::TargetHookConcept::anchor() {}
diff --git a/llvm/unittests/CodeGen/CMakeLists.txt b/llvm/unittests/CodeGen/CMakeLists.txt
index a972dc32c40a2..7291bee8b42df 100644
--- a/llvm/unittests/CodeGen/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_unittest(CodeGenTests
SelectionDAGPatternMatchTest.cpp
TypeTraitsTest.cpp
TargetOptionsTest.cpp
+ TargetPassBuilderTest.cpp
TestAsmPrinter.cpp
MLRegAllocDevelopmentFeatures.cpp
X86MCInstLowerTest.cpp
diff --git a/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp b/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
new file mode 100644
index 0000000000000..c16402f9c60a3
--- /dev/null
+++ b/llvm/unittests/CodeGen/TargetPassBuilderTest.cpp
@@ -0,0 +1,180 @@
+//===- TargetPassBuilderTest.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/TargetPassBuilder.h"
+#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
+#include "llvm/CodeGen/ExpandFp.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/TargetFrameLowering.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Target/TargetMachine.h"
+#include "gtest/gtest.h"
+#include <cassert>
+
+using namespace llvm;
+
+namespace {
+
+class TestTargetMachine : public CodeGenTargetMachineImpl {
+public:
+ TestTargetMachine()
+ : CodeGenTargetMachineImpl(Target(), "", Triple(""), "", "",
+ TargetOptions(), Reloc::Static,
+ CodeModel::Small, CodeGenOptLevel::Default) {
+ AsmInfo.reset(new MCAsmInfo());
+ }
+
+ ~TestTargetMachine() override = default;
+};
+
+TestTargetMachine *createTargetMachine() {
+ static TestTargetMachine TestTM;
+ return &TestTM;
+}
+
+template <size_t Tag, typename MemberPtrT> struct PrivateVisitor {
+ inline static MemberPtrT Ptr;
+};
+template <size_t Tag, auto MemberPtrV> struct PrivateVisitorHelper {
+ struct Assigner {
+ Assigner() { PrivateVisitor<Tag, decltype(MemberPtrV)>::Ptr = MemberPtrV; }
+ };
+ inline static Assigner A;
+};
+
+template <size_t Tag, typename MemberPtrT>
+MemberPtrT PrivatePtr = PrivateVisitor<Tag, MemberPtrT>::Ptr;
+
+struct TestDAGISelPass : public PassInfoMixin<TestDAGISelPass> {
+ PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) {
+ return PreservedAnalyses::all();
+ }
+};
+
+class TestPassBuilder : public TargetPassBuilder {
+public:
+ using TargetPassBuilder::ModulePassManager;
+
+ TestPassBuilder(PassBuilder &PB) : TargetPassBuilder(PB) {
+
+ struct TestTargetHook {
+ TestDAGISelPass getSelectionDAGISelPass() const {
+ return TestDAGISelPass();
+ }
+ } Hook;
+ setTargetHook(std::move(Hook));
+ CGPBO.RequiresCodeGenSCCOrder = true;
+
+ injectBefore<PreISelIntrinsicLoweringPass>([]() {
+ ModulePassManager MPM;
+ MPM.addPass(NoOpModulePass());
+ return MPM;
+ });
+
+ injectBefore<ExpandFpPass, ModulePassManager>([] {
+ ModulePassManager MPM;
+ MPM.addPass(NoOpModulePass());
+ return MPM;
+ });
+ }
+};
+
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::buildCodeGenIRPipeline>;
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::invokeInjectionCallbacks>;
+template struct PrivateVisitorHelper<
+ 0, &TargetPassBuilder::ModulePassManager::Passes>;
+
+TEST(TargetPassBuilder, Basic) {
+ TargetMachine *TM = createTargetMachine();
+ PassInstrumentationCallbacks PIC;
+ PassBuilder PB(TM, PipelineTuningOptions(), std::nullopt, &PIC);
+ ModulePassManager PM;
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager AM;
+
+ /// Register builtin analyses and cross-register the analysis proxies
+ PB.registerModuleAnalyses(AM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerLoopAnalyses(LAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, AM);
+
+ TestPassBuilder TPB(PB);
+ using PassList = TargetPassBuilder::PassList;
+ auto MPM = (TPB.*PrivatePtr<0, TestPassBuilder::ModulePassManager (
+ TargetPassBuilder::*)()>)();
+ auto &Passes =
+ MPM.*PrivatePtr<0, PassList TestPassBuilder::ModulePassManager::*>;
+ (TPB.*PrivatePtr<0, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // invokeInjectionCallbacks
+ auto B = Passes.begin();
+ EXPECT_EQ(B->Name, NoOpModulePass::name());
+ ++B, ++B, ++B;
+ EXPECT_EQ(B->Name, NoOpModulePass::name());
+}
+
+template struct PrivateVisitorHelper<1, &TargetPassBuilder::filtPassList>;
+
+TEST(TargetPassBuilder, StartStop) {
+ const char *Argv[] = {"CodeGenTests",
+ "--start-after=pre-isel-intrinsic-lowering",
+ "--stop-before=no-op-module,2"};
+ cl::ParseCommandLineOptions(std::size(Argv), Argv);
+
+ TargetMachine *TM = createTargetMachine();
+ PassInstrumentationCallbacks PIC;
+ PassBuilder PB(TM, PipelineTuningOptions(), std::nullopt, &PIC);
+ ModulePassManager PM;
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager AM;
+
+ /// Register builtin analyses and cross-register the analysis proxies
+ PB.registerModuleAnalyses(AM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerLoopAnalyses(LAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, AM);
+
+ using PassList = TargetPassBuilder::PassList;
+
+ TestPassBuilder TPB(PB);
+ auto MPM = (TPB.*PrivatePtr<0, TestPassBuilder::ModulePassManager (
+ TargetPassBuilder::*)()>)();
+ auto &Passes =
+ MPM.*PrivatePtr<0, PassList TestPassBuilder::ModulePassManager::*>;
+ (TPB.*PrivatePtr<0, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // invokeInjectionCallbacks
+ (TPB.*PrivatePtr<1, void (TargetPassBuilder::*)(
+ TestPassBuilder::ModulePassManager &)
+ const>)(MPM); // filtPassList
+ EXPECT_EQ(Passes.size(), 1u);
+ EXPECT_EQ(Passes.begin()->Name, ExpandLargeDivRemPass::name());
+}
+} // namespace
More information about the llvm-commits
mailing list