[llvm] [NewPM][CodeGen][WIP] Add callback style CodeGen pass pipeline builder (PR #104725)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 18 17:49:08 PDT 2024
https://github.com/paperchalice created https://github.com/llvm/llvm-project/pull/104725
In contrast to #89708, this pull request demonstrates a callback style codegen pipeline builder. Suggestions are welcome!
It supports `-start/stop-after/before` by adding `eraseIf` in pass manager and pass adaptor. To extend pass pipeline, targets can use following extension points:
- GCLoweringEPCallbacks
- ISelPrepareEPCallbacks
- MachineSSAOptimizationEarlyEPCallbacks
- MachineSSAOptimizationLastEPCallbacks
- PreRegAllocEPCallbacks
- PreRegBankSelectEPCallbacks
- PreGlobalInstructionSelectEPCallbacks
- PostGlobalInstructionSelectEPCallbacks
- ILPOptsEPCallbacks
- MachineLateOptimizationEPCallbacks
- MIEmitEPCallbacks
- PreEmitEPCallbacks
- PostRegAllocEPCallbacks
- PreSched2EPCallbacks
- PostBBSectionsEPCallbacks
Most of them are from `TargetPassConfig`, and this should be sufficient for most targets, except AArch64 and DirectX, because AArch64 tries to insert module pass in machine function pass pipeline, DirectX builds its own codegen pass pipeline.
Some features are still work in progress, like register allocator, because there are many ad hoc pipeline patches in some targets, e.g. NVPTX, RISCV.
>From 845598a907c7333f06d3852fb6f4dd50f382bc11 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 19 Aug 2024 08:19:28 +0800
Subject: [PATCH 1/2] Support eraseIf in pass manager
---
llvm/include/llvm/Analysis/CGSCCPassManager.h | 2 ++
.../include/llvm/CodeGen/MachinePassManager.h | 2 ++
llvm/include/llvm/IR/PassManager.h | 14 +++++++++++++
llvm/include/llvm/IR/PassManagerInternal.h | 20 +++++++++++++++++++
.../llvm/Transforms/Scalar/LoopPassManager.h | 2 ++
5 files changed, 40 insertions(+)
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 406d3492136fcc..4f4b35217f414e 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -347,6 +347,8 @@ class ModuleToPostOrderCGSCCPassAdaptor
static bool isRequired() { return true; }
+ void eraseIf(function_ref<bool(StringRef)> Pred) { Pass->eraseIf(Pred); }
+
private:
std::unique_ptr<PassConceptT> Pass;
};
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 253fabdac0019d..4baf7c877adf63 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -207,6 +207,8 @@ class FunctionToMachineFunctionPassAdaptor
static bool isRequired() { return true; }
+ void eraseIf(function_ref<bool(StringRef)> Pred) { Pass->eraseIf(Pred); }
+
private:
std::unique_ptr<PassConceptT> Pass;
};
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index d269221fac0701..513add6f4243e5 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -218,6 +218,18 @@ class PassManager : public PassInfoMixin<
static bool isRequired() { return true; }
+ /// Erase all passes that satisfy the predicate \p Pred.
+ /// For internal use only!
+ void eraseIf(function_ref<bool(StringRef)> Pred) {
+ for (auto I = Passes.begin(); I != Passes.end();) {
+ (*I)->eraseIf(Pred);
+ if (Pred((*I)->name()) && !(*I)->isRequired())
+ I = Passes.erase(I);
+ else
+ ++I;
+ }
+ }
+
protected:
using PassConceptT =
detail::PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...>;
@@ -836,6 +848,8 @@ class ModuleToFunctionPassAdaptor
static bool isRequired() { return true; }
+ void eraseIf(function_ref<bool(StringRef)> Pred) { Pass->eraseIf(Pred); }
+
private:
std::unique_ptr<PassConceptT> Pass;
bool EagerlyInvalidate;
diff --git a/llvm/include/llvm/IR/PassManagerInternal.h b/llvm/include/llvm/IR/PassManagerInternal.h
index 4ada6ee5dd6831..4b05d2da9b4846 100644
--- a/llvm/include/llvm/IR/PassManagerInternal.h
+++ b/llvm/include/llvm/IR/PassManagerInternal.h
@@ -59,6 +59,8 @@ struct PassConcept {
/// To opt-in, pass should implement `static bool isRequired()`. It's no-op
/// to have `isRequired` always return false since that is the default.
virtual bool isRequired() const = 0;
+
+ virtual void eraseIf(function_ref<bool(StringRef)> Pred) = 0;
};
/// A template wrapper used to implement the polymorphic API.
@@ -114,6 +116,24 @@ struct PassModel : PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...> {
bool isRequired() const override { return passIsRequiredImpl<PassT>(); }
+ template <typename T>
+ using has_erase_if_t = decltype(std::declval<T &>().eraseIf(
+ std::declval<function_ref<bool(StringRef)>>()));
+
+ template <typename T>
+ std::enable_if_t<is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)> Pred) {
+ Pass.eraseIf(Pred);
+ }
+
+ template <typename T>
+ std::enable_if_t<!is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)>) {}
+
+ void eraseIf(function_ref<bool(StringRef)> Pred) override {
+ eraseIfImpl<PassT>(Pred);
+ }
+
PassT Pass;
};
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 3858be05c61fa9..0cc54b224b2f89 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -425,6 +425,8 @@ class FunctionToLoopPassAdaptor
static bool isRequired() { return true; }
+ void eraseIf(function_ref<bool(StringRef)> Pred) { Pass->eraseIf(Pred); }
+
bool isLoopNestMode() const { return LoopNestMode; }
private:
>From 4e9f2cfaef977a7e605862403415ad5492dc89e5 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 19 Aug 2024 08:20:00 +0800
Subject: [PATCH 2/2] Build codegen pass pipeline, another approach
---
.../llvm/Passes/MachinePassRegistry.def | 5 +
llvm/include/llvm/Passes/PassBuilder.h | 223 ++++++
.../include/llvm/Target/CGPassBuilderOption.h | 3 +
llvm/lib/CodeGen/TargetPassConfig.cpp | 6 +-
llvm/lib/Passes/PassBuilder.cpp | 10 +
llvm/lib/Passes/PassBuilderPipelines.cpp | 692 ++++++++++++++++++
6 files changed, 937 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 8e669ee5791239..e03425382c293e 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -189,6 +189,7 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
#ifndef DUMMY_MACHINE_FUNCTION_PASS
#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
#endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass)
@@ -208,6 +209,8 @@ DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCall
DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass)
DUMMY_MACHINE_FUNCTION_PASS("gc-empty-basic-blocks", GCEmptyBasicBlocksPass)
+DUMMY_MACHINE_FUNCTION_PASS("gc-machine-code-insersion",
+ GCMachineCodeInsertionPass)
DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass)
DUMMY_MACHINE_FUNCTION_PASS("init-undef-pass", InitUndefPass)
DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass)
@@ -260,6 +263,8 @@ DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
DUMMY_MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass)
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
+DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination",
+ UnreachableMachineBlockElimPass)
DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e1d78a8685aed2..c0c77c93364b43 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -15,14 +15,17 @@
#ifndef LLVM_PASSES_PASSBUILDER_H
#define LLVM_PASSES_PASSBUILDER_H
+#include "llvm/ADT/StringSet.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/RegAllocCommon.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Passes/OptimizationLevel.h"
+#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/PGOOptions.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/Transforms/Instrumentation.h"
@@ -35,6 +38,7 @@ class StringRef;
class AAManager;
class TargetMachine;
class ModuleSummaryIndex;
+class MCContext;
template <typename T> class IntrusiveRefCntPtr;
namespace vfs {
class FileSystem;
@@ -307,6 +311,22 @@ class PassBuilder {
/// TargetMachine::registerDefaultAliasAnalyses().
AAManager buildDefaultAAPipeline();
+ /// Build CodeGen pass pipeline.
+ ///
+ /// {{@
+ Expected<ModulePassManager>
+ buildDefaultCodeGenPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error buildDefaultCodeGenPipeline(ModulePassManager &MPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error addRegAllocPass(MachineFunctionPassManager &MFPM, StringRef Filter);
+ template <typename... PassTs> void disablePass() {
+ (DisabledPasses.insert(PassTs::name()), ...);
+ }
+ /// @}}
+
/// Parse a textual pass pipeline description into a \c
/// ModulePassManager.
///
@@ -519,6 +539,106 @@ class PassBuilder {
FullLinkTimeOptimizationLastEPCallbacks.push_back(C);
}
+ void registerGCLoweringEPCallback(
+ const std::function<void(FunctionPassManager &)> C) {
+ GCLoweringEPCallbacks.push_back(C);
+ }
+
+ void registerISelPrepareEPCallback(
+ const std::function<void(ModulePassManager &)> &C) {
+ ISelPrepareEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationEarlyEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ MachineSSAOptimizationEarlyEPCallbacks.push_back(C);
+ }
+
+ void registerILPOptsEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ ILPOptsEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationLastEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ MachineSSAOptimizationLastEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PreRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPostRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PostRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegBankSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PreRegBankSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPreGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PreGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPostGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PostGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerMachineLateOptimizationEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ MachineLateOptimizationEPCallbacks.push_back(C);
+ }
+
+ void registerPreSched2EPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PreSched2EPCallbacks.push_back(C);
+ }
+
+ void registerPostRewriteEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PostRewriteEPCallbacks.push_back(C);
+ }
+
+ void registerPreEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PreEmitEPCallbacks.push_back(C);
+ }
+
+ void registerPostBBSectionsEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ PostBBSectionsEPCallbacks.push_back(C);
+ }
+
+ void registerMIEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ MIEmitEPCallbacks.push_back(C);
+ }
+
+ void setAddInstSelectorCallback(
+ const std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ &C) {
+ AddInstSelectorCallback = C;
+ }
+
/// Register a callback for parsing an AliasAnalysis Name to populate
/// the given AAManager \p AA
void registerParseAACallback(
@@ -639,6 +759,39 @@ class PassBuilder {
OptimizationLevel Level);
void invokePipelineEarlySimplificationEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level);
+ void invokeGCLoweringEPCallbacks(FunctionPassManager &FPM);
+ void invokeISelPrepareEPCallbacks(ModulePassManager &MPM);
+ void
+ invokeMachineSSAOptimizationEarlyEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void
+ invokeMachineSSAOptimizationLastEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePostRegAllocEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePreRegBankSelectEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void
+ invokePreGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void
+ invokePostGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void
+ invokeMachineLateOptimizationEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePostBBSectionsEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+ void invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
if (!Name.consume_front(PassName))
@@ -704,6 +857,19 @@ class PassBuilder {
void addVectorPasses(OptimizationLevel Level, FunctionPassManager &FPM,
bool IsFullLTO);
+ Error addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+ Error addInstructionSelectorPasses(MachineFunctionPassManager &MFPM,
+ const CGPassBuilderOption &Options);
+
+ void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level);
+
+ Error addRegisterAllocatorPasses(MachineFunctionPassManager &MFPM,
+ const CGPassBuilderOption &Options);
+
+ Error setStartStop(ModulePassManager &MPM);
+
static std::optional<std::vector<PipelineElement>>
parsePipelineText(StringRef Text);
@@ -766,6 +932,63 @@ class PassBuilder {
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2>
PipelineEarlySimplificationEPCallbacks;
+ // CodeGen extension point callbacks
+ SmallVector<std::function<void(FunctionPassManager &)>, 2>
+ GCLoweringEPCallbacks;
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ ISelPrepareEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ MachineSSAOptimizationEarlyEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ MachineSSAOptimizationLastEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PreRegAllocEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PostRegAllocEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PreRegBankSelectEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PreGlobalInstructionSelectEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PostGlobalInstructionSelectEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ ILPOptsEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ MachineLateOptimizationEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PreSched2EPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PostRewriteEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PreEmitEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ PostBBSectionsEPCallbacks;
+ SmallVector<
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>, 2>
+ MIEmitEPCallbacks;
+
+ std::function<void(CGPassBuilderOption &)>
+ CodeGenPassPipelineTunningCallback = [](CGPassBuilderOption &) {};
+ std::function<void(MachineFunctionPassManager &, CodeGenOptLevel)>
+ AddInstSelectorCallback;
+ std::function<void(MachineFunctionPassManager &)> AddFastRegAllocCallback;
+ std::function<void(PassBuilder &, MachineFunctionPassManager &)>
+ AddOptimizedRegAllocCallback;
+ StringSet<> DisabledPasses;
+
SmallVector<std::function<void(ModuleAnalysisManager &)>, 2>
ModuleAnalysisRegistrationCallbacks;
SmallVector<std::function<bool(StringRef, ModulePassManager &,
diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h
index 8ab6d63a00056a..51eeaec51b3ff9 100644
--- a/llvm/include/llvm/Target/CGPassBuilderOption.h
+++ b/llvm/include/llvm/Target/CGPassBuilderOption.h
@@ -41,6 +41,9 @@ struct CGPassBuilderOption {
bool PrintLSR = false;
bool DisableMergeICmps = false;
bool DisablePartialLibcallInlining = false;
+ bool DisableReplaceWithVecLib = false;
+ bool DisableLayoutFSProfileLoader = false;
+ bool DisablePrologEpilogInserterPass = false;
bool DisableConstantHoisting = false;
bool DisableSelectOptimize = true;
bool DisableAtExitBasedGlobalDtorLowering = false;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 1b0012b65b80d4..db5e3a41930342 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -184,11 +184,11 @@ static cl::opt<bool> DisableLayoutFSProfileLoader(
"disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
cl::desc("Disable MIRProfileLoader before BlockPlacement"));
// Specify FSProfile file name.
-static cl::opt<std::string>
+cl::opt<std::string>
FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile file name."), cl::Hidden);
// Specify Remapping file for FSProfile.
-static cl::opt<std::string> FSRemappingFile(
+cl::opt<std::string> FSRemappingFile(
"fs-remapping-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
@@ -495,6 +495,8 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
SET_BOOLEAN_OPTION(DisableMergeICmps)
SET_BOOLEAN_OPTION(DisableLSR)
SET_BOOLEAN_OPTION(DisableConstantHoisting)
+ SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+ SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
SET_BOOLEAN_OPTION(DisableCGP)
SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
SET_BOOLEAN_OPTION(DisableSelectOptimize)
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c528863d1870a4..a99ea349250aef 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -432,6 +432,16 @@ PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
std::optional<PGOOptions> PGOOpt,
PassInstrumentationCallbacks *PIC)
: TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
+ AddFastRegAllocCallback = [](MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(PHIEliminationPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+ MFPM.addPass(RegAllocFastPass());
+ };
+ AddOptimizedRegAllocCallback = [](PassBuilder &,
+ MachineFunctionPassManager &) {
+ // TODO: Add related passes here.
+ };
+
if (TM)
TM->registerPassBuilderCallbacks(*this);
if (PIC) {
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 0201e69f3e216a..3d4d80679a1119 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -24,13 +24,40 @@
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
+#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandLargeFpConvert.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/FinalizeISel.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/LocalStackSlotAllocation.h"
+#include "llvm/CodeGen/LowerEmuTLS.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
+#include "llvm/CodeGen/SafeStack.h"
+#include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
+#include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/CodeGen/WasmEHPrepare.h"
+#include "llvm/CodeGen/WinEHPrepare.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only!
#include "llvm/Passes/OptimizationLevel.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/PGOOptions.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Support/WithColor.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
#include "llvm/Transforms/Coroutines/CoroCleanup.h"
@@ -79,11 +106,13 @@
#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
+#include "llvm/Transforms/ObjCARC.h"
#include "llvm/Transforms/Scalar/ADCE.h"
#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
#include "llvm/Transforms/Scalar/BDCE.h"
#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
#include "llvm/Transforms/Scalar/ConstraintElimination.h"
#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
@@ -109,6 +138,7 @@
#include "llvm/Transforms/Scalar/LoopRotation.h"
#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
#include "llvm/Transforms/Scalar/LoopSink.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
@@ -116,14 +146,18 @@
#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
#include "llvm/Transforms/Scalar/NewGVN.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
#include "llvm/Transforms/Scalar/Reassociate.h"
#include "llvm/Transforms/Scalar/SCCP.h"
#include "llvm/Transforms/Scalar/SROA.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
#include "llvm/Transforms/Scalar/SimplifyCFG.h"
#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
#include "llvm/Transforms/Utils/AddDiscriminators.h"
@@ -133,6 +167,8 @@
#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
#include "llvm/Transforms/Utils/Mem2Reg.h"
#include "llvm/Transforms/Utils/MoveAutoInit.h"
#include "llvm/Transforms/Utils/NameAnonGlobals.h"
@@ -311,6 +347,8 @@ namespace llvm {
extern cl::opt<bool> EnableMemProfContextDisambiguation;
extern cl::opt<bool> EnableInferAlignmentPass;
+
+extern cl::opt<std::string> FSRemappingFile;
} // namespace llvm
PipelineTuningOptions::PipelineTuningOptions() {
@@ -392,6 +430,85 @@ void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
for (auto &C : PipelineEarlySimplificationEPCallbacks)
C(MPM, Level);
}
+void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) {
+ for (auto &C : GCLoweringEPCallbacks)
+ C(FPM);
+}
+
+void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : ISelPrepareEPCallbacks)
+ C(MPM);
+}
+void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : MachineSSAOptimizationEarlyEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : MachineSSAOptimizationLastEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level) {
+ for (auto &C : PreRegAllocEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokePreRegBankSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : PreRegBankSelectEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : PreGlobalInstructionSelectEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : PostGlobalInstructionSelectEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level) {
+ for (auto &C : ILPOptsEPCallbacks)
+ C(MFPM, Level);
+}
+void PassBuilder::invokeMachineLateOptimizationEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : MachineLateOptimizationEPCallbacks)
+ C(MFPM, Level);
+}
+
+void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level) {
+ for (auto &C : MIEmitEPCallbacks)
+ C(MFPM, Level);
+}
+
+void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level) {
+ for (auto &C : PreEmitEPCallbacks)
+ C(MFPM, Level);
+}
+
+void PassBuilder::invokePostRegAllocEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : PostRegAllocEPCallbacks)
+ C(MFPM, Level);
+}
+
+void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM,
+ CodeGenOptLevel Level) {
+ for (auto &C : PreSched2EPCallbacks)
+ C(MFPM, Level);
+}
+
+void PassBuilder::invokePostBBSectionsEPCallbacks(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ for (auto &C : PostBBSectionsEPCallbacks)
+ C(MFPM, Level);
+}
// Helper to add AnnotationRemarksPass.
static void addAnnotationRemarksPass(ModulePassManager &MPM) {
@@ -2221,3 +2338,578 @@ AAManager PassBuilder::buildDefaultAAPipeline() {
return AA;
}
+
+// Find the Profile remapping file name. The internal option takes the
+// precedence before getting from TargetMachine.
+static std::string getFSRemappingFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSRemappingFile.empty())
+ return Options.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+}
+
+// Find the FSProfile file name. The internal option takes the precedence
+// before getting from TargetMachine.
+static std::string getFSProfileFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSProfileFile.empty())
+ return Options.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+}
+
+Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) {
+ const MCAsmInfo *MCAI = TM->getMCAsmInfo();
+ if (!MCAI)
+ return make_error<StringError>("No MCAsmInfo!", inconvertibleErrorCode());
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ FPM.addPass(SjLjEHPreparePass(TM));
+ [[fallthrough]];
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ case ExceptionHandling::AIX:
+ case ExceptionHandling::ZOS:
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ FPM.addPass(WinEHPreparePass());
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true));
+ FPM.addPass(WasmEHPreparePass());
+ break;
+ case ExceptionHandling::None:
+ FPM.addPass(LowerInvokePass());
+
+ // The lower invoke pass may create unreachable code. Remove it.
+ FPM.addPass(UnreachableBlockElimPass());
+ break;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addInstructionSelectorPasses(
+ MachineFunctionPassManager &MFPM, const CGPassBuilderOption &Options) {
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ // Core ISel
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM->setO0WantsFastISel(Options.EnableFastISelOption.value_or(true));
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ Options.EnableFastISelOption.value_or(false);
+ if (Options.EnableFastISelOption.value_or(false))
+ Selector = SelectorType::FastISel;
+
+ else if (Options.EnableGlobalISelOption.value_or(false) ||
+ (TM->Options.EnableGlobalISel &&
+ !Options.EnableGlobalISelOption.value_or(false)))
+ Selector = SelectorType::GlobalISel;
+ else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM->setFastISel(true);
+ TM->setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM->setFastISel(false);
+ TM->setGlobalISel(true);
+ }
+
+ // Add instruction selector passes.
+ if (Selector == SelectorType::GlobalISel) {
+ MFPM.addPass(IRTranslatorPass());
+ MFPM.addPass(LegalizerPass());
+
+ // Before running the register bank selector, ask the target if it
+ // wants to run some passes.
+ invokePreRegBankSelectEPCallbacks(MFPM, OptLevel);
+ MFPM.addPass(RegBankSelectPass());
+
+ invokePreGlobalInstructionSelectEPCallbacks(MFPM, OptLevel);
+ MFPM.addPass(InstructionSelectPass());
+ invokePostGlobalInstructionSelectEPCallbacks(MFPM, OptLevel);
+
+ // Pass to reset the MachineFunction if the ISel failed.
+ MFPM.addPass(ResetMachineFunctionPass(
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag,
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable));
+
+ // Provide a fallback path when we do not want to abort on
+ // not-yet-supported input.
+ if (TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM, OptLevel);
+ }
+ } else {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM, OptLevel);
+ }
+ return Error::success();
+}
+
+void PassBuilder::addMachineSSAOptimizationPasses(
+ MachineFunctionPassManager &MFPM, CodeGenOptLevel Level) {
+ // Pre-ra tail duplication.
+ MFPM.addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ MFPM.addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a different pass
+ // which merges spill slots.
+ MFPM.addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ MFPM.addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level parallelism,
+ // like if-conversion. Such passes will typically need dominator trees and
+ // loop info, just like LICM and CSE below.
+ invokeILPOptsEPCallbacks(MFPM, Level);
+
+ MFPM.addPass(EarlyMachineLICMPass());
+ MFPM.addPass(MachineCSEPass());
+
+ MFPM.addPass(MachineSinkingPass());
+
+ MFPM.addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ MFPM.addPass(DeadMachineInstructionElimPass());
+}
+
+Error PassBuilder::setStartStop(ModulePassManager &MPM) {
+ auto StartStopInfoOrErr = TargetPassConfig::getStartStopInfo(*PIC);
+ if (!StartStopInfoOrErr)
+ return StartStopInfoOrErr.takeError();
+ auto &SSI = *StartStopInfoOrErr;
+
+ bool Started = SSI.StartPass.empty(), Stopped = false;
+ // Return true if pass is skipped.
+ auto Filter = [&, StartInstanceNum = 0u,
+ StopInstanceNum = 0u](StringRef Name) mutable {
+ if (!Started) {
+ if (PIC->getPassNameForClassName(Name) == SSI.StartPass)
+ ++StartInstanceNum;
+ if (StartInstanceNum == SSI.StartInstanceNum) {
+ Started = true;
+ return SSI.StartAfter;
+ }
+ return true;
+ }
+
+ if (!Stopped) {
+ if (!SSI.StopPass.empty() &&
+ PIC->getPassNameForClassName(Name) == SSI.StopPass)
+ ++StopInstanceNum;
+ if (StopInstanceNum == SSI.StopInstanceNum) {
+ Stopped = true;
+ return !SSI.StopAfter;
+ }
+ return false;
+ }
+
+ return !Started || Stopped;
+ };
+
+ MPM.eraseIf(Filter);
+ if (!Started) {
+ return make_error<StringError>(
+ "Can't find start pass \"" + SSI.StartPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ if (!Stopped && !SSI.StopPass.empty()) {
+ return make_error<StringError>(
+ "Can't find stop pass \"" + SSI.StopPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addRegisterAllocatorPasses(
+ MachineFunctionPassManager &MFPM, const CGPassBuilderOption &Options) {
+ CodeGenOptLevel Level = TM->getOptLevel();
+ const bool OptimizeRegAlloc =
+ Options.OptimizeRegAlloc.value_or(Level != CodeGenOptLevel::None);
+
+ if (OptimizeRegAlloc) {
+ AddOptimizedRegAllocCallback(*this, MFPM);
+ } else {
+ AddFastRegAllocCallback(MFPM);
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM,
+ StringRef Filter) {
+ return Error::success();
+}
+
+Expected<ModulePassManager> PassBuilder::buildDefaultCodeGenPipeline(
+ raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+ MCContext &Ctx) {
+ if (!TM)
+ return make_error<StringError>("Need a TargetMachine instance!",
+ inconvertibleErrorCode());
+
+ CGPassBuilderOption Options = getCGPassBuilderOption();
+ CodeGenPassPipelineTunningCallback(Options);
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
+ bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
+
+ ModulePassManager MPM;
+ FunctionPassManager FPM;
+ MachineFunctionPassManager MFPM;
+
+ // IR part
+ MPM.addPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
+ if (TM->useEmulatedTLS())
+ MPM.addPass(LowerEmuTLSPass());
+ MPM.addPass(PreISelIntrinsicLoweringPass(TM));
+
+ // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
+ // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
+ if (TM->getTargetTriple().isOSBinFormatMachO() &&
+ !Options.DisableAtExitBasedGlobalDtorLowering)
+ MPM.addPass(LowerGlobalDtorsPass());
+
+ FPM.addPass(ExpandLargeDivRemPass(TM));
+ FPM.addPass(ExpandLargeFpConvertPass(TM));
+
+ // Extension point?
+
+ // Run loop strength reduction before anything else.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!Options.DisableLSR)
+ FPM.addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
+ /*UseMemorySSA=*/true));
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
+ // loads and compares. ExpandMemCmpPass then tries to expand those calls
+ // into optimally-sized loads and compares. The transforms are enabled by a
+ // target lowering hook.
+ if (!Options.DisableMergeICmps)
+ FPM.addPass(MergeICmpsPass());
+ FPM.addPass(ExpandMemCmpPass(TM));
+ }
+
+ // Run GC lowering passes for builtin collectors
+ FPM.addPass(GCLoweringPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(ShadowStackGCLoweringPass());
+ FPM = FunctionPassManager();
+ invokeGCLoweringEPCallbacks(FPM);
+
+ // Make sure that no unreachable blocks are instruction selected.
+ FPM.addPass(UnreachableBlockElimPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!Options.DisableConstantHoisting)
+ FPM.addPass(ConstantHoistingPass());
+ if (!Options.DisableReplaceWithVecLib)
+ FPM.addPass(ReplaceWithVeclib());
+ if (!Options.DisablePartialLibcallInlining)
+ FPM.addPass(PartiallyInlineLibCallsPass());
+ }
+
+ // Instrument function entry after all inlining.
+ FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ FPM.addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants to.
+ // Allow disabling it for testing purposes.
+ if (!Options.DisableExpandReductions)
+ FPM.addPass(ExpandReductionsPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ FPM.addPass(TLSVariableHoistPass());
+
+ // Convert conditional moves to conditional jumps when profitable.
+ if (!Options.DisableSelectOptimize)
+ FPM.addPass(SelectOptimizePass(TM));
+
+ // CodeGen prepare
+ if (!Options.DisableCGP)
+ FPM.addPass(CodeGenPreparePass(TM));
+ }
+
+ // Turn exception handling constructs into something the code generators can
+ // handle.
+ if (auto Err = addExceptionHandlingPasses(FPM))
+ return std::move(Err);
+
+ { // Pre isel extension
+ ModulePassManager ISelPreparePasses;
+ invokeISelPrepareEPCallbacks(ISelPreparePasses);
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(std::move(ISelPreparePasses));
+ }
+
+ if (OptLevel != CodeGenOptLevel::None)
+ FPM.addPass(ObjCARCContractPass());
+ FPM.addPass(CallBrPreparePass());
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ FPM.addPass(SafeStackPass(TM));
+ FPM.addPass(StackProtectorPass(TM));
+
+ if (PrintMIR) {
+ if (Options.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(PrintMIRPreparePass(Out));
+ FPM = FunctionPassManager();
+ }
+
+ if (auto Err = addInstructionSelectorPasses(MFPM, Options))
+ return std::move(Err);
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
+ // FinalizeISel.
+ MFPM.addPass(FinalizeISelPass());
+
+ // Add passes that optimize machine instructions in SSA form.
+ if (OptLevel != CodeGenOptLevel::None) {
+ invokeMachineSSAOptimizationEarlyEPCallbacks(MFPM, OptLevel);
+ addMachineSSAOptimizationPasses(MFPM, OptLevel);
+ invokeMachineSSAOptimizationLastEPCallbacks(MFPM, OptLevel);
+ } else {
+ MFPM.addPass(LocalStackSlotAllocationPass());
+ }
+
+ if (TM->Options.EnableIPRA)
+ MFPM.addPass(RegUsageInfoPropagationPass());
+
+ // Run pre-ra passes.
+ invokePreRegAllocEPCallbacks(MFPM, OptLevel);
+
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile(TM, Options);
+ if (!ProfileFile.empty() && !Options.DisableRAFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, Options),
+ sampleprof::FSDiscriminatorPass::Pass1, nullptr));
+ }
+
+ if (auto Err = addRegisterAllocatorPasses(MFPM, Options))
+ return std::move(Err);
+
+ invokePostRegAllocEPCallbacks(MFPM, OptLevel);
+
+ MFPM.addPass(RemoveRedundantDebugValuesPass());
+ MFPM.addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (OptLevel != CodeGenOptLevel::None) {
+ MFPM.addPass(PostRAMachineSinkingPass());
+ MFPM.addPass(ShrinkWrapPass());
+ }
+
+ if (!Options.DisablePrologEpilogInserterPass)
+ MFPM.addPass(PrologEpilogInserterPass());
+ /// Add passes that optimize machine instructions after register allocation.
+ if (OptLevel != CodeGenOptLevel::None)
+ invokeMachineLateOptimizationEPCallbacks(MFPM, OptLevel);
+
+ // Expand pseudo instructions before second scheduling pass.
+ MFPM.addPass(ExpandPostRAPseudosPass());
+
+ // Run pre-sched2 passes.
+ invokePreSched2EPCallbacks(MFPM, OptLevel);
+
+ if (Options.EnableImplicitNullChecks)
+ MFPM.addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other
+ // point.
+ if (OptLevel != CodeGenOptLevel::None &&
+ !TM->targetSchedulesPostRAScheduling()) {
+ if (Options.MISchedPostRA)
+ MFPM.addPass(PostMachineSchedulerPass());
+ else
+ MFPM.addPass(PostRASchedulerPass());
+ }
+
+ // GC, replacement for GCMachineCodeAnalysis
+ MFPM.addPass(GCMachineCodeInsertionPass());
+
+ // Basic block placement.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2));
+ const std::string ProfileFile = getFSProfileFile(TM, Options);
+ if (!ProfileFile.empty() && !Options.DisableLayoutFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, Options),
+ sampleprof::FSDiscriminatorPass::Pass2, nullptr));
+ }
+ MFPM.addPass(MachineBlockPlacementPass());
+ }
+
+ // Insert before XRay Instrumentation.
+ MFPM.addPass(FEntryInserterPass());
+ MFPM.addPass(XRayInstrumentationPass());
+ MFPM.addPass(PatchableFunctionPass());
+
+ invokePreEmitEPCallbacks(MFPM, OptLevel);
+
+ if (TM->Options.EnableIPRA)
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ MFPM.addPass(RegUsageInfoCollectorPass());
+
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
+ MFPM.addPass(FuncletLayoutPass());
+
+ MFPM.addPass(StackMapLivenessPass());
+ MFPM.addPass(LiveDebugValuesPass());
+ MFPM.addPass(MachineSanitizerBinaryMetadata());
+
+ if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None &&
+ Options.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (Options.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
+ if (AddOutliner) {
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ if (Options.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(MachineOutlinerPass(RunOnAllFunctions));
+ FPM = FunctionPassManager();
+ MFPM = MachineFunctionPassManager();
+ }
+ }
+
+ if (Options.GCEmptyBlocks)
+ MFPM.addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ bool NeedsBBSections =
+ TM->getBBSectionsType() != llvm::BasicBlockSection::None;
+ // Machine function splitter uses the basic block sections feature. Both
+ // cannot be enabled at the same time. We do not apply machine function
+ // splitter if -basic-block-sections is requested.
+ if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter ||
+ Options.EnableMachineFunctionSplitter)) {
+ const std::string ProfileFile = getFSProfileFile(TM, Options);
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, Options),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.\n";
+ }
+ }
+ MFPM.addPass(MachineFunctionSplitterPass());
+ }
+ // We run the BasicBlockSections pass if either we need BB sections or BB
+ // address map (or both).
+ if (NeedsBBSections || TM->Options.BBAddrMap) {
+ if (TM->getBBSectionsType() == llvm::BasicBlockSection::List)
+ MFPM.addPass(BasicBlockPathCloningPass());
+ MFPM.addPass(BasicBlockSectionsPass());
+ }
+
+ invokePostBBSectionsEPCallbacks(MFPM, OptLevel);
+
+ if (!Options.DisableCFIFixup && TM->Options.EnableCFIFixup)
+ MFPM.addPass(CFIFixupPass());
+
+ MFPM.addPass(StackFrameLayoutAnalysisPass());
+
+ // Add passes that directly emit MI after all other MI passes.
+ invokeMIEmitEPCallbacks(MFPM, OptLevel);
+
+ if (PrintMIR)
+ MFPM.addPass(PrintMIRPass(Out));
+
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+ if (Options.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+ MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); });
+ if (auto Err = setStartStop(MPM))
+ return std::move(Err);
+ return std::move(MPM);
+}
+
+Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &MPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx) {
+ Expected<ModulePassManager> MPMOrErr =
+ buildDefaultCodeGenPipeline(Out, DwoOut, FileType, Ctx);
+ if (!MPMOrErr)
+ return MPMOrErr.takeError();
+ MPM.addPass(std::move(*MPMOrErr));
+ return Error::success();
+}
\ No newline at end of file
More information about the llvm-commits
mailing list