[llvm] [NewPM][CodeGen] Add callback style codegen pass builder (PR #108690)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 14 01:52:16 PDT 2024
https://github.com/paperchalice created https://github.com/llvm/llvm-project/pull/108690
In contrast to #89708, this pull request demonstrates a callback style codegen pipeline builder. Suggestions are welcome!
It supports `-start/stop-after/before` by adding `eraseIf` in pass manager and pass adaptor. To extend pass pipeline, targets can use following extension points:
- CodeGenIREarlyEPCallbacks
- CodeGenIRLateEPCallbacks
- CodeGenIRFinalEPCallbacks
- GCLoweringEPCallbacks
- ISelPrepareEPCallbacks
- MachineSSAOptimizationEarlyEPCallbacks
- MachineSSAOptimizationLastEPCallbacks
- PreRegAllocEPCallbacks
- PreRegBankSelectEPCallbacks
- PreGlobalInstructionSelectEPCallbacks
- PostGlobalInstructionSelectEPCallbacks
- ILPOptsEPCallbacks
- MachineLateOptimizationEPCallbacks
- MIEmitEPCallbacks
- PreEmitEPCallbacks
- PostRegAllocEPCallbacks
- PreSched2EPCallbacks
- PostBBSectionsEPCallbacks
User can control register allocator by `--regalloc-npm=regalloc-name<filter=filter1>,regalloc-name<filter=filter2>...`
Most of them are from `TargetPassConfig`, and this should be sufficient for most targets, except AArch64 and DirectX, because AArch64 tries to insert module pass in machine function pass pipeline, DirectX builds its own codegen pass pipeline.
Register allocator part should be highly customizable, there are some ad-hoc examples in AMDGPU and NVPTX, but I couldn't find a good way to handle them...
>From 3eefffe503e6d4864e1541783aca566a0f796e96 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 19 Aug 2024 08:19:28 +0800
Subject: [PATCH 1/2] Support eraseIf in pass manager
---
llvm/include/llvm/Analysis/CGSCCPassManager.h | 4 ++
.../include/llvm/CodeGen/MachinePassManager.h | 4 ++
llvm/include/llvm/IR/PassManager.h | 19 ++++++++
llvm/include/llvm/IR/PassManagerInternal.h | 35 +++++++++++++++
.../llvm/Transforms/Scalar/LoopPassManager.h | 8 ++++
llvm/lib/Analysis/CGSCCPassManager.cpp | 12 +++++
llvm/lib/IR/PassManager.cpp | 11 +++++
.../lib/Transforms/Scalar/LoopPassManager.cpp | 44 +++++++++++++++++++
8 files changed, 137 insertions(+)
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 15b7f226fd8283..1adc29c57ef2c9 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -346,6 +346,10 @@ class ModuleToPostOrderCGSCCPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
};
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 253fabdac0019d..13a825d32fa310 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -207,6 +207,10 @@ class FunctionToMachineFunctionPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
};
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index d269221fac0701..4f28b4aceb3e66 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -218,6 +218,21 @@ class PassManager : public PassInfoMixin<
static bool isRequired() { return true; }
+ /// Erase all passes that satisfy the predicate \p Pred.
+ /// For internal use only!
+ void eraseIf(function_ref<bool(StringRef)> Pred) {
+ for (auto I = Passes.begin(); I != Passes.end();) {
+ (*I)->eraseIf(Pred);
+ bool IsSpecial = (*I)->name().ends_with("PassAdaptor") ||
+ (*I)->name().contains("PassManager");
+ bool PredResult = Pred((*I)->name());
+ if ((!IsSpecial && PredResult) || (IsSpecial && (*I)->isEmpty()))
+ I = Passes.erase(I);
+ else
+ ++I;
+ }
+ }
+
protected:
using PassConceptT =
detail::PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...>;
@@ -836,6 +851,10 @@ class ModuleToFunctionPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
bool EagerlyInvalidate;
diff --git a/llvm/include/llvm/IR/PassManagerInternal.h b/llvm/include/llvm/IR/PassManagerInternal.h
index 4ada6ee5dd6831..3a75c9c4e97faf 100644
--- a/llvm/include/llvm/IR/PassManagerInternal.h
+++ b/llvm/include/llvm/IR/PassManagerInternal.h
@@ -59,6 +59,13 @@ struct PassConcept {
/// To opt-in, pass should implement `static bool isRequired()`. It's no-op
/// to have `isRequired` always return false since that is the default.
virtual bool isRequired() const = 0;
+
+ /// Polymorphic method to refurbish pass pipeline.
+ virtual void eraseIf(function_ref<bool(StringRef)> Pred) = 0;
+
+ /// There may be some empty PassManager after erasing,
+ /// use it to remove them.
+ virtual bool isEmpty() const = 0;
};
/// A template wrapper used to implement the polymorphic API.
@@ -114,6 +121,34 @@ struct PassModel : PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...> {
bool isRequired() const override { return passIsRequiredImpl<PassT>(); }
+ template <typename T>
+ using has_erase_if_t = decltype(std::declval<T &>().eraseIf(
+ std::declval<function_ref<bool(StringRef)>>()));
+
+ template <typename T>
+ std::enable_if_t<is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)> Pred) {
+ Pass.eraseIf(Pred);
+ }
+
+ template <typename T>
+ std::enable_if_t<!is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)>) {}
+
+ void eraseIf(function_ref<bool(StringRef)> Pred) override {
+ eraseIfImpl<PassT>(Pred);
+ }
+
+ template <typename T>
+ using has_is_empty_t = decltype(std::declval<T &>().isEmpty());
+
+ bool isEmpty() const override {
+ if constexpr (is_detected<has_is_empty_t, PassT>::value)
+ return Pass.isEmpty();
+ else
+ return false;
+ }
+
PassT Pass;
};
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 3858be05c61fa9..12a8579e9f8673 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -134,6 +134,10 @@ class PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
static bool isRequired() { return true; }
+ /// Erase all passes that satisfy the predicate \p Pred.
+ /// For internal use only!
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
size_t getNumLoopPasses() const { return LoopPasses.size(); }
size_t getNumLoopNestPasses() const { return LoopNestPasses.size(); }
@@ -425,6 +429,10 @@ class FunctionToLoopPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
bool isLoopNestMode() const { return LoopNestMode; }
private:
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index c32739a5655419..dabb84d5768382 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -753,6 +753,18 @@ bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
return false;
}
+void ModuleToPostOrderCGSCCPassAdaptor::eraseIf(
+ function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
} // end namespace llvm
/// When a new SCC is created for the graph we first update the
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index bf8f7906d3368d..0c9ee99f24dd19 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -145,6 +145,17 @@ PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M,
return PA;
}
+void ModuleToFunctionPassAdaptor::eraseIf(function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
template <>
void llvm::printIRUnitNameForStackTrace<Module>(raw_ostream &OS,
const Module &IR) {
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index a4f2dbf9a58289..db8f31f12ab726 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -63,6 +63,39 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
}
}
+void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
+ LPMUpdater &>::eraseIf(function_ref<bool(StringRef)> Pred) {
+ std::vector<char> IsLoopNestPassVec(
+ static_cast<size_t>(IsLoopNestPass.size()));
+ for (unsigned Idx = 0, Sz = IsLoopNestPass.size(); Idx != Sz; ++Idx)
+ IsLoopNestPassVec[Idx] = IsLoopNestPass[Idx];
+
+ auto ILP = LoopPasses.begin();
+ auto ILNP = LoopNestPasses.begin();
+ for (auto I = IsLoopNestPassVec.begin(); I != IsLoopNestPassVec.end();) {
+ if (*I) {
+ if (Pred((*ILNP)->name())) {
+ I = IsLoopNestPassVec.erase(I);
+ ILNP = LoopNestPasses.erase(ILNP);
+ continue;
+ }
+ ++ILNP;
+ } else {
+ if (Pred((*ILP)->name())) {
+ I = IsLoopNestPassVec.erase(I);
+ ILP = LoopPasses.erase(ILP);
+ continue;
+ }
+ ++ILP;
+ }
+ ++I;
+ }
+
+ IsLoopNestPass.clear();
+ for (const auto I : IsLoopNestPassVec)
+ IsLoopNestPass.push_back(I);
+}
+
// Run both loop passes and loop-nest passes on top-level loop \p L.
PreservedAnalyses
LoopPassManager::runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
@@ -363,6 +396,17 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
return PA;
}
+void FunctionToLoopPassAdaptor::eraseIf(function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
PrintLoopPass::PrintLoopPass() : OS(dbgs()) {}
PrintLoopPass::PrintLoopPass(raw_ostream &OS, const std::string &Banner)
: OS(OS), Banner(Banner) {}
>From c15dcb9631b9363d4ee26649696e5da9864b2123 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Sat, 14 Sep 2024 16:38:47 +0800
Subject: [PATCH 2/2] [NewPM] Add callback style codegen pass builder
---
llvm/include/llvm/CodeGen/TargetPassConfig.h | 3 +-
llvm/include/llvm/Passes/CodeGenPassBuilder.h | 2 +-
.../llvm/Passes/MachinePassRegistry.def | 5 +
llvm/include/llvm/Passes/PassBuilder.h | 242 +++++
.../include/llvm/Target/CGPassBuilderOption.h | 3 +
llvm/lib/CodeGen/MachinePassManager.cpp | 12 +
llvm/lib/CodeGen/TargetPassConfig.cpp | 9 +-
llvm/lib/Passes/CMakeLists.txt | 1 +
llvm/lib/Passes/PassBuilder.cpp | 3 +-
llvm/lib/Passes/PassBuilderCodeGen.cpp | 942 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 24 +
llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp | 4 +
.../lib/Transforms/Scalar/LoopPassManager.cpp | 9 +-
llvm/test/CodeGen/AMDGPU/regalloc-select.ll | 9 +
llvm/test/tools/llc/new-pm/start-stop.ll | 4 +-
llvm/tools/llc/NewPMDriver.cpp | 21 +-
16 files changed, 1269 insertions(+), 24 deletions(-)
create mode 100644 llvm/lib/Passes/PassBuilderCodeGen.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/regalloc-select.ll
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index 2f5951e3ec3bce..ad27a5471d1a7a 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -190,8 +190,7 @@ class TargetPassConfig : public ImmutablePass {
/// Returns pass name in `-stop-before` or `-stop-after`
/// NOTE: New pass manager migration only
- static Expected<StartStopInfo>
- getStartStopInfo(PassInstrumentationCallbacks &PIC);
+ static Expected<StartStopInfo> getStartStopInfo();
void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index a99fed86d168d1..fc1a15b320b5c4 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -523,7 +523,7 @@ template <typename Derived, typename TargetMachineT>
Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
CodeGenFileType FileType) const {
- auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
+ auto StartStopInfo = TargetPassConfig::getStartStopInfo();
if (!StartStopInfo)
return StartStopInfo.takeError();
setStartStopPasses(*StartStopInfo);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 4047fd0478579f..fd6aa34e498516 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -191,6 +191,7 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
#ifndef DUMMY_MACHINE_FUNCTION_PASS
#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
#endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass)
@@ -210,6 +211,8 @@ DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCall
DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass)
DUMMY_MACHINE_FUNCTION_PASS("gc-empty-basic-blocks", GCEmptyBasicBlocksPass)
+DUMMY_MACHINE_FUNCTION_PASS("gc-machine-code-insersion",
+ GCMachineCodeInsertionPass)
DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass)
DUMMY_MACHINE_FUNCTION_PASS("init-undef-pass", InitUndefPass)
DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass)
@@ -261,6 +264,8 @@ DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
DUMMY_MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass)
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
+DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination",
+ UnreachableMachineBlockElimPass)
DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e1d78a8685aed2..aade8ce85c4abc 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -15,14 +15,17 @@
#ifndef LLVM_PASSES_PASSBUILDER_H
#define LLVM_PASSES_PASSBUILDER_H
+#include "llvm/ADT/StringSet.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/RegAllocCommon.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Passes/OptimizationLevel.h"
+#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/PGOOptions.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/Transforms/Instrumentation.h"
@@ -35,6 +38,7 @@ class StringRef;
class AAManager;
class TargetMachine;
class ModuleSummaryIndex;
+class MCContext;
template <typename T> class IntrusiveRefCntPtr;
namespace vfs {
class FileSystem;
@@ -107,6 +111,7 @@ class PassBuilder {
TargetMachine *TM;
PipelineTuningOptions PTO;
std::optional<PGOOptions> PGOOpt;
+ CGPassBuilderOption CGPBO;
PassInstrumentationCallbacks *PIC;
public:
@@ -307,6 +312,24 @@ class PassBuilder {
/// TargetMachine::registerDefaultAliasAnalyses().
AAManager buildDefaultAAPipeline();
+ /// Build CodeGen pass pipeline.
+ ///
+ /// {{@
+ Expected<ModulePassManager>
+ buildDefaultCodeGenPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error buildDefaultCodeGenPipeline(ModulePassManager &MPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error addRegAllocPass(MachineFunctionPassManager &MFPM,
+ StringRef Filter = "all");
+ // TODO: Add method to build MC emission pipeline.
+ template <typename... PassTs> void disablePass() {
+ (DisabledPasses.insert(PassTs::name()), ...);
+ }
+ /// @}}
+
/// Parse a textual pass pipeline description into a \c
/// ModulePassManager.
///
@@ -519,6 +542,133 @@ class PassBuilder {
FullLinkTimeOptimizationLastEPCallbacks.push_back(C);
}
+ /// Register target specific callbacks to extend codegen pipeline.
+ /// {{@
+
+ /// If target want its own pipeline, use this callback.
+ void setCustomCodeGenPipelineBuilderCallback(
+ const std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+ raw_pwrite_stream *, CodeGenFileType,
+ MCContext &)>
+ C) {
+ CustomCodeGenPipelineBuilderCallback = C;
+ }
+
+ void registerCodeGenIREarlyEPCallback(
+ const std::function<void(ModulePassManager &)> C) {
+ CodeGenIREarlyEPCallbacks.push_back(C);
+ }
+
+ void registerCodeGenIRLateEPCallback(
+ const std::function<void(ModulePassManager &)> C) {
+ CodeGenIRLateEPCallbacks.push_back(C);
+ }
+
+ void registerCodeGenIRFinalEPCallback(
+ const std::function<void(FunctionPassManager &)> C) {
+ CodeGenIRFinalEPCallbacks.push_back(C);
+ }
+
+ void registerGCLoweringEPCallback(
+ const std::function<void(FunctionPassManager &)> C) {
+ GCLoweringEPCallbacks.push_back(C);
+ }
+
+ void registerISelPrepareEPCallback(
+ const std::function<void(ModulePassManager &)> &C) {
+ ISelPrepareEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationEarlyEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineSSAOptimizationEarlyEPCallbacks.push_back(C);
+ }
+
+ void registerILPOptsEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ ILPOptsEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationLastEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineSSAOptimizationLastEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPostRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegBankSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreRegBankSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPreGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPostGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerMachineLateOptimizationEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineLateOptimizationEPCallbacks.push_back(C);
+ }
+
+ void registerPreSched2EPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreSched2EPCallbacks.push_back(C);
+ }
+
+ void registerPostRewriteEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostRewriteEPCallbacks.push_back(C);
+ }
+
+ void registerPreEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreEmitEPCallbacks.push_back(C);
+ }
+
+ void registerPostBBSectionsEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostBBSectionsEPCallbacks.push_back(C);
+ }
+
+ void registerMIEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MIEmitEPCallbacks.push_back(C);
+ }
+
+ void setAddInstSelectorCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ AddInstSelectorCallback = C;
+ }
+
+ void setCustomFastRegAllocPipelineBuilder(
+ const std::function<Error(MachineFunctionPassManager &)> &C) {
+ AddFastRegAllocCallback = C;
+ }
+
+ void setCustomOptimizedRegAllocPipelineBuilder(
+ const std::function<Error(MachineFunctionPassManager &)> &C) {
+ AddOptimizedRegAllocCallback = C;
+ }
+ ///@}}
+
+ // New pass manager migration methods, don't use them
+ // outside llvm!
+ CGPassBuilderOption &getCGPBO() { return CGPBO; }
+
/// Register a callback for parsing an AliasAnalysis Name to populate
/// the given AAManager \p AA
void registerParseAACallback(
@@ -640,6 +790,30 @@ class PassBuilder {
void invokePipelineEarlySimplificationEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level);
+ void invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM);
+ void invokeCodeGenIRLateEPCallbacks(ModulePassManager &MPM);
+ void invokeCodeGenIRFinalEPCallbacks(FunctionPassManager &FPM);
+ void invokeGCLoweringEPCallbacks(FunctionPassManager &FPM);
+ void invokeISelPrepareEPCallbacks(ModulePassManager &MPM);
+ void invokeMachineSSAOptimizationEarlyEPCallbacks(
+ MachineFunctionPassManager &MFPM);
+ void
+ invokeMachineSSAOptimizationLastEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreRegBankSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+ void
+ invokePreGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM);
+ void invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM);
+ void
+ invokeMachineLateOptimizationEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostBBSectionsEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM);
+
static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
if (!Name.consume_front(PassName))
return false;
@@ -704,6 +878,22 @@ class PassBuilder {
void addVectorPasses(OptimizationLevel Level, FunctionPassManager &FPM,
bool IsFullLTO);
+ Error addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+ Error addInstructionSelectorPasses(MachineFunctionPassManager &MFPM);
+
+ void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM);
+
+ Error addMachinePasses(ModulePassManager &MPM, FunctionPassManager &FPM,
+ MachineFunctionPassManager &MFPM,
+ raw_pwrite_stream *Out);
+
+ Error addRegisterAllocatorPasses(MachineFunctionPassManager &MFPM);
+
+ Error parseRegAllocOption(StringRef Text);
+
+ bool isOptimizedRegAlloc() const;
+
static std::optional<std::vector<PipelineElement>>
parsePipelineText(StringRef Text);
@@ -766,6 +956,58 @@ class PassBuilder {
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2>
PipelineEarlySimplificationEPCallbacks;
+ // CodeGen extension point callbacks
+ std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+ raw_pwrite_stream *, CodeGenFileType, MCContext &)>
+ CustomCodeGenPipelineBuilderCallback;
+
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ CodeGenIREarlyEPCallbacks;
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ CodeGenIRLateEPCallbacks;
+ SmallVector<std::function<void(FunctionPassManager &)>, 2>
+ CodeGenIRFinalEPCallbacks;
+ SmallVector<std::function<void(FunctionPassManager &)>, 2>
+ GCLoweringEPCallbacks;
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ ISelPrepareEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineSSAOptimizationEarlyEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineSSAOptimizationLastEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreRegAllocEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostRegAllocEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreRegBankSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreGlobalInstructionSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostGlobalInstructionSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ ILPOptsEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineLateOptimizationEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreSched2EPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostRewriteEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreEmitEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostBBSectionsEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MIEmitEPCallbacks;
+
+ std::function<void(MachineFunctionPassManager &)> AddInstSelectorCallback;
+ std::function<Error(MachineFunctionPassManager &)> AddFastRegAllocCallback;
+ std::function<Error(MachineFunctionPassManager &)>
+ AddOptimizedRegAllocCallback;
+ StringSet<> DisabledPasses;
+ StringMap<MachineFunctionPassManager> RegAllocPasses;
+ // TODO: Add methods in LLVMTargetMachine so we can get rid of it.
+
SmallVector<std::function<void(ModuleAnalysisManager &)>, 2>
ModuleAnalysisRegistrationCallbacks;
SmallVector<std::function<bool(StringRef, ModulePassManager &,
diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h
index 8ab6d63a00056a..51eeaec51b3ff9 100644
--- a/llvm/include/llvm/Target/CGPassBuilderOption.h
+++ b/llvm/include/llvm/Target/CGPassBuilderOption.h
@@ -41,6 +41,9 @@ struct CGPassBuilderOption {
bool PrintLSR = false;
bool DisableMergeICmps = false;
bool DisablePartialLibcallInlining = false;
+ bool DisableReplaceWithVecLib = false;
+ bool DisableLayoutFSProfileLoader = false;
+ bool DisablePrologEpilogInserterPass = false;
bool DisableConstantHoisting = false;
bool DisableSelectOptimize = true;
bool DisableAtExitBasedGlobalDtorLowering = false;
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 6d540808d4cc98..83fe22b00ada8a 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -105,6 +105,18 @@ bool MachineFunctionAnalysisManagerFunctionProxy::Result::invalidate(
return false;
}
+void FunctionToMachineFunctionPassAdaptor::eraseIf(
+ function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
PreservedAnalyses
FunctionToMachineFunctionPassAdaptor::run(Function &F,
FunctionAnalysisManager &FAM) {
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 11a7752ef7a381..2a2eb670ff6392 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -184,11 +184,11 @@ static cl::opt<bool> DisableLayoutFSProfileLoader(
"disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
cl::desc("Disable MIRProfileLoader before BlockPlacement"));
// Specify FSProfile file name.
-static cl::opt<std::string>
+cl::opt<std::string>
FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile file name."), cl::Hidden);
// Specify Remapping file for FSProfile.
-static cl::opt<std::string> FSRemappingFile(
+cl::opt<std::string> FSRemappingFile(
"fs-remapping-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
@@ -495,6 +495,8 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
SET_BOOLEAN_OPTION(DisableMergeICmps)
SET_BOOLEAN_OPTION(DisableLSR)
SET_BOOLEAN_OPTION(DisableConstantHoisting)
+ SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+ SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
SET_BOOLEAN_OPTION(DisableCGP)
SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
SET_BOOLEAN_OPTION(DisableSelectOptimize)
@@ -537,8 +539,7 @@ void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
});
}
-Expected<TargetPassConfig::StartStopInfo>
-TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
+Expected<TargetPassConfig::StartStopInfo> TargetPassConfig::getStartStopInfo() {
auto [StartBefore, StartBeforeInstanceNum] =
getPassNameAndInstanceNum(StartBeforeOpt);
auto [StartAfter, StartAfterInstanceNum] =
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 6425f4934b2103..f9011cc8235084 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMPasses
OptimizationLevel.cpp
PassBuilder.cpp
PassBuilderBindings.cpp
+ PassBuilderCodeGen.cpp
PassBuilderPipelines.cpp
PassPlugin.cpp
StandardInstrumentations.cpp
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index a22abed8051a11..21b25f46ad4d74 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -431,7 +431,8 @@ class RequireAllMachineFunctionPropertiesPass
PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
std::optional<PGOOptions> PGOOpt,
PassInstrumentationCallbacks *PIC)
- : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
+ : TM(TM), PTO(PTO), PGOOpt(PGOOpt), CGPBO(getCGPassBuilderOption()),
+ PIC(PIC) {
if (TM)
TM->registerPassBuilderCallbacks(*this);
if (PIC) {
diff --git a/llvm/lib/Passes/PassBuilderCodeGen.cpp b/llvm/lib/Passes/PassBuilderCodeGen.cpp
new file mode 100644
index 00000000000000..70c0cd3f2e9cfb
--- /dev/null
+++ b/llvm/lib/Passes/PassBuilderCodeGen.cpp
@@ -0,0 +1,942 @@
+//===- Construction of code generation pass pipelines ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides the implementation of the PassBuilder based on our
+/// static pass registry as well as related functionality.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
+#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandLargeFpConvert.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/FinalizeISel.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LocalStackSlotAllocation.h"
+#include "llvm/CodeGen/LowerEmuTLS.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
+#include "llvm/CodeGen/SafeStack.h"
+#include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
+#include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/CodeGen/WasmEHPrepare.h"
+#include "llvm/CodeGen/WinEHPrepare.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only!
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+
+namespace llvm {
+extern cl::opt<std::string> FSRemappingFile;
+}
+
+using namespace llvm;
+
+void PassBuilder::invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : CodeGenIREarlyEPCallbacks)
+ C(MPM);
+}
+
+void PassBuilder::invokeCodeGenIRLateEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : CodeGenIRLateEPCallbacks)
+ C(MPM);
+}
+
+void PassBuilder::invokeCodeGenIRFinalEPCallbacks(FunctionPassManager &FPM) {
+ for (auto &C : CodeGenIRFinalEPCallbacks)
+ C(FPM);
+}
+
+void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) {
+ for (auto &C : GCLoweringEPCallbacks)
+ C(FPM);
+}
+
+void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : ISelPrepareEPCallbacks)
+ C(MPM);
+}
+
+void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineSSAOptimizationEarlyEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineSSAOptimizationLastEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreRegAllocEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreRegAllocEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreRegBankSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreRegBankSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreGlobalInstructionSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostGlobalInstructionSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : ILPOptsEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMachineLateOptimizationEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineLateOptimizationEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : MIEmitEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreEmitEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostRegAllocEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostRegAllocEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreSched2EPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostBBSectionsEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostBBSectionsEPCallbacks)
+ C(MFPM);
+}
+
+bool PassBuilder::isOptimizedRegAlloc() const {
+ return CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() !=
+ CodeGenOptLevel::None);
+}
+
+// Find the Profile remapping file name. The internal option takes the
+// precedence before getting from TargetMachine.
+static std::string getFSRemappingFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSRemappingFile.empty())
+ return Options.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+}
+
+// Find the FSProfile file name. The internal option takes the precedence
+// before getting from TargetMachine.
+static std::string getFSProfileFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSProfileFile.empty())
+ return Options.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+}
+
+Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) {
+ const MCAsmInfo *MCAI = TM->getMCAsmInfo();
+ if (!MCAI)
+ return make_error<StringError>("No MCAsmInfo!", inconvertibleErrorCode());
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ FPM.addPass(SjLjEHPreparePass(TM));
+ [[fallthrough]];
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ case ExceptionHandling::AIX:
+ case ExceptionHandling::ZOS:
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ FPM.addPass(WinEHPreparePass());
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true));
+ FPM.addPass(WasmEHPreparePass());
+ break;
+ case ExceptionHandling::None:
+ FPM.addPass(LowerInvokePass());
+
+ // The lower invoke pass may create unreachable code. Remove it.
+ FPM.addPass(UnreachableBlockElimPass());
+ break;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addInstructionSelectorPasses(
+ MachineFunctionPassManager &MFPM) {
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ // Core ISel
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true));
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ CGPBO.EnableFastISelOption.value_or(false);
+ if (CGPBO.EnableFastISelOption.value_or(false))
+ Selector = SelectorType::FastISel;
+
+ else if (CGPBO.EnableGlobalISelOption.value_or(false) ||
+ (TM->Options.EnableGlobalISel &&
+ !CGPBO.EnableGlobalISelOption.value_or(false)))
+ Selector = SelectorType::GlobalISel;
+ else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM->setFastISel(true);
+ TM->setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM->setFastISel(false);
+ TM->setGlobalISel(true);
+ }
+
+ // Add instruction selector passes.
+ if (Selector == SelectorType::GlobalISel) {
+ MFPM.addPass(IRTranslatorPass());
+ MFPM.addPass(LegalizerPass());
+
+ // Before running the register bank selector, ask the target if it
+ // wants to run some passes.
+ invokePreRegBankSelectEPCallbacks(MFPM);
+ MFPM.addPass(RegBankSelectPass());
+
+ invokePreGlobalInstructionSelectEPCallbacks(MFPM);
+ MFPM.addPass(InstructionSelectPass());
+ invokePostGlobalInstructionSelectEPCallbacks(MFPM);
+
+ // Pass to reset the MachineFunction if the ISel failed.
+ MFPM.addPass(ResetMachineFunctionPass(
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag,
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable));
+
+ // Provide a fallback path when we do not want to abort on
+ // not-yet-supported input.
+ if (TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM);
+ }
+ } else {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM);
+ }
+ return Error::success();
+}
+
+void PassBuilder::addMachineSSAOptimizationPasses(
+ MachineFunctionPassManager &MFPM) {
+ // Pre-ra tail duplication.
+ MFPM.addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ MFPM.addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a different pass
+ // which merges spill slots.
+ MFPM.addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ MFPM.addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level parallelism,
+ // like if-conversion. Such passes will typically need dominator trees and
+ // loop info, just like LICM and CSE below.
+ invokeILPOptsEPCallbacks(MFPM);
+
+ MFPM.addPass(EarlyMachineLICMPass());
+ MFPM.addPass(MachineCSEPass());
+ MFPM.addPass(MachineSinkingPass());
+ MFPM.addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ MFPM.addPass(DeadMachineInstructionElimPass());
+}
+
+static Error setStartStop(ModulePassManager &MPM,
+ PassInstrumentationCallbacks *PIC) {
+ auto StartStopInfoOrErr = TargetPassConfig::getStartStopInfo();
+ if (!StartStopInfoOrErr)
+ return StartStopInfoOrErr.takeError();
+ auto &SSI = *StartStopInfoOrErr;
+
+ if (SSI.StartPass.empty() && SSI.StopPass.empty())
+ return Error::success();
+ if (!PIC) {
+ return make_error<StringError>("Need PassInstrumentationCallbacks!",
+ inconvertibleErrorCode());
+ }
+
+ static const std::vector<StringRef> SpecialPasses = {
+ "InvalidateAnalysisPass", "MachineVerifierPass", "PrintMIRPass",
+ "PrintMIRPreparePass", "VerifierPass"};
+
+ bool Started = SSI.StartPass.empty(), Stopped = false;
+ // Return true if pass is skipped.
+ auto Filter = [&, StartInstanceNum = 0u,
+ StopInstanceNum = 0u](StringRef Name) mutable {
+ if (isSpecialPass(Name, SpecialPasses) ||
+ Name.contains("MachineModuleAnalysis"))
+ return false;
+ if (!Started) {
+ if (PIC->getPassNameForClassName(Name) == SSI.StartPass)
+ ++StartInstanceNum;
+ if (StartInstanceNum == SSI.StartInstanceNum) {
+ Started = true;
+ return SSI.StartAfter;
+ }
+ return true;
+ }
+
+ if (!Stopped) {
+ if (!SSI.StopPass.empty() &&
+ PIC->getPassNameForClassName(Name) == SSI.StopPass)
+ ++StopInstanceNum;
+ if (StopInstanceNum == SSI.StopInstanceNum) {
+ Stopped = true;
+ return !SSI.StopAfter;
+ }
+ return false;
+ }
+
+ return !Started || Stopped;
+ };
+
+ MPM.eraseIf(Filter);
+ if (!Started) {
+ return make_error<StringError>(
+ "Can't find start pass \"" + SSI.StartPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ if (!Stopped && !SSI.StopPass.empty()) {
+ return make_error<StringError>(
+ "Can't find stop pass \"" + SSI.StopPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addRegisterAllocatorPasses(
+ MachineFunctionPassManager &MFPM) {
+ if (isOptimizedRegAlloc()) {
+ if (AddOptimizedRegAllocCallback)
+ return AddOptimizedRegAllocCallback(MFPM);
+
+ MFPM.addPass(DetectDeadLanesPass());
+ MFPM.addPass(InitUndefPass());
+ MFPM.addPass(ProcessImplicitDefsPass());
+
+ // LiveVariables currently requires pure SSA form.
+ //
+ // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+ // LiveVariables can be removed completely, and LiveIntervals can be
+ // directly computed. (We still either need to regenerate kill flags after
+ // regalloc, or preferably fix the scavenger to not depend on them).
+ // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+ // When LiveVariables is removed this has to be removed/moved either.
+ // Explicit addition of UnreachableMachineBlockElim allows stopping before
+ // or after it with -stop-before/-stop-after.
+ MFPM.addPass(UnreachableMachineBlockElimPass());
+ MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>());
+
+ // Edge splitting is smarter with machine loop info.
+ MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>());
+ MFPM.addPass(PHIEliminationPass());
+
+ if (CGPBO.EarlyLiveIntervals)
+ MFPM.addPass(
+ RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
+
+ MFPM.addPass(TwoAddressInstructionPass());
+ MFPM.addPass(RegisterCoalescerPass());
+
+ // The machine scheduler may accidentally create disconnected components
+ // when moving subregister definitions around, avoid this by splitting them
+ // to separate vregs before. Splitting can also improve reg. allocation
+ // quality.
+ MFPM.addPass(RenameIndependentSubregsPass());
+
+ // PreRA instruction scheduling.
+ MFPM.addPass(MachineSchedulerPass());
+
+ if (auto Err = addRegAllocPass(MFPM))
+ return Err;
+ // Finally rewrite virtual registers.
+ MFPM.addPass(VirtRegRewriterPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ MFPM.addPass(RegAllocScoringPass());
+ } else {
+ if (AddFastRegAllocCallback)
+ return AddFastRegAllocCallback(MFPM);
+
+ MFPM.addPass(PHIEliminationPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+ if (auto Err = addRegAllocPass(MFPM))
+ return Err;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM,
+ StringRef FilterName) {
+ std::optional<RegAllocFilterFunc> FilterFunc =
+ parseRegAllocFilter(FilterName);
+ if (!FilterFunc) {
+ return make_error<StringError>(
+ formatv("Unknown register filter name: {0}", FilterName).str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+
+ if (RegAllocPasses.contains(FilterName)) {
+ MFPM.addPass(std::move(RegAllocPasses[FilterName]));
+ return Error::success();
+ }
+
+ // Add default register allocator.
+ const bool Optimized = CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() !=
+ CodeGenOptLevel::None);
+ if (Optimized) {
+ MFPM.addPass(RAGreedyPass());
+ } else {
+ RegAllocFastPassOptions Opts;
+ Opts.Filter = *FilterFunc;
+ Opts.FilterName = FilterName;
+ MFPM.addPass(RegAllocFastPass(Opts));
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addMachinePasses(ModulePassManager &MPM,
+ FunctionPassManager &FPM,
+ MachineFunctionPassManager &MFPM,
+ raw_pwrite_stream *Out) {
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
+ // FinalizeISel.
+ MFPM.addPass(FinalizeISelPass());
+
+ // Add passes that optimize machine instructions in SSA form.
+ if (OptLevel != CodeGenOptLevel::None) {
+ invokeMachineSSAOptimizationEarlyEPCallbacks(MFPM);
+ addMachineSSAOptimizationPasses(MFPM);
+ invokeMachineSSAOptimizationLastEPCallbacks(MFPM);
+ } else {
+ MFPM.addPass(LocalStackSlotAllocationPass());
+ }
+
+ if (TM->Options.EnableIPRA)
+ MFPM.addPass(RegUsageInfoPropagationPass());
+
+ // Run pre-ra passes.
+ invokePreRegAllocEPCallbacks(MFPM);
+
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::Pass1, nullptr));
+ }
+
+ if (auto Err = addRegisterAllocatorPasses(MFPM))
+ return Err;
+
+ invokePostRegAllocEPCallbacks(MFPM);
+
+ MFPM.addPass(RemoveRedundantDebugValuesPass());
+ MFPM.addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (OptLevel != CodeGenOptLevel::None) {
+ MFPM.addPass(PostRAMachineSinkingPass());
+ MFPM.addPass(ShrinkWrapPass());
+ }
+
+ if (!CGPBO.DisablePrologEpilogInserterPass)
+ MFPM.addPass(PrologEpilogInserterPass());
+ /// Add passes that optimize machine instructions after register allocation.
+ if (OptLevel != CodeGenOptLevel::None)
+ invokeMachineLateOptimizationEPCallbacks(MFPM);
+
+ // Expand pseudo instructions before second scheduling pass.
+ MFPM.addPass(ExpandPostRAPseudosPass());
+
+ // Run pre-sched2 passes.
+ invokePreSched2EPCallbacks(MFPM);
+
+ if (CGPBO.EnableImplicitNullChecks)
+ MFPM.addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other
+ // point.
+ if (OptLevel != CodeGenOptLevel::None &&
+ !TM->targetSchedulesPostRAScheduling()) {
+ if (CGPBO.MISchedPostRA)
+ MFPM.addPass(PostMachineSchedulerPass());
+ else
+ MFPM.addPass(PostRASchedulerPass());
+ }
+
+ // GC, replacement for GCMachineCodeAnalysis
+ MFPM.addPass(GCMachineCodeInsertionPass());
+
+ // Basic block placement.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2));
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::Pass2, nullptr));
+ }
+ MFPM.addPass(MachineBlockPlacementPass());
+ // Run a separate pass to collect block placement statistics.
+ if (CGPBO.EnableBlockPlacementStats)
+ MFPM.addPass(MachineBlockPlacementStatsPass());
+ }
+
+ // Insert before XRay Instrumentation.
+ MFPM.addPass(FEntryInserterPass());
+ MFPM.addPass(XRayInstrumentationPass());
+ MFPM.addPass(PatchableFunctionPass());
+
+ invokePreEmitEPCallbacks(MFPM);
+
+ if (TM->Options.EnableIPRA)
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ MFPM.addPass(RegUsageInfoCollectorPass());
+
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
+ MFPM.addPass(FuncletLayoutPass());
+
+ MFPM.addPass(StackMapLivenessPass());
+ MFPM.addPass(LiveDebugValuesPass());
+ MFPM.addPass(MachineSanitizerBinaryMetadata());
+
+ if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None &&
+ CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
+ if (AddOutliner) {
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(MachineOutlinerPass(RunOnAllFunctions));
+ FPM = FunctionPassManager();
+ MFPM = MachineFunctionPassManager();
+ }
+ }
+
+ if (CGPBO.GCEmptyBlocks)
+ MFPM.addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ bool NeedsBBSections =
+ TM->getBBSectionsType() != llvm::BasicBlockSection::None;
+ // Machine function splitter uses the basic block sections feature. Both
+ // cannot be enabled at the same time. We do not apply machine function
+ // splitter if -basic-block-sections is requested.
+ if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter ||
+ CGPBO.EnableMachineFunctionSplitter)) {
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.\n";
+ }
+ }
+ MFPM.addPass(MachineFunctionSplitterPass());
+ }
+
+ // We run the BasicBlockSections pass if either we need BB sections or BB
+ // address map (or both).
+ if (NeedsBBSections || TM->Options.BBAddrMap) {
+ if (TM->getBBSectionsType() == llvm::BasicBlockSection::List)
+ MFPM.addPass(BasicBlockPathCloningPass());
+ MFPM.addPass(BasicBlockSectionsPass());
+ }
+
+ invokePostBBSectionsEPCallbacks(MFPM);
+
+ if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup)
+ MFPM.addPass(CFIFixupPass());
+
+ MFPM.addPass(StackFrameLayoutAnalysisPass());
+
+ // Add passes that directly emit MI after all other MI passes.
+ invokeMIEmitEPCallbacks(MFPM);
+
+ if (Out)
+ MFPM.addPass(PrintMIRPass(*Out));
+ return Error::success();
+}
+
+Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx) {
+ if (!TM)
+ return make_error<StringError>("Need a TargetMachine instance!",
+ inconvertibleErrorCode());
+
+ if (CustomCodeGenPipelineBuilderCallback)
+ return CustomCodeGenPipelineBuilderCallback(TopLevelMPM, Out, DwoOut,
+ FileType, Ctx);
+
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+ if (auto Err = parseRegAllocOption(CGPBO.RegAlloc))
+ return Err;
+
+ bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
+ bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
+
+ ModulePassManager MPM;
+ FunctionPassManager FPM;
+ MachineFunctionPassManager MFPM;
+
+ if (!CGPBO.DisableVerify)
+ MPM.addPass(VerifierPass());
+
+ // IR part
+ // TODO: Remove RequireAnalysisPass<MachineModuleAnalysis, Module>()
+ // when we port AsmPrinter.
+ MPM.addPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
+
+ invokeCodeGenIREarlyEPCallbacks(MPM);
+
+ if (TM->useEmulatedTLS())
+ MPM.addPass(LowerEmuTLSPass());
+ MPM.addPass(PreISelIntrinsicLoweringPass(TM));
+
+ // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
+ // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
+ if (TM->getTargetTriple().isOSBinFormatMachO() &&
+ !CGPBO.DisableAtExitBasedGlobalDtorLowering)
+ MPM.addPass(LowerGlobalDtorsPass());
+
+ FPM.addPass(ExpandLargeDivRemPass(TM));
+ FPM.addPass(ExpandLargeFpConvertPass(TM));
+
+ // Run loop strength reduction before anything else.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableLSR) {
+ LoopPassManager LPM;
+ LPM.addPass(LoopStrengthReducePass());
+ FPM.addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
+ /*UseMemorySSA=*/true));
+ }
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
+ // loads and compares. ExpandMemCmpPass then tries to expand those calls
+ // into optimally-sized loads and compares. The transforms are enabled by a
+ // target lowering hook.
+ if (!CGPBO.DisableMergeICmps)
+ FPM.addPass(MergeICmpsPass());
+ FPM.addPass(ExpandMemCmpPass(TM));
+ }
+
+ // Run GC lowering passes for builtin collectors
+ FPM.addPass(GCLoweringPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(ShadowStackGCLoweringPass());
+ FPM = FunctionPassManager();
+ invokeGCLoweringEPCallbacks(FPM);
+
+ // Make sure that no unreachable blocks are instruction selected.
+ FPM.addPass(UnreachableBlockElimPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableConstantHoisting)
+ FPM.addPass(ConstantHoistingPass());
+ if (!CGPBO.DisableReplaceWithVecLib)
+ FPM.addPass(ReplaceWithVeclib());
+ if (!CGPBO.DisablePartialLibcallInlining)
+ FPM.addPass(PartiallyInlineLibCallsPass());
+ }
+
+ // Instrument function entry after all inlining.
+ FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ FPM.addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants to.
+ // Allow disabling it for testing purposes.
+ if (!CGPBO.DisableExpandReductions)
+ FPM.addPass(ExpandReductionsPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ FPM.addPass(TLSVariableHoistPass());
+
+ // Convert conditional moves to conditional jumps when profitable.
+ if (!CGPBO.DisableSelectOptimize)
+ FPM.addPass(SelectOptimizePass(TM));
+ }
+
+ {
+ ModulePassManager EPMPM;
+ invokeCodeGenIRLateEPCallbacks(EPMPM);
+ if (!EPMPM.isEmpty()) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(std::move(EPMPM));
+ FPM = FunctionPassManager();
+ }
+ }
+
+ // CodeGen prepare
+ if (OptLevel != CodeGenOptLevel::None && !CGPBO.DisableCGP)
+ FPM.addPass(CodeGenPreparePass(TM));
+
+ invokeCodeGenIRFinalEPCallbacks(FPM);
+
+ // Now all IR passes are added.
+
+ // Turn exception handling constructs into something the code generators can
+ // handle.
+ if (auto Err = addExceptionHandlingPasses(FPM))
+ return Err;
+
+ { // Pre isel extension
+ ModulePassManager ISelPreparePasses;
+ invokeISelPrepareEPCallbacks(ISelPreparePasses);
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(std::move(ISelPreparePasses));
+ }
+
+ if (OptLevel != CodeGenOptLevel::None)
+ FPM.addPass(ObjCARCContractPass());
+ FPM.addPass(CallBrPreparePass());
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ FPM.addPass(SafeStackPass(TM));
+ FPM.addPass(StackProtectorPass(TM));
+
+ // All passes which modify the LLVM IR are now complete; run the verifier
+ // to ensure that the IR is valid.
+ if (!CGPBO.DisableVerify)
+ FPM.addPass(VerifierPass());
+
+ if (PrintMIR) {
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(PrintMIRPreparePass(Out));
+ FPM = FunctionPassManager();
+ }
+
+ if (auto Err = addInstructionSelectorPasses(MFPM))
+ return Err;
+
+ if (auto Err = addMachinePasses(MPM, FPM, MFPM, PrintMIR ? &Out : nullptr))
+ return Err;
+
+ if (!CGPBO.DisableVerify)
+ MFPM.addPass(MachineVerifierPass());
+
+ // TODO: Add AsmPrinter.
+ (void)Ctx;
+
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+ MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); });
+ if (auto Err = setStartStop(MPM, PIC))
+ return Err;
+ TopLevelMPM.addPass(std::move(MPM));
+ return Error::success();
+}
+
+Expected<ModulePassManager> PassBuilder::buildDefaultCodeGenPipeline(
+ raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+ MCContext &Ctx) {
+ ModulePassManager MPM;
+ Error Err = buildDefaultCodeGenPipeline(MPM, Out, DwoOut, FileType, Ctx);
+ if (Err)
+ return std::move(Err);
+ return std::move(MPM);
+}
+
+static bool isRegAllocPass(StringRef Name) {
+ // TODO: Add all register allocators.
+ return Name.starts_with("regallocfast");
+}
+
+Error PassBuilder::parseRegAllocOption(StringRef Text) {
+ if (Text == "default")
+ return Error::success();
+
+ RegAllocPasses.clear();
+ while (!Text.empty()) {
+ StringRef SinglePass;
+ std::tie(SinglePass, Text) = Text.split(',');
+
+ if (!isRegAllocPass(SinglePass)) {
+ return make_error<StringError>(
+ formatv("{0} is not a register allocator!", SinglePass).str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ if (!isOptimizedRegAlloc() && !SinglePass.starts_with("regallocfast")) {
+ return make_error<StringError>(
+ "Must use fast (default) register allocator for unoptimized "
+ "regalloc.",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+
+ MachineFunctionPassManager MFPM;
+ if (auto Err = parsePassPipeline(MFPM, SinglePass))
+ return Err;
+
+ auto FilterPos = SinglePass.find("filter=");
+ if (FilterPos == std::string::npos) {
+ bool Success = RegAllocPasses.try_emplace("all", std::move(MFPM)).second;
+ if (!Success) {
+ return make_error<StringError>(
+ formatv("Already set register allocator '{0}' for all registers!",
+ SinglePass)
+ .str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ continue;
+ }
+
+ StringRef FilterName = SinglePass.drop_front(FilterPos);
+ FilterName.consume_front("filter=");
+ FilterName =
+ FilterName.take_until([](char C) { return C == ';' || C == '>'; });
+ bool Success =
+ RegAllocPasses.try_emplace(FilterName, std::move(MFPM)).second;
+ if (!Success) {
+ return make_error<StringError>(
+ formatv("Already set register allocator '{0}' for filter {1}!",
+ SinglePass, FilterName)
+ .str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ }
+ return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 9c9c5051393730..4862a082554586 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -52,9 +52,11 @@
#include "llvm/CodeGen/GlobalISel/Localizer.h"
#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
#include "llvm/CodeGen/MIRParser/MIParser.h"
+#include "llvm/CodeGen/PHIElimination.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TwoAddressInstructionPass.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
@@ -684,6 +686,8 @@ parseAMDGPUAttributorPassOptions(StringRef Params) {
}
void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
+ CGPassBuilderOption &CGPTO = PB.getCGPBO();
+ CGPTO.RequiresCodeGenSCCOrder = true;
#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
#include "llvm/Passes/TargetPassRegistry.inc"
@@ -785,6 +789,26 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
return onlyAllocateVGPRs;
return nullptr;
});
+
+ PB.setAddInstSelectorCallback([&](MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(AMDGPUISelDAGToDAGPass(*this));
+ MFPM.addPass(SIFixSGPRCopiesPass());
+ MFPM.addPass(SILowerI1CopiesPass());
+ });
+
+ PB.setCustomFastRegAllocPipelineBuilder(
+ [&](MachineFunctionPassManager &MFPM) -> Error {
+ // TODO: Add complete pipeline.
+ MFPM.addPass(PHIEliminationPass());
+ // MFPM.addPass(SILowerControlFlowPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+ // MFPM.addPass(SIWholeQuadModePass());
+ if (auto Err = PB.addRegAllocPass(MFPM, "sgpr"))
+ return Err;
+ if (auto Err = PB.addRegAllocPass(MFPM, "vgpr"))
+ return Err;
+ return Error::success();
+ });
}
int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) {
diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
index d979517e12af6d..f911b45b76fccd 100644
--- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
@@ -53,6 +53,10 @@ Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
#define GET_PASS_REGISTRY "X86PassRegistry.def"
#include "llvm/Passes/TargetPassRegistry.inc"
+
+ PB.setAddInstSelectorCallback([this](MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(X86ISelDAGToDAGPass(*this));
+ });
}
Error X86TargetMachine::buildCodeGenPipeline(
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index db8f31f12ab726..45c69692ec0bef 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -65,8 +65,10 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
LPMUpdater &>::eraseIf(function_ref<bool(StringRef)> Pred) {
- std::vector<char> IsLoopNestPassVec(
- static_cast<size_t>(IsLoopNestPass.size()));
+ assert(LoopPasses.size() + LoopNestPasses.size() == IsLoopNestPass.size() &&
+ "Wrong precondition!");
+
+ std::vector<char> IsLoopNestPassVec(IsLoopNestPass.size());
for (unsigned Idx = 0, Sz = IsLoopNestPass.size(); Idx != Sz; ++Idx)
IsLoopNestPassVec[Idx] = IsLoopNestPass[Idx];
@@ -94,6 +96,9 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
IsLoopNestPass.clear();
for (const auto I : IsLoopNestPassVec)
IsLoopNestPass.push_back(I);
+
+ assert(LoopPasses.size() + LoopNestPasses.size() == IsLoopNestPass.size() &&
+ "Wrong postcondition!");
}
// Run both loop passes and loop-nest passes on top-level loop \p L.
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-select.ll b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll
new file mode 100644
index 00000000000000..7327e1bf95e1e4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll
@@ -0,0 +1,9 @@
+; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=DEFAULT
+; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -regalloc-npm='regallocfast<filter=sgpr;no-clear-vregs>' -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=CUSTOM
+
+; DEFAULT: regallocfast<filter=sgpr>
+; DEFAULT: regallocfast<filter=vgpr>
+
+; Just a proof of concept that we can modify parameters of register allocator.
+; CUSTOM: regallocfast<filter=sgpr;no-clear-vregs>
+; CUSTOM: regallocfast<filter=vgpr>
diff --git a/llvm/test/tools/llc/new-pm/start-stop.ll b/llvm/test/tools/llc/new-pm/start-stop.ll
index 9c3b9f009178fe..48a6104c91ad96 100644
--- a/llvm/test/tools/llc/new-pm/start-stop.ll
+++ b/llvm/test/tools/llc/new-pm/start-stop.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -filetype=null %s | FileCheck --match-full-lines %s --check-prefix=NULL
; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -o /dev/null %s | FileCheck --match-full-lines %s --check-prefix=OBJ
-; NULL: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument<post-inline>,verify)
-; OBJ: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument<post-inline>,verify),PrintMIRPreparePass,function(machine-function(print),invalidate<machine-function-info>)
+; NULL: verify,require<MachineModuleAnalysis>,function(mergeicmps,expand-memcmp,gc-lowering),function(verify,machine-function(verify),invalidate<machine-function-info>)
+; OBJ: verify,require<MachineModuleAnalysis>,function(mergeicmps,expand-memcmp,gc-lowering),function(verify),PrintMIRPreparePass,function(machine-function(print,verify),invalidate<machine-function-info>)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 2a49eaff1d7cbc..445a89d55dc389 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -100,20 +100,10 @@ int llvm::compileModuleWithNewPM(
}
LLVMTargetMachine &LLVMTM = static_cast<LLVMTargetMachine &>(*Target);
-
raw_pwrite_stream *OS = &Out->os();
-
- // Fetch options from TargetPassConfig
- CGPassBuilderOption Opt = getCGPassBuilderOption();
- Opt.DisableVerify = VK != VerifierKind::InputOutput;
- Opt.DebugPM = DebugPM;
- Opt.RegAlloc = RegAlloc;
-
MachineModuleInfo MMI(&LLVMTM);
PassInstrumentationCallbacks PIC;
- StandardInstrumentations SI(Context, Opt.DebugPM,
- VK == VerifierKind::EachPass);
registerCodeGenCallback(PIC, LLVMTM);
MachineFunctionAnalysisManager MFAM;
@@ -121,13 +111,19 @@ int llvm::compileModuleWithNewPM(
FunctionAnalysisManager FAM;
CGSCCAnalysisManager CGAM;
ModuleAnalysisManager MAM;
+
PassBuilder PB(Target.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+ CGPassBuilderOption &CGPBO = PB.getCGPBO();
+ CGPBO.DisableVerify = VK != VerifierKind::InputOutput;
+ CGPBO.RegAlloc = RegAlloc;
PB.registerModuleAnalyses(MAM);
PB.registerCGSCCAnalyses(CGAM);
PB.registerFunctionAnalyses(FAM);
PB.registerLoopAnalyses(LAM);
PB.registerMachineFunctionAnalyses(MFAM);
PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+ StandardInstrumentations SI(Context, DebugPM, VK == VerifierKind::EachPass);
SI.registerCallbacks(PIC, &MAM);
FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
@@ -158,8 +154,9 @@ int llvm::compileModuleWithNewPM(
if (MIR->parseMachineFunctions(*M, MAM))
return 1;
} else {
- ExitOnErr(LLVMTM.buildCodeGenPipeline(
- MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC));
+ ExitOnErr(PB.buildDefaultCodeGenPipeline(MPM, *OS,
+ DwoOut ? &DwoOut->os() : nullptr,
+ FileType, MMI.getContext()));
}
if (PrintPipelinePasses) {
More information about the llvm-commits
mailing list