[llvm] [NewPM][CodeGen] Add callback style codegen pass builder (PR #108690)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Nov 16 22:13:01 PST 2024
https://github.com/paperchalice updated https://github.com/llvm/llvm-project/pull/108690
>From 4944da9e37363ab73f7dc0f0c053d0e61e7772ef Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 19 Aug 2024 08:19:28 +0800
Subject: [PATCH 1/5] Support eraseIf in pass manager
---
llvm/include/llvm/Analysis/CGSCCPassManager.h | 6 +++
.../include/llvm/CodeGen/MachinePassManager.h | 4 ++
llvm/include/llvm/IR/PassManager.h | 19 ++++++++
llvm/include/llvm/IR/PassManagerInternal.h | 35 +++++++++++++++
.../llvm/Transforms/Scalar/LoopPassManager.h | 8 ++++
llvm/lib/Analysis/CGSCCPassManager.cpp | 23 ++++++++++
llvm/lib/IR/PassManager.cpp | 11 +++++
.../lib/Transforms/Scalar/LoopPassManager.cpp | 44 +++++++++++++++++++
8 files changed, 150 insertions(+)
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 15b7f226fd8283..4a49bfb283b6a2 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -346,6 +346,10 @@ class ModuleToPostOrderCGSCCPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
};
@@ -488,6 +492,8 @@ class CGSCCToFunctionPassAdaptor
static bool isRequired() { return true; }
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
bool EagerlyInvalidate;
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 69b5f6e92940c4..3d7c7d87af480a 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -207,6 +207,10 @@ class FunctionToMachineFunctionPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
};
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index d269221fac0701..4f28b4aceb3e66 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -218,6 +218,21 @@ class PassManager : public PassInfoMixin<
static bool isRequired() { return true; }
+ /// Erase all passes that satisfy the predicate \p Pred.
+ /// For internal use only!
+ void eraseIf(function_ref<bool(StringRef)> Pred) {
+ for (auto I = Passes.begin(); I != Passes.end();) {
+ (*I)->eraseIf(Pred);
+ bool IsSpecial = (*I)->name().ends_with("PassAdaptor") ||
+ (*I)->name().contains("PassManager");
+ bool PredResult = Pred((*I)->name());
+ if ((!IsSpecial && PredResult) || (IsSpecial && (*I)->isEmpty()))
+ I = Passes.erase(I);
+ else
+ ++I;
+ }
+ }
+
protected:
using PassConceptT =
detail::PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...>;
@@ -836,6 +851,10 @@ class ModuleToFunctionPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
private:
std::unique_ptr<PassConceptT> Pass;
bool EagerlyInvalidate;
diff --git a/llvm/include/llvm/IR/PassManagerInternal.h b/llvm/include/llvm/IR/PassManagerInternal.h
index 4ada6ee5dd6831..3a75c9c4e97faf 100644
--- a/llvm/include/llvm/IR/PassManagerInternal.h
+++ b/llvm/include/llvm/IR/PassManagerInternal.h
@@ -59,6 +59,13 @@ struct PassConcept {
/// To opt-in, pass should implement `static bool isRequired()`. It's no-op
/// to have `isRequired` always return false since that is the default.
virtual bool isRequired() const = 0;
+
+ /// Polymorphic method to refurbish pass pipeline.
+ virtual void eraseIf(function_ref<bool(StringRef)> Pred) = 0;
+
+ /// There may be some empty PassManager after erasing,
+ /// use it to remove them.
+ virtual bool isEmpty() const = 0;
};
/// A template wrapper used to implement the polymorphic API.
@@ -114,6 +121,34 @@ struct PassModel : PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...> {
bool isRequired() const override { return passIsRequiredImpl<PassT>(); }
+ template <typename T>
+ using has_erase_if_t = decltype(std::declval<T &>().eraseIf(
+ std::declval<function_ref<bool(StringRef)>>()));
+
+ template <typename T>
+ std::enable_if_t<is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)> Pred) {
+ Pass.eraseIf(Pred);
+ }
+
+ template <typename T>
+ std::enable_if_t<!is_detected<has_erase_if_t, T>::value>
+ eraseIfImpl(function_ref<bool(StringRef)>) {}
+
+ void eraseIf(function_ref<bool(StringRef)> Pred) override {
+ eraseIfImpl<PassT>(Pred);
+ }
+
+ template <typename T>
+ using has_is_empty_t = decltype(std::declval<T &>().isEmpty());
+
+ bool isEmpty() const override {
+ if constexpr (is_detected<has_is_empty_t, PassT>::value)
+ return Pass.isEmpty();
+ else
+ return false;
+ }
+
PassT Pass;
};
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index f55022fbff07c1..33ebbd711bec04 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -134,6 +134,10 @@ class PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
static bool isRequired() { return true; }
+ /// Erase all passes that satisfy the predicate \p Pred.
+ /// For internal use only!
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
size_t getNumLoopPasses() const { return LoopPasses.size(); }
size_t getNumLoopNestPasses() const { return LoopNestPasses.size(); }
@@ -425,6 +429,10 @@ class FunctionToLoopPassAdaptor
static bool isRequired() { return true; }
+ bool isEmpty() const { return Pass == nullptr; }
+
+ void eraseIf(function_ref<bool(StringRef)> Pred);
+
bool isLoopNestMode() const { return LoopNestMode; }
private:
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 948bc2435ab275..d87ed86655ab30 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -576,6 +576,17 @@ PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C,
return PA;
}
+void CGSCCToFunctionPassAdaptor::eraseIf(function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
bool CGSCCAnalysisManagerModuleProxy::Result::invalidate(
Module &M, const PreservedAnalyses &PA,
ModuleAnalysisManager::Invalidator &Inv) {
@@ -751,6 +762,18 @@ bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
return false;
}
+void ModuleToPostOrderCGSCCPassAdaptor::eraseIf(
+ function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
} // end namespace llvm
/// When a new SCC is created for the graph we first update the
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index bf8f7906d3368d..0c9ee99f24dd19 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -145,6 +145,17 @@ PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M,
return PA;
}
+void ModuleToFunctionPassAdaptor::eraseIf(function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
template <>
void llvm::printIRUnitNameForStackTrace<Module>(raw_ostream &OS,
const Module &IR) {
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 9f4270f5d62f5c..081303c2dda4c9 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -62,6 +62,39 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
}
}
+void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
+ LPMUpdater &>::eraseIf(function_ref<bool(StringRef)> Pred) {
+ std::vector<char> IsLoopNestPassVec(
+ static_cast<size_t>(IsLoopNestPass.size()));
+ for (unsigned Idx = 0, Sz = IsLoopNestPass.size(); Idx != Sz; ++Idx)
+ IsLoopNestPassVec[Idx] = IsLoopNestPass[Idx];
+
+ auto ILP = LoopPasses.begin();
+ auto ILNP = LoopNestPasses.begin();
+ for (auto I = IsLoopNestPassVec.begin(); I != IsLoopNestPassVec.end();) {
+ if (*I) {
+ if (Pred((*ILNP)->name())) {
+ I = IsLoopNestPassVec.erase(I);
+ ILNP = LoopNestPasses.erase(ILNP);
+ continue;
+ }
+ ++ILNP;
+ } else {
+ if (Pred((*ILP)->name())) {
+ I = IsLoopNestPassVec.erase(I);
+ ILP = LoopPasses.erase(ILP);
+ continue;
+ }
+ ++ILP;
+ }
+ ++I;
+ }
+
+ IsLoopNestPass.clear();
+ for (const auto I : IsLoopNestPassVec)
+ IsLoopNestPass.push_back(I);
+}
+
// Run both loop passes and loop-nest passes on top-level loop \p L.
PreservedAnalyses
LoopPassManager::runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
@@ -359,6 +392,17 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
return PA;
}
+void FunctionToLoopPassAdaptor::eraseIf(function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
PrintLoopPass::PrintLoopPass() : OS(dbgs()) {}
PrintLoopPass::PrintLoopPass(raw_ostream &OS, const std::string &Banner)
: OS(OS), Banner(Banner) {}
>From 2a9a04ebd213f2edc729d1ae6802d49715bb23df Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Sat, 14 Sep 2024 16:38:47 +0800
Subject: [PATCH 2/5] [NewPM] Add callback style codegen pass builder
---
llvm/include/llvm/CodeGen/TargetPassConfig.h | 3 +-
llvm/include/llvm/Passes/CodeGenPassBuilder.h | 2 +-
.../llvm/Passes/MachinePassRegistry.def | 5 +
llvm/include/llvm/Passes/PassBuilder.h | 236 +++++
.../include/llvm/Target/CGPassBuilderOption.h | 3 +
llvm/lib/CodeGen/MachinePassManager.cpp | 12 +
llvm/lib/CodeGen/TargetPassConfig.cpp | 9 +-
llvm/lib/Passes/CMakeLists.txt | 1 +
llvm/lib/Passes/PassBuilder.cpp | 19 +-
llvm/lib/Passes/PassBuilderCodeGen.cpp | 950 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 29 +
llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp | 4 +
.../lib/Transforms/Scalar/LoopPassManager.cpp | 9 +-
llvm/test/CodeGen/AMDGPU/regalloc-select.ll | 9 +
llvm/test/tools/llc/new-pm/start-stop.ll | 4 +-
llvm/tools/llc/NewPMDriver.cpp | 8 +-
16 files changed, 1288 insertions(+), 15 deletions(-)
create mode 100644 llvm/lib/Passes/PassBuilderCodeGen.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/regalloc-select.ll
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index 66c79c74f2be55..5a5cd24ee577b4 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -190,8 +190,7 @@ class TargetPassConfig : public ImmutablePass {
/// Returns pass name in `-stop-before` or `-stop-after`
/// NOTE: New pass manager migration only
- static Expected<StartStopInfo>
- getStartStopInfo(PassInstrumentationCallbacks &PIC);
+ static Expected<StartStopInfo> getStartStopInfo();
void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index ab16d79900d851..4f80af4fa487e9 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -533,7 +533,7 @@ template <typename Derived, typename TargetMachineT>
Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
CodeGenFileType FileType) const {
- auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
+ auto StartStopInfo = TargetPassConfig::getStartStopInfo();
if (!StartStopInfo)
return StartStopInfo.takeError();
setStartStopPasses(*StartStopInfo);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 375709e0a05f4c..49a2398ddf8453 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -205,6 +205,7 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
#ifndef DUMMY_MACHINE_FUNCTION_PASS
#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
#endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass)
@@ -221,6 +222,8 @@ DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCall
DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass)
DUMMY_MACHINE_FUNCTION_PASS("gc-empty-basic-blocks", GCEmptyBasicBlocksPass)
+DUMMY_MACHINE_FUNCTION_PASS("gc-machine-code-insersion",
+ GCMachineCodeInsertionPass)
DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass)
DUMMY_MACHINE_FUNCTION_PASS("init-undef-pass", InitUndefPass)
DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass)
@@ -266,6 +269,8 @@ DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
+DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination",
+ UnreachableMachineBlockElimPass)
DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index e7bc3a58f414f1..dfb27925db7221 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -15,14 +15,17 @@
#ifndef LLVM_PASSES_PASSBUILDER_H
#define LLVM_PASSES_PASSBUILDER_H
+#include "llvm/ADT/StringSet.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/RegAllocCommon.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Passes/OptimizationLevel.h"
+#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/PGOOptions.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
@@ -34,6 +37,7 @@ class StringRef;
class AAManager;
class TargetMachine;
class ModuleSummaryIndex;
+class MCContext;
template <typename T> class IntrusiveRefCntPtr;
namespace vfs {
class FileSystem;
@@ -106,6 +110,7 @@ class PassBuilder {
TargetMachine *TM;
PipelineTuningOptions PTO;
std::optional<PGOOptions> PGOOpt;
+ CGPassBuilderOption CGPBO;
PassInstrumentationCallbacks *PIC;
public:
@@ -308,6 +313,24 @@ class PassBuilder {
/// TargetMachine::registerDefaultAliasAnalyses().
AAManager buildDefaultAAPipeline();
+ /// Build CodeGen pass pipeline.
+ ///
+ /// {{@
+ Expected<ModulePassManager>
+ buildDefaultCodeGenPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error buildDefaultCodeGenPipeline(ModulePassManager &MPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+ Error addRegAllocPass(MachineFunctionPassManager &MFPM,
+ StringRef Filter = "all");
+ // TODO: Add method to build MC emission pipeline.
+ template <typename... PassTs> void disablePass() {
+ (DisabledPasses.insert(PassTs::name()), ...);
+ }
+ /// @}}
+
/// Parse a textual pass pipeline description into a \c
/// ModulePassManager.
///
@@ -523,6 +546,133 @@ class PassBuilder {
FullLinkTimeOptimizationLastEPCallbacks.push_back(C);
}
+ /// Register target specific callbacks to extend codegen pipeline.
+ /// {{@
+
+ /// If target want its own pipeline, use this callback.
+ void setCustomCodeGenPipelineBuilderCallback(
+ const std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+ raw_pwrite_stream *, CodeGenFileType,
+ MCContext &)>
+ C) {
+ CustomCodeGenPipelineBuilderCallback = C;
+ }
+
+ void registerCodeGenIREarlyEPCallback(
+ const std::function<void(ModulePassManager &)> C) {
+ CodeGenIREarlyEPCallbacks.push_back(C);
+ }
+
+ void registerGCLoweringEPCallback(
+ const std::function<void(FunctionPassManager &)> C) {
+ GCLoweringEPCallbacks.push_back(C);
+ }
+
+ void registerISelPrepareEPCallback(
+ const std::function<void(ModulePassManager &)> &C) {
+ ISelPrepareEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationEarlyEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineSSAOptimizationEarlyEPCallbacks.push_back(C);
+ }
+
+ void registerILPOptsEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ ILPOptsEPCallbacks.push_back(C);
+ }
+
+ void registerMachineSSAOptimizationLastEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineSSAOptimizationLastEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPostRegAllocEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostRegAllocEPCallbacks.push_back(C);
+ }
+
+ void registerPreRegBankSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreRegBankSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPreGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerPostGlobalInstructionSelectEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostGlobalInstructionSelectEPCallbacks.push_back(C);
+ }
+
+ void registerMachineLateOptimizationEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MachineLateOptimizationEPCallbacks.push_back(C);
+ }
+
+ void registerPreSched2EPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreSched2EPCallbacks.push_back(C);
+ }
+
+ void registerPostRewriteEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostRewriteEPCallbacks.push_back(C);
+ }
+
+ void registerPreEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PreEmitEPCallbacks.push_back(C);
+ }
+
+ void registerPostBBSectionsEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ PostBBSectionsEPCallbacks.push_back(C);
+ }
+
+ void registerMIEmitEPCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ MIEmitEPCallbacks.push_back(C);
+ }
+
+ void setAddInstSelectorCallback(
+ const std::function<void(MachineFunctionPassManager &)> &C) {
+ AddInstSelectorCallback = C;
+ }
+
+ void setCodeGenPreparePassesCallback(
+ const std::function<void(ModulePassManager &)> C) {
+ AddCodeGenPreparePassesCallback = C;
+ }
+
+ void setRegAllocFastCallback(
+ const std::function<Error(MachineFunctionPassManager &)> &C) {
+ AddRegAllocFastCallback = C;
+ }
+
+ void setRegAllocOptimizedCallback(
+ const std::function<Error(MachineFunctionPassManager &)> &C) {
+ AddRegAllocOptimizedCallback = C;
+ }
+ ///@}}
+
+ /// Building block callbacks for codegen pipeline.
+ void addDefaultCodeGenPreparePasses(ModulePassManager &MPM);
+ Error addDefaultRegAllocFastPasses(MachineFunctionPassManager &MFPM);
+ Error addDefaultRegAllocOptimizedPasses(MachineFunctionPassManager &MFPM);
+
+ // New pass manager migration methods, don't use them
+ // outside llvm!
+ CGPassBuilderOption &getCGPBO() { return CGPBO; }
+
/// Register a callback for parsing an AliasAnalysis Name to populate
/// the given AAManager \p AA
void registerParseAACallback(
@@ -647,6 +797,28 @@ class PassBuilder {
OptimizationLevel Level,
ThinOrFullLTOPhase Phase);
+ void invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM);
+ void invokeGCLoweringEPCallbacks(FunctionPassManager &FPM);
+ void invokeISelPrepareEPCallbacks(ModulePassManager &MPM);
+ void invokeMachineSSAOptimizationEarlyEPCallbacks(
+ MachineFunctionPassManager &MFPM);
+ void
+ invokeMachineSSAOptimizationLastEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreRegBankSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+ void
+ invokePreGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM);
+ void invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM);
+ void
+ invokeMachineLateOptimizationEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePostBBSectionsEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+ void invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM);
+
static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
if (!Name.consume_front(PassName))
return false;
@@ -711,6 +883,21 @@ class PassBuilder {
void addVectorPasses(OptimizationLevel Level, FunctionPassManager &FPM,
bool IsFullLTO);
+ Error addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+ Error addInstructionSelectorPasses(MachineFunctionPassManager &MFPM);
+
+ void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM);
+
+ Error addMachinePasses(ModulePassManager &MPM, FunctionPassManager &FPM,
+ MachineFunctionPassManager &MFPM);
+
+ Error addRegisterAllocatorPasses(MachineFunctionPassManager &MFPM);
+
+ Error parseRegAllocOption(StringRef Text);
+
+ bool isOptimizedRegAlloc() const;
+
static std::optional<std::vector<PipelineElement>>
parsePipelineText(StringRef Text);
@@ -779,6 +966,55 @@ class PassBuilder {
2>
PipelineEarlySimplificationEPCallbacks;
+ // CodeGen extension point callbacks
+ std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+ raw_pwrite_stream *, CodeGenFileType, MCContext &)>
+ CustomCodeGenPipelineBuilderCallback;
+
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ CodeGenIREarlyEPCallbacks;
+ SmallVector<std::function<void(FunctionPassManager &)>, 2>
+ GCLoweringEPCallbacks;
+ SmallVector<std::function<void(ModulePassManager &)>, 2>
+ ISelPrepareEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineSSAOptimizationEarlyEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineSSAOptimizationLastEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreRegAllocEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostRegAllocEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreRegBankSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreGlobalInstructionSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostGlobalInstructionSelectEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ ILPOptsEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MachineLateOptimizationEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreSched2EPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostRewriteEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PreEmitEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ PostBBSectionsEPCallbacks;
+ SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+ MIEmitEPCallbacks;
+
+ std::function<void(ModulePassManager &)> AddCodeGenPreparePassesCallback;
+ std::function<void(MachineFunctionPassManager &)> AddInstSelectorCallback;
+ std::function<Error(MachineFunctionPassManager &)> AddRegAllocFastCallback;
+ std::function<Error(MachineFunctionPassManager &)>
+ AddRegAllocOptimizedCallback;
+ StringSet<> DisabledPasses;
+ StringMap<MachineFunctionPassManager> RegAllocPasses;
+ // TODO: Add methods in LLVMTargetMachine so we can get rid of it.
+
SmallVector<std::function<void(ModuleAnalysisManager &)>, 2>
ModuleAnalysisRegistrationCallbacks;
SmallVector<std::function<bool(StringRef, ModulePassManager &,
diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h
index 29bdb9c1746d3c..d51fe1e0e82ac4 100644
--- a/llvm/include/llvm/Target/CGPassBuilderOption.h
+++ b/llvm/include/llvm/Target/CGPassBuilderOption.h
@@ -42,6 +42,9 @@ struct CGPassBuilderOption {
bool PrintLSR = false;
bool DisableMergeICmps = false;
bool DisablePartialLibcallInlining = false;
+ bool DisableReplaceWithVecLib = false;
+ bool DisableLayoutFSProfileLoader = false;
+ bool DisablePrologEpilogInserterPass = false;
bool DisableConstantHoisting = false;
bool DisableSelectOptimize = true;
bool DisableAtExitBasedGlobalDtorLowering = false;
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 6d540808d4cc98..83fe22b00ada8a 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -105,6 +105,18 @@ bool MachineFunctionAnalysisManagerFunctionProxy::Result::invalidate(
return false;
}
+void FunctionToMachineFunctionPassAdaptor::eraseIf(
+ function_ref<bool(StringRef)> Pred) {
+ StringRef PassName = Pass->name();
+ if (PassName.contains("PassManager") || PassName.ends_with("PassAdaptor")) {
+ Pass->eraseIf(Pred);
+ if (Pass->isEmpty())
+ Pass.reset();
+ } else if (Pred(PassName)) {
+ Pass.reset();
+ }
+}
+
PreservedAnalyses
FunctionToMachineFunctionPassAdaptor::run(Function &F,
FunctionAnalysisManager &FAM) {
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 249407ff1c921c..cc8aff1a712d2b 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -188,11 +188,11 @@ static cl::opt<bool> DisableLayoutFSProfileLoader(
"disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
cl::desc("Disable MIRProfileLoader before BlockPlacement"));
// Specify FSProfile file name.
-static cl::opt<std::string>
+cl::opt<std::string>
FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile file name."), cl::Hidden);
// Specify Remapping file for FSProfile.
-static cl::opt<std::string> FSRemappingFile(
+cl::opt<std::string> FSRemappingFile(
"fs-remapping-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
@@ -500,6 +500,8 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
SET_BOOLEAN_OPTION(DisableMergeICmps)
SET_BOOLEAN_OPTION(DisableLSR)
SET_BOOLEAN_OPTION(DisableConstantHoisting)
+ SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+ SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
SET_BOOLEAN_OPTION(DisableCGP)
SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
SET_BOOLEAN_OPTION(DisableSelectOptimize)
@@ -542,8 +544,7 @@ void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
});
}
-Expected<TargetPassConfig::StartStopInfo>
-TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
+Expected<TargetPassConfig::StartStopInfo> TargetPassConfig::getStartStopInfo() {
auto [StartBefore, StartBeforeInstanceNum] =
getPassNameAndInstanceNum(StartBeforeOpt);
auto [StartAfter, StartAfterInstanceNum] =
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 6425f4934b2103..f9011cc8235084 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMPasses
OptimizationLevel.cpp
PassBuilder.cpp
PassBuilderBindings.cpp
+ PassBuilderCodeGen.cpp
PassBuilderPipelines.cpp
PassPlugin.cpp
StandardInstrumentations.cpp
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index a181a28f502f59..d3c2a755121c80 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -446,9 +446,24 @@ class RequireAllMachineFunctionPropertiesPass
PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
std::optional<PGOOptions> PGOOpt,
PassInstrumentationCallbacks *PIC)
- : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
- if (TM)
+ : TM(TM), PTO(PTO), PGOOpt(PGOOpt), CGPBO(getCGPassBuilderOption()),
+ PIC(PIC) {
+ if (TM) {
+ if (TM->Options.EnableIPRA)
+ CGPBO.RequiresCodeGenSCCOrder = true;
+ AddCodeGenPreparePassesCallback = [&](ModulePassManager &MPM) {
+ addDefaultCodeGenPreparePasses(MPM);
+ };
+ AddRegAllocFastCallback = [&](MachineFunctionPassManager &MFPM) {
+ return addDefaultRegAllocFastPasses(MFPM);
+ };
+ AddRegAllocOptimizedCallback = [&](MachineFunctionPassManager &MFPM) {
+ return addDefaultRegAllocOptimizedPasses(MFPM);
+ };
+
TM->registerPassBuilderCallbacks(*this);
+ }
+
if (PIC) {
PIC->registerClassToPassNameCallback([this, PIC]() {
// MSVC requires this to be captured if it's used inside decltype.
diff --git a/llvm/lib/Passes/PassBuilderCodeGen.cpp b/llvm/lib/Passes/PassBuilderCodeGen.cpp
new file mode 100644
index 00000000000000..d6bdad224d3546
--- /dev/null
+++ b/llvm/lib/Passes/PassBuilderCodeGen.cpp
@@ -0,0 +1,950 @@
+//===- Construction of code generation pass pipelines ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides the implementation of the PassBuilder based on our
+/// static pass registry as well as related functionality.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
+#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandLargeFpConvert.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/FinalizeISel.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LocalStackSlotAllocation.h"
+#include "llvm/CodeGen/LowerEmuTLS.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
+#include "llvm/CodeGen/SafeStack.h"
+#include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
+#include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/CodeGen/WasmEHPrepare.h"
+#include "llvm/CodeGen/WinEHPrepare.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only!
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+
+namespace llvm {
+extern cl::opt<std::string> FSRemappingFile;
+}
+
+using namespace llvm;
+
+void PassBuilder::invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : CodeGenIREarlyEPCallbacks)
+ C(MPM);
+}
+
+void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) {
+ for (auto &C : GCLoweringEPCallbacks)
+ C(FPM);
+}
+
+void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) {
+ for (auto &C : ISelPrepareEPCallbacks)
+ C(MPM);
+}
+
+void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineSSAOptimizationEarlyEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineSSAOptimizationLastEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreRegAllocEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreRegAllocEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreRegBankSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreRegBankSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreGlobalInstructionSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostGlobalInstructionSelectEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : ILPOptsEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMachineLateOptimizationEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : MachineLateOptimizationEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : MIEmitEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreEmitEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostRegAllocEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostRegAllocEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM) {
+ for (auto &C : PreSched2EPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::invokePostBBSectionsEPCallbacks(
+ MachineFunctionPassManager &MFPM) {
+ for (auto &C : PostBBSectionsEPCallbacks)
+ C(MFPM);
+}
+
+void PassBuilder::addDefaultCodeGenPreparePasses(ModulePassManager &MPM) {
+ FunctionPassManager FPM;
+ // CodeGen prepare
+ if (TM->getOptLevel() != CodeGenOptLevel::None && !CGPBO.DisableCGP)
+ FPM.addPass(CodeGenPreparePass(TM));
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+}
+
+Error PassBuilder::addDefaultRegAllocFastPasses(
+ MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(PHIEliminationPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+ if (auto Err = addRegAllocPass(MFPM))
+ return Err;
+ return Error::success();
+}
+
+Error PassBuilder::addDefaultRegAllocOptimizedPasses(
+ MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(DetectDeadLanesPass());
+ MFPM.addPass(InitUndefPass());
+ MFPM.addPass(ProcessImplicitDefsPass());
+
+ // LiveVariables currently requires pure SSA form.
+ //
+ // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+ // LiveVariables can be removed completely, and LiveIntervals can be
+ // directly computed. (We still either need to regenerate kill flags after
+ // regalloc, or preferably fix the scavenger to not depend on them).
+ // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+ // When LiveVariables is removed this has to be removed/moved either.
+ // Explicit addition of UnreachableMachineBlockElim allows stopping before
+ // or after it with -stop-before/-stop-after.
+ MFPM.addPass(UnreachableMachineBlockElimPass());
+ // FIXME: Some X86 tests failed because of incomplete pipeline.
+ // MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis,
+ // MachineFunction>());
+
+ // Edge splitting is smarter with machine loop info.
+ MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>());
+ MFPM.addPass(PHIEliminationPass());
+
+ if (CGPBO.EarlyLiveIntervals)
+ MFPM.addPass(RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
+
+ MFPM.addPass(TwoAddressInstructionPass());
+ MFPM.addPass(RegisterCoalescerPass());
+
+ // The machine scheduler may accidentally create disconnected components
+ // when moving subregister definitions around, avoid this by splitting them
+ // to separate vregs before. Splitting can also improve reg. allocation
+ // quality.
+ MFPM.addPass(RenameIndependentSubregsPass());
+
+ // PreRA instruction scheduling.
+ MFPM.addPass(MachineSchedulerPass());
+
+ if (auto Err = addRegAllocPass(MFPM))
+ return Err;
+ // Finally rewrite virtual registers.
+ MFPM.addPass(VirtRegRewriterPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ MFPM.addPass(RegAllocScoringPass());
+
+ return Error::success();
+}
+
+bool PassBuilder::isOptimizedRegAlloc() const {
+ return CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() !=
+ CodeGenOptLevel::None);
+}
+
+// Find the Profile remapping file name. The internal option takes the
+// precedence before getting from TargetMachine.
+static std::string getFSRemappingFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSRemappingFile.empty())
+ return Options.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+}
+
+// Find the FSProfile file name. The internal option takes the precedence
+// before getting from TargetMachine.
+static std::string getFSProfileFile(const TargetMachine *TM,
+ const CGPassBuilderOption &Options) {
+ if (!Options.FSProfileFile.empty())
+ return Options.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+}
+
+Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) {
+ const MCAsmInfo *MCAI = TM->getMCAsmInfo();
+ if (!MCAI)
+ return make_error<StringError>("No MCAsmInfo!", inconvertibleErrorCode());
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ FPM.addPass(SjLjEHPreparePass(TM));
+ [[fallthrough]];
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ case ExceptionHandling::AIX:
+ case ExceptionHandling::ZOS:
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ FPM.addPass(WinEHPreparePass());
+ FPM.addPass(DwarfEHPreparePass(TM));
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true));
+ FPM.addPass(WasmEHPreparePass());
+ break;
+ case ExceptionHandling::None:
+ FPM.addPass(LowerInvokePass());
+
+ // The lower invoke pass may create unreachable code. Remove it.
+ FPM.addPass(UnreachableBlockElimPass());
+ break;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addInstructionSelectorPasses(
+ MachineFunctionPassManager &MFPM) {
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ // Core ISel
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true));
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ CGPBO.EnableFastISelOption.value_or(false);
+ if (CGPBO.EnableFastISelOption.value_or(false))
+ Selector = SelectorType::FastISel;
+
+ else if (CGPBO.EnableGlobalISelOption.value_or(false) ||
+ (TM->Options.EnableGlobalISel &&
+ !CGPBO.EnableGlobalISelOption.value_or(false)))
+ Selector = SelectorType::GlobalISel;
+ else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM->setFastISel(true);
+ TM->setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM->setFastISel(false);
+ TM->setGlobalISel(true);
+ }
+
+ // Add instruction selector passes.
+ if (Selector == SelectorType::GlobalISel) {
+ MFPM.addPass(IRTranslatorPass());
+ MFPM.addPass(LegalizerPass());
+
+ // Before running the register bank selector, ask the target if it
+ // wants to run some passes.
+ invokePreRegBankSelectEPCallbacks(MFPM);
+ MFPM.addPass(RegBankSelectPass());
+
+ invokePreGlobalInstructionSelectEPCallbacks(MFPM);
+ MFPM.addPass(InstructionSelectPass());
+ invokePostGlobalInstructionSelectEPCallbacks(MFPM);
+
+ // Pass to reset the MachineFunction if the ISel failed.
+ MFPM.addPass(ResetMachineFunctionPass(
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag,
+ TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable));
+
+ // Provide a fallback path when we do not want to abort on
+ // not-yet-supported input.
+ if (TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM);
+ }
+ } else {
+ if (!AddInstSelectorCallback)
+ return make_error<StringError>("No InstSelectorCallback!",
+ inconvertibleErrorCode());
+ AddInstSelectorCallback(MFPM);
+ }
+ return Error::success();
+}
+
+void PassBuilder::addMachineSSAOptimizationPasses(
+ MachineFunctionPassManager &MFPM) {
+ // Pre-ra tail duplication.
+ MFPM.addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ MFPM.addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a different pass
+ // which merges spill slots.
+ MFPM.addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ MFPM.addPass(LocalStackSlotAllocationPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ MFPM.addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level parallelism,
+ // like if-conversion. Such passes will typically need dominator trees and
+ // loop info, just like LICM and CSE below.
+ invokeILPOptsEPCallbacks(MFPM);
+
+ MFPM.addPass(EarlyMachineLICMPass());
+ MFPM.addPass(MachineCSEPass());
+ MFPM.addPass(MachineSinkingPass());
+ MFPM.addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ MFPM.addPass(DeadMachineInstructionElimPass());
+}
+
+static Error setStartStop(ModulePassManager &MPM,
+ PassInstrumentationCallbacks *PIC) {
+ auto StartStopInfoOrErr = TargetPassConfig::getStartStopInfo();
+ if (!StartStopInfoOrErr)
+ return StartStopInfoOrErr.takeError();
+ auto &SSI = *StartStopInfoOrErr;
+
+ if (SSI.StartPass.empty() && SSI.StopPass.empty())
+ return Error::success();
+ if (!PIC) {
+ return make_error<StringError>("Need PassInstrumentationCallbacks!",
+ inconvertibleErrorCode());
+ }
+
+ static const std::vector<StringRef> SpecialPasses = {
+ "InvalidateAnalysisPass", "MachineVerifierPass", "PrintMIRPass",
+ "PrintMIRPreparePass", "RequireAnalysisPass", "VerifierPass"};
+
+ bool Started = SSI.StartPass.empty(), Stopped = false;
+ // Return true if pass is skipped.
+ auto Filter = [&, StartInstanceNum = 0u,
+ StopInstanceNum = 0u](StringRef Name) mutable {
+ if (isSpecialPass(Name, SpecialPasses))
+ return false;
+
+ bool ShouldDrop = true;
+ StringRef CurPassName = PIC->getPassNameForClassName(Name);
+
+ // Set instance counters correctly.
+ if (!SSI.StartPass.empty() && CurPassName == SSI.StartPass) {
+ ++StartInstanceNum;
+ if (StartInstanceNum == SSI.StartInstanceNum)
+ Started = true;
+ }
+ if (!SSI.StopPass.empty() && CurPassName == SSI.StopPass) {
+ ++StopInstanceNum;
+ if (StopInstanceNum == SSI.StopInstanceNum)
+ Stopped = true;
+ }
+
+ // Border case.
+ const bool AtStartBorder = !SSI.StartPass.empty() && Started &&
+ CurPassName == SSI.StartPass &&
+ StartInstanceNum == SSI.StartInstanceNum;
+ const bool AtStopBorder = !SSI.StopPass.empty() && Stopped &&
+ CurPassName == SSI.StopPass &&
+ StopInstanceNum == SSI.StopInstanceNum;
+ if (AtStartBorder)
+ ShouldDrop = SSI.StartAfter;
+ if (AtStopBorder)
+ ShouldDrop = !SSI.StopAfter;
+ if (!AtStartBorder && !AtStopBorder)
+ ShouldDrop = !Started || Stopped;
+
+ return ShouldDrop;
+ };
+
+ MPM.eraseIf(Filter);
+ if (!Started) {
+ return make_error<StringError>(
+ "Can't find start pass \"" + SSI.StartPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ if (!Stopped && !SSI.StopPass.empty()) {
+ return make_error<StringError>(
+ "Can't find stop pass \"" + SSI.StopPass + "\".",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addRegisterAllocatorPasses(
+ MachineFunctionPassManager &MFPM) {
+ return isOptimizedRegAlloc() ? AddRegAllocOptimizedCallback(MFPM)
+ : AddRegAllocFastCallback(MFPM);
+}
+
+Error PassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM,
+ StringRef FilterName) {
+ std::optional<RegAllocFilterFunc> FilterFunc =
+ parseRegAllocFilter(FilterName);
+ if (!FilterFunc) {
+ return make_error<StringError>(
+ formatv("Unknown register filter name: {0}", FilterName).str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+
+ if (RegAllocPasses.contains(FilterName)) {
+ MFPM.addPass(std::move(RegAllocPasses[FilterName]));
+ return Error::success();
+ }
+
+ // Add default register allocator.
+ if (isOptimizedRegAlloc()) {
+ MFPM.addPass(RAGreedyPass());
+ } else {
+ RegAllocFastPassOptions Opts;
+ Opts.Filter = *FilterFunc;
+ Opts.FilterName = FilterName;
+ MFPM.addPass(RegAllocFastPass(Opts));
+ }
+ return Error::success();
+}
+
+Error PassBuilder::addMachinePasses(ModulePassManager &MPM,
+ FunctionPassManager &FPM,
+ MachineFunctionPassManager &MFPM) {
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
+ // FinalizeISel.
+ MFPM.addPass(FinalizeISelPass());
+
+ // Add passes that optimize machine instructions in SSA form.
+ if (OptLevel != CodeGenOptLevel::None) {
+ invokeMachineSSAOptimizationEarlyEPCallbacks(MFPM);
+ addMachineSSAOptimizationPasses(MFPM);
+ invokeMachineSSAOptimizationLastEPCallbacks(MFPM);
+ } else {
+ MFPM.addPass(LocalStackSlotAllocationPass());
+ }
+
+ if (TM->Options.EnableIPRA)
+ MFPM.addPass(RegUsageInfoPropagationPass());
+
+ // Run pre-ra passes.
+ invokePreRegAllocEPCallbacks(MFPM);
+
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::Pass1, nullptr));
+ }
+
+ if (auto Err = addRegisterAllocatorPasses(MFPM))
+ return Err;
+
+ invokePostRegAllocEPCallbacks(MFPM);
+
+ MFPM.addPass(RemoveRedundantDebugValuesPass());
+ MFPM.addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (OptLevel != CodeGenOptLevel::None) {
+ MFPM.addPass(PostRAMachineSinkingPass());
+ MFPM.addPass(ShrinkWrapPass());
+ }
+
+ if (!CGPBO.DisablePrologEpilogInserterPass)
+ MFPM.addPass(PrologEpilogInserterPass());
+ /// Add passes that optimize machine instructions after register allocation.
+ if (OptLevel != CodeGenOptLevel::None)
+ invokeMachineLateOptimizationEPCallbacks(MFPM);
+
+ // Expand pseudo instructions before second scheduling pass.
+ MFPM.addPass(ExpandPostRAPseudosPass());
+
+ // Run pre-sched2 passes.
+ invokePreSched2EPCallbacks(MFPM);
+
+ if (CGPBO.EnableImplicitNullChecks)
+ MFPM.addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other
+ // point.
+ if (OptLevel != CodeGenOptLevel::None &&
+ !TM->targetSchedulesPostRAScheduling()) {
+ if (CGPBO.MISchedPostRA)
+ MFPM.addPass(PostMachineSchedulerPass());
+ else
+ MFPM.addPass(PostRASchedulerPass());
+ }
+
+ // GC, replacement for GCMachineCodeAnalysis
+ MFPM.addPass(GCMachineCodeInsertionPass());
+
+ // Basic block placement.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2));
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader)
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::Pass2, nullptr));
+ }
+ MFPM.addPass(MachineBlockPlacementPass());
+ // Run a separate pass to collect block placement statistics.
+ if (CGPBO.EnableBlockPlacementStats)
+ MFPM.addPass(MachineBlockPlacementStatsPass());
+ }
+
+ // Insert before XRay Instrumentation.
+ MFPM.addPass(FEntryInserterPass());
+ MFPM.addPass(XRayInstrumentationPass());
+ MFPM.addPass(PatchableFunctionPass());
+
+ invokePreEmitEPCallbacks(MFPM);
+
+ if (TM->Options.EnableIPRA)
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ MFPM.addPass(RegUsageInfoCollectorPass());
+
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
+ MFPM.addPass(FuncletLayoutPass());
+
+ MFPM.addPass(StackMapLivenessPass());
+ MFPM.addPass(LiveDebugValuesPass());
+ MFPM.addPass(MachineSanitizerBinaryMetadata());
+
+ if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None &&
+ CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
+ if (AddOutliner) {
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(MachineOutlinerPass(RunOnAllFunctions));
+ FPM = FunctionPassManager();
+ MFPM = MachineFunctionPassManager();
+ }
+ }
+
+ if (CGPBO.GCEmptyBlocks)
+ MFPM.addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ MFPM.addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ bool NeedsBBSections =
+ TM->getBBSectionsType() != llvm::BasicBlockSection::None;
+ // Machine function splitter uses the basic block sections feature. Both
+ // cannot be enabled at the same time. We do not apply machine function
+ // splitter if -basic-block-sections is requested.
+ if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter ||
+ CGPBO.EnableMachineFunctionSplitter)) {
+ const std::string ProfileFile = getFSProfileFile(TM, CGPBO);
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ MFPM.addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(TM, CGPBO),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.\n";
+ }
+ }
+ MFPM.addPass(MachineFunctionSplitterPass());
+ }
+
+ // We run the BasicBlockSections pass if either we need BB sections or BB
+ // address map (or both).
+ if (NeedsBBSections || TM->Options.BBAddrMap) {
+ if (TM->getBBSectionsType() == llvm::BasicBlockSection::List)
+ MFPM.addPass(BasicBlockPathCloningPass());
+ MFPM.addPass(BasicBlockSectionsPass());
+ }
+
+ invokePostBBSectionsEPCallbacks(MFPM);
+
+ if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup)
+ MFPM.addPass(CFIFixupPass());
+
+ MFPM.addPass(StackFrameLayoutAnalysisPass());
+
+ // Add passes that directly emit MI after all other MI passes.
+ invokeMIEmitEPCallbacks(MFPM);
+
+ return Error::success();
+}
+
+Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM,
+ raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType,
+ MCContext &Ctx) {
+ if (!TM)
+ return make_error<StringError>("Need a TargetMachine instance!",
+ inconvertibleErrorCode());
+
+ if (CustomCodeGenPipelineBuilderCallback) {
+ ModulePassManager MPM;
+ if (auto Err = CustomCodeGenPipelineBuilderCallback(MPM, Out, DwoOut,
+ FileType, Ctx))
+ return Err;
+ MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); });
+ if (auto Err = setStartStop(MPM, PIC))
+ return Err;
+ TopLevelMPM.addPass(std::move(MPM));
+ return Error::success();
+ }
+
+ CodeGenOptLevel OptLevel = TM->getOptLevel();
+ if (auto Err = parseRegAllocOption(CGPBO.RegAlloc))
+ return Err;
+
+ bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
+ bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
+
+ ModulePassManager MPM;
+ FunctionPassManager FPM;
+ MachineFunctionPassManager MFPM;
+
+ if (!CGPBO.DisableVerify)
+ MPM.addPass(VerifierPass());
+
+ // IR part
+ // TODO: Remove RequireAnalysisPass<MachineModuleAnalysis, Module>()
+ // when we port AsmPrinter.
+ MPM.addPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ MPM.addPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
+
+ invokeCodeGenIREarlyEPCallbacks(MPM);
+
+ if (TM->useEmulatedTLS())
+ MPM.addPass(LowerEmuTLSPass());
+ MPM.addPass(PreISelIntrinsicLoweringPass(TM));
+
+ // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
+ // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
+ if (TM->getTargetTriple().isOSBinFormatMachO() &&
+ !CGPBO.DisableAtExitBasedGlobalDtorLowering)
+ MPM.addPass(LowerGlobalDtorsPass());
+
+ FPM.addPass(ExpandLargeDivRemPass(TM));
+ FPM.addPass(ExpandLargeFpConvertPass(TM));
+
+ // Run loop strength reduction before anything else.
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableLSR) {
+ LoopPassManager LPM;
+ LPM.addPass(LoopStrengthReducePass());
+ FPM.addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
+ /*UseMemorySSA=*/true));
+ }
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
+ // loads and compares. ExpandMemCmpPass then tries to expand those calls
+ // into optimally-sized loads and compares. The transforms are enabled by a
+ // target lowering hook.
+ if (!CGPBO.DisableMergeICmps)
+ FPM.addPass(MergeICmpsPass());
+ FPM.addPass(ExpandMemCmpPass(TM));
+ }
+
+ // Run GC lowering passes for builtin collectors
+ FPM.addPass(GCLoweringPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(ShadowStackGCLoweringPass());
+ FPM = FunctionPassManager();
+ invokeGCLoweringEPCallbacks(FPM);
+
+ // Make sure that no unreachable blocks are instruction selected.
+ FPM.addPass(UnreachableBlockElimPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ if (!CGPBO.DisableConstantHoisting)
+ FPM.addPass(ConstantHoistingPass());
+ if (!CGPBO.DisableReplaceWithVecLib)
+ FPM.addPass(ReplaceWithVeclib());
+ if (!CGPBO.DisablePartialLibcallInlining)
+ FPM.addPass(PartiallyInlineLibCallsPass());
+ }
+
+ // Instrument function entry after all inlining.
+ FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ FPM.addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants to.
+ // Allow disabling it for testing purposes.
+ if (!CGPBO.DisableExpandReductions)
+ FPM.addPass(ExpandReductionsPass());
+
+ if (OptLevel != CodeGenOptLevel::None) {
+ FPM.addPass(TLSVariableHoistPass());
+
+ // Convert conditional moves to conditional jumps when profitable.
+ if (!CGPBO.DisableSelectOptimize)
+ FPM.addPass(SelectOptimizePass(TM));
+ }
+
+ {
+ ModulePassManager CGMPM;
+ AddCodeGenPreparePassesCallback(CGMPM);
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(std::move(CGMPM));
+ FPM = FunctionPassManager();
+ }
+
+ // Turn exception handling constructs into something the code generators can
+ // handle.
+ if (auto Err = addExceptionHandlingPasses(FPM))
+ return Err;
+
+ // All passes after this point need to handle cgscc.
+
+ { // Pre isel extension
+ ModulePassManager ISelPreparePasses;
+ invokeISelPrepareEPCallbacks(ISelPreparePasses);
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(std::move(ISelPreparePasses));
+ }
+
+ if (OptLevel != CodeGenOptLevel::None)
+ FPM.addPass(ObjCARCContractPass());
+ FPM.addPass(CallBrPreparePass());
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ FPM.addPass(SafeStackPass(TM));
+ FPM.addPass(StackProtectorPass(TM));
+
+ // All passes which modify the LLVM IR are now complete; run the verifier
+ // to ensure that the IR is valid.
+ if (!CGPBO.DisableVerify)
+ FPM.addPass(VerifierPass());
+
+ if (PrintMIR) {
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ MPM.addPass(PrintMIRPreparePass(Out));
+ FPM = FunctionPassManager();
+ }
+
+ if (auto Err = addInstructionSelectorPasses(MFPM))
+ return Err;
+
+ if (auto Err = addMachinePasses(MPM, FPM, MFPM))
+ return Err;
+
+ if (!CGPBO.DisableVerify)
+ MFPM.addPass(MachineVerifierPass());
+
+ if (PrintMIR)
+ MFPM.addPass(PrintMIRPass(Out));
+
+ // TODO: Add AsmPrinter.
+ (void)Ctx;
+
+ FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+ FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+ if (CGPBO.RequiresCodeGenSCCOrder)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM))));
+ else
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+ MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); });
+ if (auto Err = setStartStop(MPM, PIC))
+ return Err;
+ TopLevelMPM.addPass(std::move(MPM));
+ return Error::success();
+}
+
+Expected<ModulePassManager> PassBuilder::buildDefaultCodeGenPipeline(
+ raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+ MCContext &Ctx) {
+ ModulePassManager MPM;
+ Error Err = buildDefaultCodeGenPipeline(MPM, Out, DwoOut, FileType, Ctx);
+ if (Err)
+ return std::move(Err);
+ return std::move(MPM);
+}
+
+static bool isRegAllocPass(StringRef Name) {
+ // TODO: Add all register allocator names.
+ return Name.starts_with("regallocfast");
+}
+
+Error PassBuilder::parseRegAllocOption(StringRef Text) {
+ if (Text == "default")
+ return Error::success();
+
+ RegAllocPasses.clear();
+ while (!Text.empty()) {
+ StringRef SinglePass;
+ std::tie(SinglePass, Text) = Text.split(',');
+
+ if (!isRegAllocPass(SinglePass)) {
+ return make_error<StringError>(
+ formatv("{0} is not a register allocator!", SinglePass).str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ if (!isOptimizedRegAlloc() && !SinglePass.starts_with("regallocfast")) {
+ return make_error<StringError>(
+ "Must use fast (default) register allocator for unoptimized "
+ "regalloc.",
+ std::make_error_code(std::errc::invalid_argument));
+ }
+
+ MachineFunctionPassManager MFPM;
+ if (auto Err = parsePassPipeline(MFPM, SinglePass))
+ return Err;
+
+ auto FilterPos = SinglePass.find("filter=");
+ if (FilterPos == std::string::npos) {
+ bool Success = RegAllocPasses.try_emplace("all", std::move(MFPM)).second;
+ if (!Success) {
+ return make_error<StringError>(
+ formatv("Already set register allocator '{0}' for all registers!",
+ SinglePass)
+ .str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ continue;
+ }
+
+ StringRef FilterName = SinglePass.drop_front(FilterPos);
+ FilterName.consume_front("filter=");
+ FilterName =
+ FilterName.take_until([](char C) { return C == ';' || C == '>'; });
+ bool Success =
+ RegAllocPasses.try_emplace(FilterName, std::move(MFPM)).second;
+ if (!Success) {
+ return make_error<StringError>(
+ formatv("Already set register allocator '{0}' for filter {1}!",
+ SinglePass, FilterName)
+ .str(),
+ std::make_error_code(std::errc::invalid_argument));
+ }
+ }
+ return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index da18f2b20f1427..427d13ba00c6ca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -58,9 +58,11 @@
#include "llvm/CodeGen/MIRParser/MIParser.h"
#include "llvm/CodeGen/MachineCSE.h"
#include "llvm/CodeGen/MachineLICM.h"
+#include "llvm/CodeGen/PHIElimination.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TwoAddressInstructionPass.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
@@ -739,6 +741,8 @@ parseAMDGPUAttributorPassOptions(StringRef Params) {
}
void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
+ CGPassBuilderOption &CGPTO = PB.getCGPBO();
+ CGPTO.RequiresCodeGenSCCOrder = true;
#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
#include "llvm/Passes/TargetPassRegistry.inc"
@@ -849,6 +853,31 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
return onlyAllocateVGPRs;
return nullptr;
});
+
+ // CodeGen pass builder part
+ PB.registerISelPrepareEPCallback([](ModulePassManager &MPM) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ RequireAnalysisPass<UniformityInfoAnalysis, Function>()));
+ });
+
+ PB.setAddInstSelectorCallback([&](MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(AMDGPUISelDAGToDAGPass(*this));
+ MFPM.addPass(SIFixSGPRCopiesPass());
+ MFPM.addPass(SILowerI1CopiesPass());
+ });
+
+ PB.setRegAllocFastCallback([&](MachineFunctionPassManager &MFPM) -> Error {
+ // TODO: Add complete pipeline.
+ MFPM.addPass(PHIEliminationPass());
+ // MFPM.addPass(SILowerControlFlowPass());
+ MFPM.addPass(TwoAddressInstructionPass());
+ // MFPM.addPass(SIWholeQuadModePass());
+ if (auto Err = PB.addRegAllocPass(MFPM, "sgpr"))
+ return Err;
+ if (auto Err = PB.addRegAllocPass(MFPM, "vgpr"))
+ return Err;
+ return Error::success();
+ });
}
int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) {
diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
index d979517e12af6d..f911b45b76fccd 100644
--- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
@@ -53,6 +53,10 @@ Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
#define GET_PASS_REGISTRY "X86PassRegistry.def"
#include "llvm/Passes/TargetPassRegistry.inc"
+
+ PB.setAddInstSelectorCallback([this](MachineFunctionPassManager &MFPM) {
+ MFPM.addPass(X86ISelDAGToDAGPass(*this));
+ });
}
Error X86TargetMachine::buildCodeGenPipeline(
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 081303c2dda4c9..442ffce04f3c8a 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -64,8 +64,10 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
LPMUpdater &>::eraseIf(function_ref<bool(StringRef)> Pred) {
- std::vector<char> IsLoopNestPassVec(
- static_cast<size_t>(IsLoopNestPass.size()));
+ assert(LoopPasses.size() + LoopNestPasses.size() == IsLoopNestPass.size() &&
+ "Wrong precondition!");
+
+ std::vector<char> IsLoopNestPassVec(IsLoopNestPass.size());
for (unsigned Idx = 0, Sz = IsLoopNestPass.size(); Idx != Sz; ++Idx)
IsLoopNestPassVec[Idx] = IsLoopNestPass[Idx];
@@ -93,6 +95,9 @@ void PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
IsLoopNestPass.clear();
for (const auto I : IsLoopNestPassVec)
IsLoopNestPass.push_back(I);
+
+ assert(LoopPasses.size() + LoopNestPasses.size() == IsLoopNestPass.size() &&
+ "Wrong postcondition!");
}
// Run both loop passes and loop-nest passes on top-level loop \p L.
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-select.ll b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll
new file mode 100644
index 00000000000000..7327e1bf95e1e4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll
@@ -0,0 +1,9 @@
+; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=DEFAULT
+; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -regalloc-npm='regallocfast<filter=sgpr;no-clear-vregs>' -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=CUSTOM
+
+; DEFAULT: regallocfast<filter=sgpr>
+; DEFAULT: regallocfast<filter=vgpr>
+
+; Just a proof of concept that we can modify parameters of register allocator.
+; CUSTOM: regallocfast<filter=sgpr;no-clear-vregs>
+; CUSTOM: regallocfast<filter=vgpr>
diff --git a/llvm/test/tools/llc/new-pm/start-stop.ll b/llvm/test/tools/llc/new-pm/start-stop.ll
index 9c3b9f009178fe..8949dd35f67fe4 100644
--- a/llvm/test/tools/llc/new-pm/start-stop.ll
+++ b/llvm/test/tools/llc/new-pm/start-stop.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -filetype=null %s | FileCheck --match-full-lines %s --check-prefix=NULL
; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -o /dev/null %s | FileCheck --match-full-lines %s --check-prefix=OBJ
-; NULL: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument<post-inline>,verify)
-; OBJ: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument<post-inline>,verify),PrintMIRPreparePass,function(machine-function(print),invalidate<machine-function-info>)
+; NULL: verify,require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(mergeicmps,expand-memcmp,gc-lowering),function(verify,machine-function(require<machine-loops>,verify),invalidate<machine-function-info>)
+; OBJ: verify,require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,function(mergeicmps,expand-memcmp,gc-lowering),function(verify),PrintMIRPreparePass,function(machine-function(require<machine-loops>,verify,print),invalidate<machine-function-info>)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 3892fbb8c74f78..f40aac39c76b2a 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -120,6 +120,9 @@ int llvm::compileModuleWithNewPM(
CGSCCAnalysisManager CGAM;
ModuleAnalysisManager MAM;
PassBuilder PB(Target.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+ CGPassBuilderOption &CGPBO = PB.getCGPBO();
+ CGPBO.DisableVerify = VK != VerifierKind::InputOutput;
+ CGPBO.RegAlloc = RegAlloc;
PB.registerModuleAnalyses(MAM);
PB.registerCGSCCAnalyses(CGAM);
PB.registerFunctionAnalyses(FAM);
@@ -156,8 +159,9 @@ int llvm::compileModuleWithNewPM(
if (MIR->parseMachineFunctions(*M, MAM))
return 1;
} else {
- ExitOnErr(Target->buildCodeGenPipeline(
- MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC));
+ ExitOnErr(PB.buildDefaultCodeGenPipeline(MPM, *OS,
+ DwoOut ? &DwoOut->os() : nullptr,
+ FileType, MMI.getContext()));
}
if (PrintPipelinePasses) {
>From eeaf7f17960431cf54f80909bb563cb675db3807 Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Mon, 23 Sep 2024 16:18:29 +0800
Subject: [PATCH 3/5] remove else
---
llvm/include/llvm/IR/PassManagerInternal.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/include/llvm/IR/PassManagerInternal.h b/llvm/include/llvm/IR/PassManagerInternal.h
index 3a75c9c4e97faf..36caf7cd85a3d5 100644
--- a/llvm/include/llvm/IR/PassManagerInternal.h
+++ b/llvm/include/llvm/IR/PassManagerInternal.h
@@ -145,8 +145,7 @@ struct PassModel : PassConcept<IRUnitT, AnalysisManagerT, ExtraArgTs...> {
bool isEmpty() const override {
if constexpr (is_detected<has_is_empty_t, PassT>::value)
return Pass.isEmpty();
- else
- return false;
+ return false;
}
PassT Pass;
>From 6809c00e90186ee543cf4841686b139f37fd44cc Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Tue, 1 Oct 2024 20:13:20 +0800
Subject: [PATCH 4/5] avoid (*I)
---
llvm/include/llvm/IR/PassManager.h | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index 4f28b4aceb3e66..e06643bdb2b738 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -222,11 +222,12 @@ class PassManager : public PassInfoMixin<
/// For internal use only!
void eraseIf(function_ref<bool(StringRef)> Pred) {
for (auto I = Passes.begin(); I != Passes.end();) {
- (*I)->eraseIf(Pred);
- bool IsSpecial = (*I)->name().ends_with("PassAdaptor") ||
- (*I)->name().contains("PassManager");
- bool PredResult = Pred((*I)->name());
- if ((!IsSpecial && PredResult) || (IsSpecial && (*I)->isEmpty()))
+ auto &P = *I;
+ P->eraseIf(Pred);
+ bool IsSpecial = P->name().ends_with("PassAdaptor") ||
+ P->name().contains("PassManager");
+ bool PredResult = Pred(P->name());
+ if ((!IsSpecial && PredResult) || (IsSpecial && P->isEmpty()))
I = Passes.erase(I);
else
++I;
>From 918e37853c17b3a84c8baf287284acb95fb77d7e Mon Sep 17 00:00:00 2001
From: PaperChalice <liujunchang97 at outlook.com>
Date: Sun, 17 Nov 2024 14:12:42 +0800
Subject: [PATCH 5/5] Use `GlobalMergeFuncPass`
---
llvm/lib/Passes/PassBuilderCodeGen.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Passes/PassBuilderCodeGen.cpp b/llvm/lib/Passes/PassBuilderCodeGen.cpp
index d6bdad224d3546..6fef4e3f99b9a4 100644
--- a/llvm/lib/Passes/PassBuilderCodeGen.cpp
+++ b/llvm/lib/Passes/PassBuilderCodeGen.cpp
@@ -21,6 +21,7 @@
#include "llvm/CodeGen/ExpandReductions.h"
#include "llvm/CodeGen/FinalizeISel.h"
#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/GlobalMergeFunctions.h"
#include "llvm/CodeGen/LiveVariables.h"
#include "llvm/CodeGen/LocalStackSlotAllocation.h"
#include "llvm/CodeGen/LowerEmuTLS.h"
@@ -53,7 +54,6 @@
#include "llvm/Transforms/Scalar/MergeICmps.h"
#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
-#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
#include "llvm/Transforms/Utils/LowerInvoke.h"
@@ -797,8 +797,6 @@ Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM,
FPM.addPass(ExpandReductionsPass());
if (OptLevel != CodeGenOptLevel::None) {
- FPM.addPass(TLSVariableHoistPass());
-
// Convert conditional moves to conditional jumps when profitable.
if (!CGPBO.DisableSelectOptimize)
FPM.addPass(SelectOptimizePass(TM));
@@ -806,6 +804,8 @@ Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM,
{
ModulePassManager CGMPM;
+ if (CGPBO.EnableGlobalMergeFunc)
+ CGMPM.addPass(GlobalMergeFuncPass());
AddCodeGenPreparePassesCallback(CGMPM);
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
MPM.addPass(std::move(CGMPM));
More information about the llvm-commits
mailing list