[llvm] [NewPM][CodeGen] add TargetPassConfig like API (PR #70906)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 19 03:54:40 PST 2023
https://github.com/paperchalice updated https://github.com/llvm/llvm-project/pull/70906
>From 286e4bc366233f2aab7bc5d35f7f0a19ecb6144c Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97 at outlook.com>
Date: Wed, 1 Nov 2023 10:41:58 +0800
Subject: [PATCH] [NewPM][CodeGen] add TargetPassConfig like API
---
.../include/llvm/CodeGen/CodeGenPassBuilder.h | 372 ++++++++++++++----
.../include/llvm/CodeGen/MachinePassManager.h | 11 +-
llvm/lib/CodeGen/MachinePassManager.cpp | 64 ++-
3 files changed, 341 insertions(+), 106 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index 32d6e5f91f7b09..4459d6a7689cc4 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -19,14 +19,18 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/CallBrPrepare.h"
#include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandLargeDivRem.h"
+#include "llvm/CodeGen/ExpandLargeFpConvert.h"
#include "llvm/CodeGen/ExpandMemCmp.h"
#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/IndirectBrExpand.h"
#include "llvm/CodeGen/InterleavedAccess.h"
@@ -51,6 +55,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/WithColor.h"
#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/CFGuard.h"
@@ -61,7 +66,10 @@
#include "llvm/Transforms/Scalar/MergeICmps.h"
#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
+#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
#include "llvm/Transforms/Utils/LowerInvoke.h"
#include <cassert>
#include <type_traits>
@@ -142,6 +150,36 @@ template <typename DerivedT> class CodeGenPassBuilder {
raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
CodeGenFileType FileType) const;
+ /// Parse single target-specific MIR pass
+ /// @param Name the pass name
+ /// @return true if failed
+ bool parseTargetMIRPass(MachineFunctionPassManager &MFPM,
+ StringRef Name) const {
+ llvm_unreachable("parseTargetMIRPass is not overridden");
+ }
+
+ /// Parse single non-target-specific MIR pass
+ /// @param Name the pass name
+ /// @return true if failed
+ bool parseMIRPass(MachineFunctionPassManager &MFPM, StringRef Name) const;
+
+ /// Parse MIR pass pipeline. Unlike IR pass pipeline,
+ /// there is only one pass manager for machine function
+ /// so there is no need to specify the pass nesting.
+ /// @param Text a comma separated pass name list
+ Error parseMIRPipeline(MachineFunctionPassManager &MFPM,
+ StringRef Text) const {
+ for (auto [LHS, RHS] = Text.split(','); LHS != "";
+ std::tie(LHS, RHS) = RHS.split(',')) {
+ if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, LHS)) {
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ Twine('\"') + Twine(LHS) + Twine("\" pass could not be found."));
+ }
+ }
+ return Error::success();
+ }
+
void registerModuleAnalyses(ModuleAnalysisManager &) const;
void registerFunctionAnalyses(FunctionAnalysisManager &) const;
void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const;
@@ -154,12 +192,18 @@ template <typename DerivedT> class CodeGenPassBuilder {
}
PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
- return PIC;
+ static PassInstrumentationCallbacks PseudoPIC;
+ return PIC ? PIC : &PseudoPIC;
}
-protected:
- template <typename PassT> using has_key_t = decltype(PassT::Key);
+ /// Allow the target to disable a specific standard pass by default.
+ template <typename PassT> void disablePass() {
+ DisabledPasses.insert(PassT::ID());
+ getPassInstrumentationCallbacks()->registerShouldRunOptionalPassCallback(
+ [](StringRef P, Any IR) { return P != PassT::name(); });
+ }
+protected:
template <typename PassT>
using is_module_pass_t = decltype(std::declval<PassT &>().run(
std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
@@ -218,38 +262,55 @@ template <typename DerivedT> class CodeGenPassBuilder {
AddMachinePass(MachineFunctionPassManager &PM) : PM(PM) {}
template <typename PassT> void operator()(PassT &&Pass) {
- static_assert(
- is_detected<has_key_t, PassT>::value,
- "Machine function pass must define a static member variable `Key`.");
for (auto &C : BeforeCallbacks)
- if (!C(&PassT::Key))
+ if (!C(PassT::ID()))
return;
PM.addPass(std::forward<PassT>(Pass));
for (auto &C : AfterCallbacks)
- C(&PassT::Key);
- }
-
- template <typename PassT> void insertPass(AnalysisKey *ID, PassT Pass) {
- AfterCallbacks.emplace_back(
- [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) {
- if (PassID == ID)
- this->PM.addPass(std::move(Pass));
- });
+ C(PassT::ID(), PassT::name());
}
- void disablePass(AnalysisKey *ID) {
- BeforeCallbacks.emplace_back(
- [ID](AnalysisKey *PassID) { return PassID != ID; });
+ template <typename PassT> void insertPass(MachinePassKey *ID, PassT Pass) {
+ AfterCallbacks.emplace_back([this, ID, Pass = std::move(Pass)](
+ MachinePassKey *PassID, StringRef) {
+ if (PassID == ID)
+ this->PM.addPass(std::move(Pass));
+ });
}
MachineFunctionPassManager releasePM() { return std::move(PM); }
private:
MachineFunctionPassManager &PM;
- SmallVector<llvm::unique_function<bool(AnalysisKey *)>, 4> BeforeCallbacks;
- SmallVector<llvm::unique_function<void(AnalysisKey *)>, 4> AfterCallbacks;
+ SmallVector<llvm::unique_function<bool(MachinePassKey *)>, 4>
+ BeforeCallbacks;
+ SmallVector<llvm::unique_function<void(MachinePassKey *, StringRef)>, 4>
+ AfterCallbacks;
};
+ // Find the FSProfile file name. The internal option takes the precedence
+ // before getting from TargetMachine.
+ std::string getFSProfileFile() const {
+ if (!Opt.FSProfileFile.empty())
+ return Opt.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM.getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+ }
+
+ // Find the Profile remapping file name. The internal option takes the
+ // precedence before getting from TargetMachine.
+ std::string getFSRemappingFile() const {
+ if (!Opt.FSRemappingFile.empty())
+ return Opt.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM.getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+ }
+
+ DenseSet<MachinePassKey *> DisabledPasses;
LLVMTargetMachine &TM;
CGPassBuilderOption Opt;
PassInstrumentationCallbacks *PIC;
@@ -258,9 +319,6 @@ template <typename DerivedT> class CodeGenPassBuilder {
void registerTargetAnalysis(ModuleAnalysisManager &) const {}
void registerTargetAnalysis(FunctionAnalysisManager &) const {}
void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {}
- std::pair<StringRef, bool> getTargetPassNameFromLegacyName(StringRef) const {
- return {"", false};
- }
template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); }
@@ -326,14 +384,16 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// This pass may be implemented by targets that want to run passes
/// immediately before machine code is emitted.
- void addPreEmitPass(AddMachinePass &) const {}
+ void addPreEmitPass(AddMachinePass &addPass) const {}
+
+ /// This pass may be implemented by targets that want to run passes
+ /// immediately after basic block sections are assigned.
+ void addPostBBSections(AddMachinePass &) const {}
/// Targets may add passes immediately before machine code is emitted in this
/// callback. This is called even later than `addPreEmitPass`.
- // FIXME: Rename `addPreEmitPass` to something more sensible given its actual
- // position and remove the `2` suffix here as this callback is what
- // `addPreEmitPass` *should* be but in reality isn't.
- void addPreEmitPass2(AddMachinePass &) const {}
+ /// This function replaces `addPreEmitPass2` in TargetConfig.
+ void addPrecedingEmitPass(AddMachinePass &) const {}
/// {{@ For GlobalISel
///
@@ -393,7 +453,7 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// representation to the MI representation.
/// Adds IR based lowering and target specific optimization passes and finally
/// the core instruction selection passes.
- void addISelPasses(AddIRPass &) const;
+ Error addISelPasses(AddIRPass &, AddMachinePass &) const;
/// Add the actual instruction selection passes. This does not include
/// preparation passes on IR.
@@ -433,6 +493,14 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// are required for fast register allocation.
Error addFastRegAlloc(AddMachinePass &) const;
+ /// addPostFastRegAllocRewrite - Add passes to the optimized register
+ /// allocation pipeline after fast register allocation is complete.
+ Error addPostFastRegAllocRewrite(AddMachinePass &) const {
+ return make_error<StringError>(
+ "addPostFastRegAllocRewrite is not overridden",
+ inconvertibleErrorCode());
+ }
+
/// addOptimizedRegAlloc - Add passes related to register allocation.
/// LLVMTargetMachine provides standard regalloc passes for most targets.
void addOptimizedRegAlloc(AddMachinePass &) const;
@@ -440,14 +508,54 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// Add passes that optimize machine instructions after register allocation.
void addMachineLateOptimization(AddMachinePass &) const;
- /// addGCPasses - Add late codegen passes that analyze code for garbage
+ /// registerGCPasses - Add late codegen passes that analyze code for garbage
/// collection. This should return true if GC info should be printed after
/// these passes.
- void addGCPasses(AddMachinePass &) const {}
+ bool registerGCPasses(MachineFunctionAnalysisManager &MFAM) const {
+ MFAM.registerPass([] { return GCMachineCodeAnalysisPass(); });
+ return true;
+ }
/// Add standard basic block placement passes.
void addBlockPlacement(AddMachinePass &) const;
+ /// Add a pass to print the machine function if printing is enabled.
+ void addPrintPass(AddMachinePass &addPass, const std::string &Banner) const {
+ if (Opt.PrintAfterISel)
+ addPass(MachineFunctionPrinterPass(dbgs(), Banner));
+ }
+
+ /// Add a pass to perform basic verification of the machine function if
+ /// verification is enabled.
+ void addVerifyPass(MachineFunctionPassManager &MFPM,
+ const std::string &Banner) const {
+ bool Verify = Opt.VerifyMachineCode.value_or(false);
+#ifdef EXPENSIVE_CHECKS
+ if (!Opt.VerifyMachineCode)
+ Verify = TM->isMachineVerifierClean();
+#endif
+ if (Verify)
+ MFPM.addPass(MachineVerifierPass(Banner));
+ }
+
+ void addVerifyPass(AddMachinePass &addPass, const std::string &Banner) const {
+ bool Verify = Opt.VerifyMachineCode.value_or(false);
+#ifdef EXPENSIVE_CHECKS
+ if (!Opt.VerifyMachineCode)
+ Verify = TM->isMachineVerifierClean();
+#endif
+ if (Verify)
+ addPass(MachineVerifierPass(Banner));
+ }
+
+ /// printAndVerify - Add a pass to dump then verify the machine function, if
+ /// those steps are enabled.
+ void printAndVerify(AddMachinePass &addPass,
+ const std::string &Banner) const {
+ addPrintPass(addPass, Banner);
+ addVerifyPass(addPass, Banner);
+ }
+
using CreateMCStreamer =
std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
@@ -465,10 +573,10 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// regalloc pass.
void addRegAllocPass(AddMachinePass &, bool Optimized) const;
- /// Add core register alloator passes which do the actual register assignment
- /// and rewriting. \returns true if any passes were added.
- Error addRegAssignmentFast(AddMachinePass &) const;
- Error addRegAssignmentOptimized(AddMachinePass &) const;
+ /// Add core register allocator passes which do the actual register assignment
+ /// and rewriting. \returns Error::success() if any passes were added.
+ Error addRegAssignAndRewriteFast(AddMachinePass &addPass) const;
+ Error addRegAssignAndRewriteOptimized(AddMachinePass &addPass) const;
private:
DerivedT &derived() { return static_cast<DerivedT &>(*this); }
@@ -485,10 +593,8 @@ Error CodeGenPassBuilder<Derived>::buildPipeline(
AddIRPass addIRPass(MPM, Opt.DebugPM);
// `ProfileSummaryInfo` is always valid.
addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
- addISelPasses(addIRPass);
-
AddMachinePass addPass(MFPM);
- if (auto Err = addCoreISelPasses(addPass))
+ if (auto Err = addISelPasses(addIRPass, addPass))
return std::move(Err);
if (auto Err = derived().addMachinePasses(addPass))
@@ -525,18 +631,21 @@ void CodeGenPassBuilder<Derived>::registerModuleAnalyses(
ModuleAnalysisManager &MAM) const {
#define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
derived().registerTargetAnalysis(MAM);
+ if (Opt.RequiresCodeGenSCCOrder)
+ MAM.registerPass([&] { return CallGraphAnalysis(); });
}
template <typename Derived>
void CodeGenPassBuilder<Derived>::registerFunctionAnalyses(
FunctionAnalysisManager &FAM) const {
- FAM.registerPass([this] { return registerAAAnalyses(); });
+ if (getOptLevel() != CodeGenOptLevel::None)
+ FAM.registerPass([this] { return registerAAAnalyses(); });
#define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
derived().registerTargetAnalysis(FAM);
}
@@ -545,7 +654,8 @@ void CodeGenPassBuilder<Derived>::registerMachineFunctionAnalyses(
MachineFunctionAnalysisManager &MFAM) const {
#define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
+
derived().registerTargetAnalysis(MFAM);
}
@@ -596,17 +706,45 @@ CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {
return Ret;
}
+template <typename DerivedT>
+bool llvm::CodeGenPassBuilder<DerivedT>::parseMIRPass(
+ MachineFunctionPassManager &MFPM, StringRef Name) const {
+#define ADD_PASS(NAME, PASS_NAME) \
+ if (Name == NAME) { \
+ MFPM.addPass(PASS_NAME()); \
+ return false; \
+ }
+#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME)
+#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME)
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME)
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME)
+#include "llvm/CodeGen/MachinePassRegistry.def"
+#undef ADD_PASS
+ return true;
+}
+
template <typename Derived>
-void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
+Error CodeGenPassBuilder<Derived>::addISelPasses(
+ AddIRPass &addPass, AddMachinePass &addMachinePass) const {
if (TM.useEmulatedTLS())
addPass(LowerEmuTLSPass());
addPass(PreISelIntrinsicLoweringPass(TM));
+ addPass(createModuleToFunctionPassAdaptor(ExpandLargeDivRemPass(&TM)));
+ addPass(createModuleToFunctionPassAdaptor(ExpandLargeFpConvertPass(&TM)));
derived().addIRPasses(addPass);
derived().addCodeGenPrepare(addPass);
addPassesToHandleExceptions(addPass);
derived().addISelPrepare(addPass);
+
+ if (auto Err = addCoreISelPasses(addMachinePass))
+ return std::move(Err);
+ return Error::success();
}
/// Add common target configurable passes that perform LLVM IR to IR transforms
@@ -618,16 +756,15 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
if (!Opt.DisableVerify)
addPass(VerifierPass());
- // Run loop strength reduction before anything else.
- if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
- addPass(createFunctionToLoopPassAdaptor(
- LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
- // FIXME: use -stop-after so we could remove PrintLSR
- if (Opt.PrintLSR)
- addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
- }
-
if (getOptLevel() != CodeGenOptLevel::None) {
+ // Run loop strength reduction before anything else.
+ if (!Opt.DisableLSR) {
+ addPass(createFunctionToLoopPassAdaptor(
+ CanonicalizeFreezeInLoopsPass(), /*UseMemorySSA*/ true, Opt.DebugPM));
+ addPass(createFunctionToLoopPassAdaptor(
+ LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
+ }
+
// The MergeICmpsPass tries to create memcmp calls by grouping sequences of
// loads and compares. ExpandMemCmpPass then tries to expand those calls
// into optimally-sized loads and compares. The transforms are enabled by a
@@ -643,6 +780,12 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
addPass(ShadowStackGCLoweringPass());
addPass(LowerConstantIntrinsicsPass());
+ // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
+ // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
+ if (TM.getTargetTriple().isOSBinFormatMachO() &&
+ !Opt.DisableAtExitBasedGlobalDtorLowering)
+ addPass(LowerGlobalDtorsPass());
+
// Make sure that no unreachable blocks are instruction selected.
addPass(UnreachableBlockElimPass());
@@ -659,8 +802,10 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
!Opt.DisablePartialLibcallInlining)
addPass(PartiallyInlineLibCallsPass());
- // Instrument function entry and exit, e.g. with calls to mcount().
- addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+ // Expand vector predication intrinsics into standard IR instructions.
+ // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
+ // passes since it emits those kinds of intrinsics.
+ addPass(ExpandVectorPredicationPass());
// Add scalarization of target's unsupported masked memory intrinsics pass.
// the unsupported intrinsic will be replaced with a chain of basic blocks,
@@ -668,7 +813,12 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
addPass(ScalarizeMaskedMemIntrinPass());
// Expand reduction intrinsics into shuffle sequences if the target wants to.
- addPass(ExpandReductionsPass());
+ // Allow disabling it for testing purposes.
+ if (!Opt.DisableExpandReductions)
+ addPass(ExpandReductionsPass());
+
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(TLSVariableHoistPass());
// Convert conditional moves to conditional jumps when profitable.
if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
@@ -727,8 +877,6 @@ template <typename Derived>
void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
addPass(CodeGenPreparePass());
- // TODO: Default ctor'd RewriteSymbolPass is no-op.
- // addPass(RewriteSymbolPass());
}
/// Add common passes that perform LLVM IR to IR transforms in preparation for
@@ -825,8 +973,7 @@ Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
addPass(FinalizeISelPass());
// // Print the instruction selected machine code...
- // printAndVerify("After Instruction Selection");
-
+ printAndVerify(addPass, "After Instruction Selection");
return Error::success();
}
@@ -863,9 +1010,22 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
// Run pre-ra passes.
derived().addPreRegAlloc(addPass);
+ // Add a FSDiscriminator pass right before RA, so that we could get
+ // more precise SampleFDO profile for RA.
+ if (EnableFSDiscriminator) {
+ addPass(MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile();
+ if (!ProfileFile.empty() && !Opt.DisableRAFSProfileLoader)
+ addPass(MIRProfileLoaderNewPass(ProfileFile, getFSRemappingFile(),
+ sampleprof::FSDiscriminatorPass::Pass1,
+ nullptr));
+ }
+
// Run register allocation and passes that are tightly coupled with it,
// including phi elimination and scheduling.
- if (*Opt.OptimizeRegAlloc) {
+ bool IsOptimizeRegAlloc =
+ Opt.OptimizeRegAlloc.value_or(getOptLevel() != None);
+ if (IsOptimizeRegAlloc) {
derived().addOptimizedRegAlloc(addPass);
} else {
if (auto Err = derived().addFastRegAlloc(addPass))
@@ -877,13 +1037,19 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(RemoveRedundantDebugValuesPass());
- // Insert prolog/epilog code. Eliminate abstract frame index references...
+ addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index
+ // references...
if (getOptLevel() != CodeGenOptLevel::None) {
addPass(PostRAMachineSinkingPass());
addPass(ShrinkWrapPass());
}
- addPass(PrologEpilogInserterPass());
+ // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
+ // do so if it hasn't been disabled, substituted, or overridden.
+ if (!DisabledPasses.contains(PrologEpilogCodeInserterPass::ID()))
+ addPass(PrologEpilogInserterPass());
/// Add passes that optimize machine instructions after register allocation.
if (getOptLevel() != CodeGenOptLevel::None)
@@ -909,9 +1075,6 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(PostRASchedulerPass());
}
- // GC
- derived().addGCPasses(addPass);
-
// Basic block placement.
if (getOptLevel() != CodeGenOptLevel::None)
derived().addBlockPlacement(addPass);
@@ -929,6 +1092,8 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
// clobbered registers, to be used to optimize call sites.
addPass(RegUsageInfoCollectorPass());
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
addPass(FuncletLayoutPass());
addPass(StackMapLivenessPass());
@@ -945,8 +1110,51 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(MachineOutlinerPass(RunOnAllFunctions));
}
+ if (Opt.GCEmptyBlocks)
+ addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ // Machine function splitter uses the basic block sections feature. Both
+ // cannot be enabled at the same time. Basic block sections takes precedence.
+ // FIXME: In principle, BasicBlockSection::Labels and splitting can used
+ // together. Update this check once we have addressed any issues.
+ if (TM.getBBSectionsType() != llvm::BasicBlockSection::None) {
+ if (TM.getBBSectionsType() == llvm::BasicBlockSection::List) {
+ addPass(
+ BasicBlockSectionsProfileReaderPass(TM.getBBSectionsFuncListBuf()));
+ }
+ addPass(BasicBlockSectionsPass());
+ } else if (TM.Options.EnableMachineFunctionSplitter ||
+ Opt.EnableMachineFunctionSplitter) {
+ const std::string ProfileFile = getFSProfileFile();
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ addPass(MIRProfileLoaderNewPass(
+ ProfileFile, getFSRemappingFile(),
+ sampleprof::FSDiscriminatorPass::PassLast, nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.";
+ }
+ }
+ addPass(MachineFunctionSplitterPass());
+ }
+
+ derived().addPostBBSections(addPass);
+
+ if (!Opt.DisableCFIFixup && TM.Options.EnableCFIFixup)
+ addPass(CFIFixupPass());
+
+ addPass(StackFrameLayoutAnalysisPass());
+
// Add passes that directly emit MI after all other MI passes.
- derived().addPreEmitPass2(addPass);
+ derived().addPrecedingEmitPass(addPass);
return Error::success();
}
@@ -1035,7 +1243,7 @@ void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
}
template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
+Error CodeGenPassBuilder<Derived>::addRegAssignAndRewriteFast(
AddMachinePass &addPass) const {
if (Opt.RegAlloc != RegAllocType::Default &&
Opt.RegAlloc != RegAllocType::Fast)
@@ -1043,26 +1251,27 @@ Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
"Must use fast (default) register allocator for unoptimized regalloc.",
inconvertibleErrorCode());
- addRegAllocPass(addPass, false);
- return Error::success();
+ addPass(RegAllocPass(false));
+
+ // Allow targets to change the register assignments after
+ // fast register allocation.
+ return derived().addPostFastRegAllocRewrite(addPass);
}
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
+template <typename DerivedT>
+Error llvm::CodeGenPassBuilder<DerivedT>::addRegAssignAndRewriteOptimized(
AddMachinePass &addPass) const {
// Add the selected register allocation pass.
addRegAllocPass(addPass, true);
-
// Allow targets to change the register assignments before rewriting.
- derived().addPreRewrite(addPass);
+ addPreRewrite(addPass);
// Finally rewrite virtual registers.
addPass(VirtRegRewriterPass());
- // Perform stack slot coloring and post-ra machine LICM.
- //
- // FIXME: Re-enable coloring with register when it's capable of adding
- // kill markers.
- addPass(StackSlotColoringPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ addPass(RegAllocScoringPass());
return Error::success();
}
@@ -1074,7 +1283,7 @@ Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
AddMachinePass &addPass) const {
addPass(PHIEliminationPass());
addPass(TwoAddressInstructionPass());
- return derived().addRegAssignmentFast(addPass);
+ return derived().addRegAssignAndRewriteFast(addPass);
}
/// Add standard target-independent passes that are tightly coupled with
@@ -1105,7 +1314,8 @@ void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
// PreRA instruction scheduling.
addPass(MachineSchedulerPass());
- if (derived().addRegAssignmentOptimized(addPass)) {
+ Error Err = derived().addRegAssignAndRewriteOptimized(addPass);
+ if (!Err) {
// Allow targets to expand pseudo instructions depending on the choice of
// registers before MachineCopyPropagation.
derived().addPostRewrite(addPass);
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index a2641a8223646d..ec2a815f0cdfc7 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -27,6 +27,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Error.h"
+#include "llvm/Target/CGPassBuilderOption.h"
#include <map>
@@ -150,10 +151,7 @@ class MachineFunctionPassManager
using Base = PassManager<MachineFunction, MachineFunctionAnalysisManager>;
public:
- MachineFunctionPassManager(bool RequireCodeGenSCCOrder = false,
- bool VerifyMachineFunction = false)
- : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder),
- VerifyMachineFunction(VerifyMachineFunction) {}
+ MachineFunctionPassManager() : Opt(getCGPassBuilderOption()){};
MachineFunctionPassManager(MachineFunctionPassManager &&) = default;
MachineFunctionPassManager &
operator=(MachineFunctionPassManager &&) = default;
@@ -261,10 +259,7 @@ class MachineFunctionPassManager
using PassIndex = decltype(Passes)::size_type;
std::map<PassIndex, llvm::unique_function<FuncTy>> MachineModulePasses;
- // Run codegen in the SCC order.
- bool RequireCodeGenSCCOrder;
-
- bool VerifyMachineFunction;
+ CGPassBuilderOption Opt;
};
} // end namespace llvm
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 914e6b19fde9ac..ab7bc3bcc239c5 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -17,6 +17,51 @@
using namespace llvm;
+namespace {
+
+/// RAII class to replace addMachinePrePasses/addMachinePostPasses
+/// If we need more complex logics, consider add class like
+/// CodeGenInstrumentation
+// TODO: skip FreeMachineFunctionPass and AsmPrinter
+class PrePostPasses {
+ MachineFunction &MF;
+ MachineFunctionAnalysisManager &MFAM;
+ StringRef PassName;
+ const CGPassBuilderOption &Opt;
+
+public:
+ PrePostPasses(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM,
+ StringRef PassName, const CGPassBuilderOption &Opt)
+ : MF(MF), MFAM(MFAM), PassName(PassName), Opt(Opt) {
+ if (Opt.DebugifyCheckAndStripAll.value_or(false) ||
+ Opt.DebugifyAndStripAll.value_or(false)) {
+ // TODO: DebugifyPass().run(*MF.getMMI().getModule(), MFAM.MAM);
+ }
+
+ if (Opt.DebugifyAndStripAll.value_or(false)) {
+ // TODO: StripDebugPass().run(*MF.getMMI().getModule(), MFAM.MAM);
+ }
+ };
+
+ ~PrePostPasses() {
+ if (Opt.DebugifyCheckAndStripAll.value_or(false)) {
+ // TODO: CheckDebugPass().run(*MF.getMMI().getModule(), MFAM.MAM);
+ // TODO: StripDebugPass().run(*MF.getMMI().getModule(), MFAM.MAM);
+ }
+
+ if (Opt.DebugifyAndStripAll.value_or(false)) {
+ // TODO: StripDebugPass().run(*MF.getMMI().getModule(), MFAM.MAM);
+ }
+
+ if (Opt.VerifyMachineCode) {
+ std::string Banner = ("After " + PassName).str();
+ verifyMachineFunction(&MFAM, Banner, MF);
+ }
+ }
+};
+
+} // namespace
+
namespace llvm {
template class AllAnalysesOn<MachineFunction>;
template class AnalysisManager<MachineFunction>;
@@ -30,26 +75,11 @@ Error MachineFunctionPassManager::run(Module &M,
// result of MachineModuleAnalysis. MMI should not be recomputed.
auto &MMI = MFAM.getResult<MachineModuleAnalysis>(M);
- (void)RequireCodeGenSCCOrder;
- assert(!RequireCodeGenSCCOrder && "not implemented");
+ assert(!Opt.RequiresCodeGenSCCOrder && "not implemented");
// M is unused here
PassInstrumentation PI = MFAM.getResult<PassInstrumentationAnalysis>(M);
- // Add a PIC to verify machine functions.
- if (VerifyMachineFunction) {
- // No need to pop this callback later since MIR pipeline is flat which means
- // current pipeline is the top-level pipeline. Callbacks are not used after
- // current pipeline.
- PI.pushBeforeNonSkippedPassCallback([&MFAM](StringRef PassID, Any IR) {
- assert(llvm::any_cast<const MachineFunction *>(&IR));
- const MachineFunction *MF = llvm::any_cast<const MachineFunction *>(IR);
- assert(MF && "Machine function should be valid for printing");
- std::string Banner = std::string("After ") + std::string(PassID);
- verifyMachineFunction(&MFAM, Banner, *MF);
- });
- }
-
for (auto &F : InitializationFuncs) {
if (auto Err = F(M, MFAM))
return Err;
@@ -91,7 +121,7 @@ Error MachineFunctionPassManager::run(Module &M,
if (!PI.runBeforePass<MachineFunction>(*P, MF))
continue;
-
+ PrePostPasses Hook(MF, MFAM, P->name(), Opt);
// TODO: EmitSizeRemarks
PreservedAnalyses PassPA = P->run(MF, MFAM);
MFAM.invalidate(MF, PassPA);
More information about the llvm-commits
mailing list