[llvm] [NewPM][CodeGen] add TargetPassConfig like API (PR #70906)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 1 01:47:26 PDT 2023
https://github.com/paperchalice updated https://github.com/llvm/llvm-project/pull/70906
>From ed726eb26e1012cdec12daa3fa4f4d26b46f24ed Mon Sep 17 00:00:00 2001
From: liujunchang <lgamma at 163.com>
Date: Wed, 1 Nov 2023 10:41:58 +0800
Subject: [PATCH 1/4] [NewPM][CodeGen] add TargetPassConfig like API
---
.../include/llvm/CodeGen/CodeGenPassBuilder.h | 429 +++++++++++++++---
llvm/include/llvm/Target/TargetMachine.h | 18 +-
2 files changed, 382 insertions(+), 65 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index d7739e8bb597e4e..9f0887123195732 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -19,10 +19,12 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/ReplaceWithVeclib.h"
@@ -36,6 +38,8 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/SaveAndRestore.h"
+#include "llvm/Support/WithColor.h"
#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Scalar/ConstantHoisting.h"
@@ -45,7 +49,10 @@
#include "llvm/Transforms/Scalar/MergeICmps.h"
#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
+#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
#include "llvm/Transforms/Utils/LowerInvoke.h"
#include <cassert>
#include <type_traits>
@@ -122,6 +129,26 @@ template <typename DerivedT> class CodeGenPassBuilder {
raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
CodeGenFileType FileType) const;
+ bool parseTargetMIRPass(MachineFunctionPassManager &MFPM,
+ StringRef Name) const {
+ llvm_unreachable("parseTargetMIRPass is not overridden");
+ }
+
+ bool parseMIRPass(MachineFunctionPassManager &MFPM, StringRef Name) const;
+
+ Error parseMIRPipeline(MachineFunctionPassManager &MFPM,
+ StringRef Text) const {
+ for (auto [LHS, RHS] = Text.split(','); RHS != "";
+ std::tie(LHS, RHS) = RHS.split(',')) {
+ if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, Text)) {
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ Twine('\"') + Twine(LHS) + Twine("\" pass could not be found."));
+ }
+ }
+ return Error::success();
+ }
+
void registerModuleAnalyses(ModuleAnalysisManager &) const;
void registerFunctionAnalyses(FunctionAnalysisManager &) const;
void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const;
@@ -134,7 +161,19 @@ template <typename DerivedT> class CodeGenPassBuilder {
}
PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
- return PIC;
+ static PassInstrumentationCallbacks PseudoPIC;
+ return PIC ? PIC : &PseudoPIC;
+ }
+
+ /// Allow the target to enable a specific standard pass by default.
+ // TODO: implement it
+ template <typename PassT> void enablePass() {}
+
+ /// Allow the target to disable a specific standard pass by default.
+ template <typename PassT> void disablePass() {
+ DisabledPasses.insert(&PassT::Key);
+ getPassInstrumentationCallbacks()->registerShouldRunOptionalPassCallback(
+ [](StringRef P, Any IR) { return P != PassT::name(); });
}
protected:
@@ -195,7 +234,22 @@ template <typename DerivedT> class CodeGenPassBuilder {
// Function object to maintain state while adding codegen machine passes.
class AddMachinePass {
public:
- AddMachinePass(MachineFunctionPassManager &PM) : PM(PM) {}
+ AddMachinePass(const CodeGenPassBuilder &Builder,
+ MachineFunctionPassManager &PM, bool AddPrePostHook = false)
+ : AddPrePostHook(AddPrePostHook), Builder(Builder), PM(PM) {
+ BeforeCallbacks.emplace_back([this](AnalysisKey *) {
+ if (this->AddPrePostHook)
+ this->Builder.addMachinePrePasses(this->PM);
+ return true;
+ });
+
+ AfterCallbacks.emplace_back([this](AnalysisKey *, StringRef Name) {
+ if (this->AddPrePostHook) {
+ std::string Banner = "After " + Name.str();
+ this->Builder.addMachinePostPasses(this->PM, Banner);
+ }
+ });
+ }
template <typename PassT> void operator()(PassT &&Pass) {
static_assert(
@@ -206,41 +260,62 @@ template <typename DerivedT> class CodeGenPassBuilder {
return;
PM.addPass(std::forward<PassT>(Pass));
for (auto &C : AfterCallbacks)
- C(&PassT::Key);
+ C(&PassT::Key, PassT::name());
}
template <typename PassT> void insertPass(AnalysisKey *ID, PassT Pass) {
AfterCallbacks.emplace_back(
- [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) {
+ [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID, StringRef) {
if (PassID == ID)
this->PM.addPass(std::move(Pass));
});
}
- void disablePass(AnalysisKey *ID) {
- BeforeCallbacks.emplace_back(
- [ID](AnalysisKey *PassID) { return PassID != ID; });
- }
-
MachineFunctionPassManager releasePM() { return std::move(PM); }
+ bool AddPrePostHook;
+
private:
+ const CodeGenPassBuilder &Builder;
MachineFunctionPassManager &PM;
SmallVector<llvm::unique_function<bool(AnalysisKey *)>, 4> BeforeCallbacks;
- SmallVector<llvm::unique_function<void(AnalysisKey *)>, 4> AfterCallbacks;
+ SmallVector<llvm::unique_function<void(AnalysisKey *, StringRef)>, 4>
+ AfterCallbacks;
};
+ // Find the FSProfile file name. The internal option takes the precedence
+ // before getting from TargetMachine.
+ std::string getFSProfileFile() const {
+ if (!Opt.FSProfileFile.empty())
+ return Opt.FSProfileFile;
+ const std::optional<PGOOptions> &PGOOpt = TM.getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileFile;
+ }
+
+ // Find the Profile remapping file name. The internal option takes the
+ // precedence before getting from TargetMachine.
+ std::string getFSRemappingFile() const {
+ if (!Opt.FSRemappingFile.empty())
+ return Opt.FSRemappingFile;
+ const std::optional<PGOOptions> &PGOOpt = TM.getPGOOption();
+ if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
+ return std::string();
+ return PGOOpt->ProfileRemappingFile;
+ }
+
+ DenseSet<AnalysisKey *> DisabledPasses;
LLVMTargetMachine &TM;
CGPassBuilderOption Opt;
PassInstrumentationCallbacks *PIC;
+ mutable bool DebugifyIsSafe = true;
+ mutable bool AddGCInfoPrinter = false;
/// Target override these hooks to parse target-specific analyses.
void registerTargetAnalysis(ModuleAnalysisManager &) const {}
void registerTargetAnalysis(FunctionAnalysisManager &) const {}
void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {}
- std::pair<StringRef, bool> getTargetPassNameFromLegacyName(StringRef) const {
- return {"", false};
- }
template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); }
@@ -308,6 +383,10 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// immediately before machine code is emitted.
void addPreEmitPass(AddMachinePass &) const {}
+ /// This pass may be implemented by targets that want to run passes
+ /// immediately after basic block sections are assigned.
+ void addPostBBSections(AddMachinePass &) const {}
+
/// Targets may add passes immediately before machine code is emitted in this
/// callback. This is called even later than `addPreEmitPass`.
// FIXME: Rename `addPreEmitPass` to something more sensible given its actual
@@ -383,6 +462,15 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// Fully developed targets will not generally override this.
Error addMachinePasses(AddMachinePass &) const;
+ /// Add single pass to pass manager from `llc -passes=` option
+ template <typename PassT>
+ void addMachinePass(MachineFunctionPassManager &MFPM, PassT &&Pass) const {
+ addMachinePrePasses(MFPM);
+ std::string Banner = "After " + PassT::name().str();
+ MFPM.addPass(std::forward<PassT>(Pass));
+ addMachinePostPasses(MFPM, Banner);
+ }
+
/// Add passes to lower exception handling for the code generator.
void addPassesToHandleExceptions(AddIRPass &) const;
@@ -413,6 +501,14 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// are required for fast register allocation.
Error addFastRegAlloc(AddMachinePass &) const;
+ /// addPostFastRegAllocRewrite - Add passes to the optimized register
+ /// allocation pipeline after fast register allocation is complete.
+ Error addPostFastRegAllocRewrite(AddMachinePass &) const {
+ return make_error<StringError>(
+ "addPostFastRegAllocRewrite is not overridden",
+ inconvertibleErrorCode());
+ }
+
/// addOptimizedRegAlloc - Add passes related to register allocation.
/// LLVMTargetMachine provides standard regalloc passes for most targets.
void addOptimizedRegAlloc(AddMachinePass &) const;
@@ -420,14 +516,63 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// Add passes that optimize machine instructions after register allocation.
void addMachineLateOptimization(AddMachinePass &) const;
- /// addGCPasses - Add late codegen passes that analyze code for garbage
+ /// registerGCPasses - Add late codegen passes that analyze code for garbage
/// collection. This should return true if GC info should be printed after
/// these passes.
- void addGCPasses(AddMachinePass &) const {}
+ bool registerGCPasses(MachineFunctionAnalysisManager &MFAM) const {
+ MFAM.registerPass([] { return GCMachineCodeAnalysisPass(); });
+ return true;
+ }
/// Add standard basic block placement passes.
void addBlockPlacement(AddMachinePass &) const;
+ /// Add a pass to print the machine function if printing is enabled.
+ void addPrintPass(AddMachinePass &addPass, const std::string &Banner) const {
+ if (Opt.PrintAfterISel)
+ addPass(MachineFunctionPrinterPass(dbgs(), Banner));
+ }
+
+ /// Add a pass to perform basic verification of the machine function if
+ /// verification is enabled.
+ void addVerifyPass(MachineFunctionPassManager &MFPM,
+ const std::string &Banner) const {
+ bool Verify = Opt.VerifyMachineCode.value_or(true);
+#ifdef EXPENSIVE_CHECKS
+ if (!Opt.VerifyMachineCode)
+ Verify = TM->isMachineVerifierClean();
+#endif
+ if (Verify)
+ MFPM.addPass(MachineVerifierPass(Banner));
+ }
+
+ void addVerifyPass(AddMachinePass &addPass, const std::string &Banner) const {
+ bool Verify = Opt.VerifyMachineCode.value_or(true);
+#ifdef EXPENSIVE_CHECKS
+ if (!Opt.VerifyMachineCode)
+ Verify = TM->isMachineVerifierClean();
+#endif
+ if (Verify)
+ addPass(MachineVerifierPass(Banner));
+ }
+
+ /// printAndVerify - Add a pass to dump then verify the machine function, if
+ /// those steps are enabled.
+ void printAndVerify(AddMachinePass &addPass,
+ const std::string &Banner) const {
+ addPrintPass(addPass, Banner);
+ addVerifyPass(addPass, Banner);
+ }
+
+ /// Add standard passes before a pass that's about to be added. For example,
+ /// the DebugifyMachineModulePass if it is enabled.
+ void addMachinePrePasses(MachineFunctionPassManager &MFPM) const;
+
+ /// Add standard passes after a pass that has just been added. For example,
+ /// the MachineVerifier if it is enabled.
+ void addMachinePostPasses(MachineFunctionPassManager &addPass,
+ const std::string &Banner) const;
+
using CreateMCStreamer =
std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
@@ -445,10 +590,10 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// regalloc pass.
void addRegAllocPass(AddMachinePass &, bool Optimized) const;
- /// Add core register alloator passes which do the actual register assignment
- /// and rewriting. \returns true if any passes were added.
- Error addRegAssignmentFast(AddMachinePass &) const;
- Error addRegAssignmentOptimized(AddMachinePass &) const;
+ /// Add core register allocator passes which do the actual register assignment
+ /// and rewriting. \returns Error::success() if any passes were added.
+ Error addRegAssignAndRewriteFast(AddMachinePass &addPass) const;
+ Error addRegAssignAndRewriteOptimized(AddMachinePass &addPass) const;
private:
DerivedT &derived() { return static_cast<DerivedT &>(*this); }
@@ -465,12 +610,14 @@ Error CodeGenPassBuilder<Derived>::buildPipeline(
AddIRPass addIRPass(MPM, Opt.DebugPM);
addISelPasses(addIRPass);
- AddMachinePass addPass(MFPM);
+ AddMachinePass addPass(*this, MFPM);
if (auto Err = addCoreISelPasses(addPass))
return std::move(Err);
+ addPass.AddPrePostHook = true;
if (auto Err = derived().addMachinePasses(addPass))
return std::move(Err);
+ addPass.AddPrePostHook = false;
derived().addAsmPrinter(
addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
@@ -503,18 +650,21 @@ void CodeGenPassBuilder<Derived>::registerModuleAnalyses(
ModuleAnalysisManager &MAM) const {
#define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
derived().registerTargetAnalysis(MAM);
+ if (Opt.RequiresCodeGenSCCOrder)
+ MAM.registerPass([&] { return CallGraphAnalysis(); });
}
template <typename Derived>
void CodeGenPassBuilder<Derived>::registerFunctionAnalyses(
FunctionAnalysisManager &FAM) const {
- FAM.registerPass([this] { return registerAAAnalyses(); });
+ if (getOptLevel() != CodeGenOptLevel::None)
+ FAM.registerPass([this] { return registerAAAnalyses(); });
#define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
derived().registerTargetAnalysis(FAM);
}
@@ -523,7 +673,9 @@ void CodeGenPassBuilder<Derived>::registerMachineFunctionAnalyses(
MachineFunctionAnalysisManager &MFAM) const {
#define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
-#include "MachinePassRegistry.def"
+#include "llvm/CodeGen/MachinePassRegistry.def"
+
+ AddGCInfoPrinter = derived().registerGCPasses(MFAM) && Opt.PrintGCInfo;
derived().registerTargetAnalysis(MFAM);
}
@@ -574,12 +726,47 @@ CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {
return Ret;
}
+template <typename DerivedT>
+bool llvm::CodeGenPassBuilder<DerivedT>::parseMIRPass(
+ MachineFunctionPassManager &MFPM, StringRef Name) const {
+#define ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) { \
+ { \
+ addMachinePass(MFPM, PASS_NAME CONSTRUCTOR); \
+ } \
+ return false; \
+ }
+#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) { \
+ { \
+ addMachinePass(MFPM, PASS_NAME()); \
+ } \
+ return false; \
+ }
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) { \
+ { \
+ addMachinePass(MFPM, PASS_NAME()); \
+ } \
+ return false; \
+ }
+#include "llvm/CodeGen/MachinePassRegistry.def"
+#undef ADD_PASS
+ return true;
+}
+
template <typename Derived>
void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
if (TM.useEmulatedTLS())
addPass(LowerEmuTLSPass());
addPass(PreISelIntrinsicLoweringPass(TM));
+ addPass(createModuleToFunctionPassAdaptor(ExpandLargeDivRemPass()));
+ addPass(createModuleToFunctionPassAdaptor(ExpandLargeFpConvertPass()));
derived().addIRPasses(addPass);
derived().addCodeGenPrepare(addPass);
@@ -596,16 +783,18 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
if (!Opt.DisableVerify)
addPass(VerifierPass());
- // Run loop strength reduction before anything else.
- if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
- addPass(createFunctionToLoopPassAdaptor(
- LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
- // FIXME: use -stop-after so we could remove PrintLSR
- if (Opt.PrintLSR)
- addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
- }
-
if (getOptLevel() != CodeGenOptLevel::None) {
+ // Run loop strength reduction before anything else.
+ if (!Opt.DisableLSR) {
+ addPass(createFunctionToLoopPassAdaptor(
+ CanonicalizeFreezeInLoopsPass(), /*UseMemorySSA*/ true, Opt.DebugPM));
+ addPass(createFunctionToLoopPassAdaptor(
+ LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
+ // FIXME: use -stop-after so we could remove PrintLSR
+ if (Opt.PrintLSR)
+ addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
+ }
+
// The MergeICmpsPass tries to create memcmp calls by grouping sequences of
// loads and compares. ExpandMemCmpPass then tries to expand those calls
// into optimally-sized loads and compares. The transforms are enabled by a
@@ -621,6 +810,15 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
addPass(ShadowStackGCLoweringPass());
addPass(LowerConstantIntrinsicsPass());
+ if (AddGCInfoPrinter)
+ addPass(GCInfoPrinterPass(dbgs));
+
+ // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
+ // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
+ if (TM.getTargetTriple().isOSBinFormatMachO() &&
+ !Opt.DisableAtExitBasedGlobalDtorLowering)
+ addPass(LowerGlobalDtorsPass());
+
// Make sure that no unreachable blocks are instruction selected.
addPass(UnreachableBlockElimPass());
@@ -637,8 +835,10 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
!Opt.DisablePartialLibcallInlining)
addPass(PartiallyInlineLibCallsPass());
- // Instrument function entry and exit, e.g. with calls to mcount().
- addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+ // Expand vector predication intrinsics into standard IR instructions.
+ // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
+ // passes since it emits those kinds of intrinsics.
+ addPass(ExpandVectorPredicationPass());
// Add scalarization of target's unsupported masked memory intrinsics pass.
// the unsupported intrinsic will be replaced with a chain of basic blocks,
@@ -646,7 +846,12 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
addPass(ScalarizeMaskedMemIntrinPass());
// Expand reduction intrinsics into shuffle sequences if the target wants to.
- addPass(ExpandReductionsPass());
+ // Allow disabling it for testing purposes.
+ if (!Opt.DisableExpandReductions)
+ addPass(ExpandReductionsPass());
+
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(TLSVariableHoistPass());
// Convert conditional moves to conditional jumps when profitable.
if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
@@ -705,8 +910,6 @@ template <typename Derived>
void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
addPass(CodeGenPreparePass());
- // TODO: Default ctor'd RewriteSymbolPass is no-op.
- // addPass(RewriteSymbolPass());
}
/// Add common passes that perform LLVM IR to IR transforms in preparation for
@@ -716,6 +919,7 @@ void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
derived().addPreISel(addPass);
addPass(CallBrPrepare());
+
// Add both the safe stack and the stack protection passes: each of them will
// only protect functions that have corresponding attributes.
addPass(SafeStackPass());
@@ -754,6 +958,19 @@ Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
else
Selector = SelectorType::SelectionDAG;
+ // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
+ // analyses needing to be re-run. This can result in being unable to
+ // schedule passes (particularly with 'Function Alias Analysis
+ // Results'). It's not entirely clear why but AFAICT this seems to be
+ // due to one FunctionPassManager not being able to use analyses from a
+ // previous one. As we're injecting a ModulePass we break the usual
+ // pass manager into two. GlobalISel with the fallback path disabled
+ // and -run-pass seem to be unaffected. The majority of GlobalISel
+ // testing uses -run-pass so this probably isn't too bad.
+ SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
+ if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
+ DebugifyIsSafe = false;
+
// Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
if (Selector == SelectorType::FastISel) {
TM.setFastISel(true);
@@ -765,6 +982,7 @@ Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
// Add instruction selector passes.
if (Selector == SelectorType::GlobalISel) {
+ SaveAndRestore SavedAddingMachinePasses(addPass.AddPrePostHook, true);
if (auto Err = derived().addIRTranslator(addPass))
return std::move(Err);
@@ -803,8 +1021,7 @@ Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
addPass(FinalizeISelPass());
// // Print the instruction selected machine code...
- // printAndVerify("After Instruction Selection");
-
+ printAndVerify(addPass, "After Instruction Selection");
return Error::success();
}
@@ -841,9 +1058,28 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
// Run pre-ra passes.
derived().addPreRegAlloc(addPass);
+ // Debugifying the register allocator passes seems to provoke some
+ // non-determinism that affects CodeGen and there doesn't seem to be a point
+ // where it becomes safe again so stop debugifying here.
+ DebugifyIsSafe = false;
+
+ // Add a FSDiscriminator pass right before RA, so that we could get
+ // more precise SampleFDO profile for RA.
+ if (EnableFSDiscriminator) {
+ addPass(MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1));
+ const std::string ProfileFile = getFSProfileFile();
+ if (!ProfileFile.empty() && !Opt.DisableRAFSProfileLoader)
+ addPass(MIRProfileLoaderPass(ProfileFile, getFSRemappingFile(),
+ sampleprof::FSDiscriminatorPass::Pass1,
+ nullptr));
+ }
+
// Run register allocation and passes that are tightly coupled with it,
// including phi elimination and scheduling.
- if (*Opt.OptimizeRegAlloc) {
+ bool IsOptimizeRegAlloc = Opt.OptimizeRegAlloc.has_value()
+ ? *Opt.OptimizeRegAlloc
+ : getOptLevel() != CodeGenOptLevel::None;
+ if (IsOptimizeRegAlloc) {
derived().addOptimizedRegAlloc(addPass);
} else {
if (auto Err = derived().addFastRegAlloc(addPass))
@@ -855,13 +1091,20 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(RemoveRedundantDebugValuesPass());
- // Insert prolog/epilog code. Eliminate abstract frame index references...
+ addPass(FixupStatepointCallerSavedPass());
+
+ // Insert prolog/epilog code. Eliminate abstract frame index
+ // references...
if (getOptLevel() != CodeGenOptLevel::None) {
addPass(PostRAMachineSinkingPass());
addPass(ShrinkWrapPass());
}
- addPass(PrologEpilogInserterPass());
+ // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
+ // do so if it hasn't been disabled, substituted, or overridden.
+
+ if (!DisabledPasses.contains(&PrologEpilogCodeInserterPass::Key))
+ addPass(PrologEpilogInserterPass());
/// Add passes that optimize machine instructions after register allocation.
if (getOptLevel() != CodeGenOptLevel::None)
@@ -887,9 +1130,6 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(PostRASchedulerPass());
}
- // GC
- derived().addGCPasses(addPass);
-
// Basic block placement.
if (getOptLevel() != CodeGenOptLevel::None)
derived().addBlockPlacement(addPass);
@@ -907,6 +1147,8 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
// clobbered registers, to be used to optimize call sites.
addPass(RegUsageInfoCollectorPass());
+ // FIXME: Some backends are incompatible with running the verifier after
+ // addPreEmitPass. Maybe only pass "false" here for those targets?
addPass(FuncletLayoutPass());
addPass(StackMapLivenessPass());
@@ -923,12 +1165,77 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
addPass(MachineOutlinerPass(RunOnAllFunctions));
}
+ if (Opt.GCEmptyBlocks)
+ addPass(GCEmptyBasicBlocksPass());
+
+ if (EnableFSDiscriminator)
+ addPass(
+ MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast));
+
+ // Machine function splitter uses the basic block sections feature. Both
+ // cannot be enabled at the same time. Basic block sections takes precedence.
+ // FIXME: In principle, BasicBlockSection::Labels and splitting can used
+ // together. Update this check once we have addressed any issues.
+ if (TM.getBBSectionsType() != llvm::BasicBlockSection::None) {
+ if (TM.getBBSectionsType() == llvm::BasicBlockSection::List) {
+ addPass(
+ BasicBlockSectionsProfileReaderPass(TM.getBBSectionsFuncListBuf()));
+ }
+ addPass(BasicBlockSectionsPass());
+ } else if (TM.Options.EnableMachineFunctionSplitter ||
+ Opt.EnableMachineFunctionSplitter) {
+ const std::string ProfileFile = getFSProfileFile();
+ if (!ProfileFile.empty()) {
+ if (EnableFSDiscriminator) {
+ addPass(MIRProfileLoaderPass(ProfileFile, getFSRemappingFile(),
+ sampleprof::FSDiscriminatorPass::PassLast,
+ nullptr));
+ } else {
+ // Sample profile is given, but FSDiscriminator is not
+ // enabled, this may result in performance regression.
+ WithColor::warning()
+ << "Using AutoFDO without FSDiscriminator for MFS may regress "
+ "performance.";
+ }
+ }
+ addPass(MachineFunctionSplitterPass());
+ }
+
+ derived().addPostBBSections(addPass);
+
+ if (!Opt.DisableCFIFixup && TM.Options.EnableCFIFixup)
+ addPass(CFIFixupPass());
+
+ addPass(StackFrameLayoutAnalysisPass());
+
// Add passes that directly emit MI after all other MI passes.
derived().addPreEmitPass2(addPass);
return Error::success();
}
+template <typename DerivedT>
+void CodeGenPassBuilder<DerivedT>::addMachinePrePasses(
+ MachineFunctionPassManager &MFPM) const {
+ if (DebugifyIsSafe &&
+ ((Opt.DebugifyAndStripAll && *Opt.DebugifyAndStripAll) ||
+ (Opt.DebugifyCheckAndStripAll && *Opt.DebugifyCheckAndStripAll)))
+ MFPM.addPass(DebugifyMachineModule());
+}
+
+template <typename DerivedT>
+void llvm::CodeGenPassBuilder<DerivedT>::addMachinePostPasses(
+ MachineFunctionPassManager &MFPM, const std::string &Banner) const {
+ if (DebugifyIsSafe) {
+ if (Opt.DebugifyCheckAndStripAll && *Opt.DebugifyCheckAndStripAll) {
+ MFPM.addPass(CheckDebugMachineModulePass());
+ MFPM.addPass(StripDebugMachineModulePass(/*OnlyDebugified=*/true));
+ } else if (Opt.DebugifyAndStripAll && *Opt.DebugifyAndStripAll)
+ MFPM.addPass(StripDebugMachineModulePass(/*OnlyDebugified=*/true));
+ }
+ addVerifyPass(MFPM, Banner);
+}
+
/// Add passes that optimize machine instructions in SSA form.
template <typename Derived>
void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
@@ -1013,7 +1320,7 @@ void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
}
template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
+Error CodeGenPassBuilder<Derived>::addRegAssignAndRewriteFast(
AddMachinePass &addPass) const {
if (Opt.RegAlloc != RegAllocType::Default &&
Opt.RegAlloc != RegAllocType::Fast)
@@ -1021,26 +1328,27 @@ Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
"Must use fast (default) register allocator for unoptimized regalloc.",
inconvertibleErrorCode());
- addRegAllocPass(addPass, false);
- return Error::success();
+ addPass(RegAllocPass(false));
+
+ // Allow targets to change the register assignments after
+ // fast register allocation.
+ return derived().addPostFastRegAllocRewrite(addPass);
}
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
+template <typename DerivedT>
+Error llvm::CodeGenPassBuilder<DerivedT>::addRegAssignAndRewriteOptimized(
AddMachinePass &addPass) const {
// Add the selected register allocation pass.
addRegAllocPass(addPass, true);
-
// Allow targets to change the register assignments before rewriting.
- derived().addPreRewrite(addPass);
+ addPreRewrite(addPass);
// Finally rewrite virtual registers.
addPass(VirtRegRewriterPass());
- // Perform stack slot coloring and post-ra machine LICM.
- //
- // FIXME: Re-enable coloring with register when it's capable of adding
- // kill markers.
- addPass(StackSlotColoringPass());
+
+ // Regalloc scoring for ML-driven eviction - noop except when learning a new
+ // eviction policy.
+ addPass(RegAllocScoringPass());
return Error::success();
}
@@ -1052,7 +1360,7 @@ Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
AddMachinePass &addPass) const {
addPass(PHIEliminationPass());
addPass(TwoAddressInstructionPass());
- return derived().addRegAssignmentFast(addPass);
+ return derived().addRegAssignAndRewriteFast(addPass);
}
/// Add standard target-independent passes that are tightly coupled with
@@ -1083,7 +1391,8 @@ void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
// PreRA instruction scheduling.
addPass(MachineSchedulerPass());
- if (derived().addRegAssignmentOptimized(addPass)) {
+ Error Err = derived().addRegAssignAndRewriteOptimized(addPass);
+ if (!Err) {
// Allow targets to expand pseudo instructions depending on the choice of
// registers before MachineCopyPropagation.
derived().addPostRewrite(addPass);
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index c1d05b25ea21f8a..e28a5c00f7c9a6d 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -14,6 +14,7 @@
#define LLVM_TARGET_TARGETMACHINE_H
#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Allocator.h"
@@ -450,19 +451,26 @@ class LLVMTargetMachine : public TargetMachine {
bool DisableVerify = true,
MachineModuleInfoWrapperPass *MMIWP = nullptr) override;
- virtual Error buildCodeGenPipeline(ModulePassManager &,
- MachineFunctionPassManager &,
- MachineFunctionAnalysisManager &,
+ virtual Error buildCodeGenPipeline(ModulePassManager &MPM,
+ MachineFunctionPassManager &MFPM,
raw_pwrite_stream &, raw_pwrite_stream *,
CodeGenFileType, CGPassBuilderOption,
+ MachineFunctionAnalysisManager &,
PassInstrumentationCallbacks *) {
return make_error<StringError>("buildCodeGenPipeline is not overridden",
inconvertibleErrorCode());
}
virtual std::pair<StringRef, bool> getPassNameFromLegacyName(StringRef) {
- llvm_unreachable(
- "getPassNameFromLegacyName parseMIRPipeline is not overridden");
+ llvm_unreachable("getPassNameFromLegacyName is not overridden");
+ }
+
+ virtual Error parseMIRPipeline(MachineFunctionPassManager &MFPM,
+ StringRef PipelineText,
+ CGPassBuilderOption Opts,
+ MachineFunctionAnalysisManager &MFAM,
+ PassInstrumentationCallbacks *PIC) {
+ llvm_unreachable("parseMIRPipeline is not overridden");
}
/// Add passes to the specified pass manager to get machine code emitted with
>From 38ba52e18be8ba565cb9a1699f77cfe2c0bb27e6 Mon Sep 17 00:00:00 2001
From: liujunchang <lgamma at 163.com>
Date: Wed, 1 Nov 2023 16:39:43 +0800
Subject: [PATCH 2/4] [CodeGenPassBuilder] Add pass string format description
---
llvm/include/llvm/CodeGen/CodeGenPassBuilder.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index 9f0887123195732..bc14f49890db723 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -136,6 +136,10 @@ template <typename DerivedT> class CodeGenPassBuilder {
bool parseMIRPass(MachineFunctionPassManager &MFPM, StringRef Name) const;
+ /// Parse MIR pass pipeline. Unlike IR pass pipeline
+ /// there is only one pass manager for machine function
+ /// so there is no need to specify the passs nesting
+ /// @param Text a comma separated pass name list
Error parseMIRPipeline(MachineFunctionPassManager &MFPM,
StringRef Text) const {
for (auto [LHS, RHS] = Text.split(','); RHS != "";
>From 0720cab410ba25686fa3b99a88f3a7fe812a137a Mon Sep 17 00:00:00 2001
From: liujunchang <lgamma at 163.com>
Date: Wed, 1 Nov 2023 16:41:20 +0800
Subject: [PATCH 3/4] fix stop condition in parseMIRPipeline
---
llvm/include/llvm/CodeGen/CodeGenPassBuilder.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index bc14f49890db723..d3e35b115447837 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -142,7 +142,7 @@ template <typename DerivedT> class CodeGenPassBuilder {
/// @param Text a comma separated pass name list
Error parseMIRPipeline(MachineFunctionPassManager &MFPM,
StringRef Text) const {
- for (auto [LHS, RHS] = Text.split(','); RHS != "";
+ for (auto [LHS, RHS] = Text.split(','); LHS != "";
std::tie(LHS, RHS) = RHS.split(',')) {
if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, Text)) {
return createStringError(
>From b4ca36a545bc9e21997b21f7d763e758c40e4d81 Mon Sep 17 00:00:00 2001
From: liujunchang <lgamma at 163.com>
Date: Wed, 1 Nov 2023 16:47:06 +0800
Subject: [PATCH 4/4] pass the correct parameter to parseTargetMIRPass
---
llvm/include/llvm/CodeGen/CodeGenPassBuilder.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index d3e35b115447837..f4b1b508e4b16d9 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -144,7 +144,7 @@ template <typename DerivedT> class CodeGenPassBuilder {
StringRef Text) const {
for (auto [LHS, RHS] = Text.split(','); LHS != "";
std::tie(LHS, RHS) = RHS.split(',')) {
- if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, Text)) {
+ if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, LHS)) {
return createStringError(
std::make_error_code(std::errc::invalid_argument),
Twine('\"') + Twine(LHS) + Twine("\" pass could not be found."));
More information about the llvm-commits
mailing list