[llvm] 31ecf8d - [NewPM][CodeGen] Introduce CodeGenPassBuilder to help build codegen pipeline
Yuanfang Chen via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 16:41:48 PDT 2020
Author: Yuanfang Chen
Date: 2020-09-11T16:41:17-07:00
New Revision: 31ecf8d29d81d196374a562c6d2bd2c25a62861e
URL: https://github.com/llvm/llvm-project/commit/31ecf8d29d81d196374a562c6d2bd2c25a62861e
DIFF: https://github.com/llvm/llvm-project/commit/31ecf8d29d81d196374a562c6d2bd2c25a62861e.diff
LOG: [NewPM][CodeGen] Introduce CodeGenPassBuilder to help build codegen pipeline
Following up on D67687.
Please refer to the RFC here http://lists.llvm.org/pipermail/llvm-dev/2020-July/143309.html
`CodeGenPassBuilder` is the NPM counterpart of `TargetPassConfig` with below differences.
- Debugging features (MIR print/verify, disable pass, start/stop-before/after, etc.) living in `TargetPassConfig` are moved to use PassInstrument as much as possible. (Implementation also lives in `TargetPassConfig.cpp`)
- `TargetPassConfig` is a polymorphic base (virtual inheritance) to build the target-dependent pipeline whereas `CodeGenPassBuilder` is the CRTP base/helper to implement the target-dependent pipeline. The motivation is flexibility for targets to customize the pipeline, inlining opportunity, and fits the overall NPM value semantics design.
- `TargetPassConfig` is a legacy immutable pass to declare hooks for targets to customize some target-independent codegen layer behavior. This is partially ported to TargetMachine::options. The rest, such as `createMachineScheduler/createPostMachineScheduler`, are left out for now. They should be implemented in LLVMTargetMachine in the future.
Reviewed By: arsenm
Differential Revision: https://reviews.llvm.org/D83608
Added:
llvm/include/llvm/CodeGen/CGPassBuilderOption.h
llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
llvm/include/llvm/CodeGen/MachinePassRegistry.def
llvm/lib/CodeGen/CodeGenPassBuilder.cpp
Modified:
llvm/include/llvm/Passes/StandardInstrumentations.h
llvm/include/llvm/Target/TargetMachine.h
llvm/lib/CodeGen/CMakeLists.txt
llvm/lib/CodeGen/LLVMTargetMachine.cpp
llvm/lib/CodeGen/TargetPassConfig.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/CGPassBuilderOption.h b/llvm/include/llvm/CodeGen/CGPassBuilderOption.h
new file mode 100644
index 000000000000..4553060e687b
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/CGPassBuilderOption.h
@@ -0,0 +1,110 @@
+//===- CGPassBuilderOption.h - Options for pass builder ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the options influencing building of codegen pipeline.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_CGPASSBUILDEROPTION_H
+#define LLVM_CODEGEN_CGPASSBUILDEROPTION_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Target/TargetOptions.h"
+#include <vector>
+
+namespace llvm {
+class TargetMachine;
+
+enum class RunOutliner { TargetDefault, AlwaysOutline, NeverOutline };
+enum class RegAllocType { Default, Basic, Fast, Greedy, PBQP };
+enum class CFLAAType { None, Steensgaard, Andersen, Both };
+
+// Not one-on-one but mostly corresponding to commandline options in
+// TargetPassConfig.cpp
+struct CGPassBuilderOption {
+ // Enable optimized register allocation compilation path
+ Optional<bool> OptimizeRegAlloc;
+
+ // Enable interprocedural register allocation to reduce load/store at
+ // procedure calls
+ Optional<bool> EnableIPRA;
+
+ // Enable debug logging of pass pipeline
+ bool DebugPM = false;
+
+ // Disable machine function verification
+ bool DisableVerify = false;
+
+ // Fold null checks into faulting memory operations
+ bool EnableImplicitNullChecksPass = false;
+
+ // Collect probability-driven block placement stats
+ bool EnableMachineBlockPlacementStatsPass = false;
+
+ // Run MachineScheduler post regalloc (independent of preRA sched)
+ bool EnablePostMachineSchedulerPass = false;
+
+ // Run live interval analysis earlier in the pipeline
+ bool EnableLiveIntervalsPass = false;
+
+ // Disable Loop Strength Reduction Pass
+ bool DisableLoopStrengthReducePass = false;
+
+ // Disable Codegen Prepare
+ bool DisableCodeGenPreparePass = false;
+
+ // Disable MergeICmps Pass
+ bool DisableMergeICmpsPass = false;
+
+ // Disable Partial Libcall Inlining Pass
+ bool DisablePartiallyInlineLibCallsPass = false;
+
+ // Disable ConstantHoisting Pass
+ bool DisableConstantHoistingPass = false;
+
+ // Print LLVM IR produced by the loop-reduce pass
+ bool PrintAfterLSR = false;
+
+ // Print LLVM IR input to isel pass
+ bool PrintISelInput = false;
+
+ // Dump garbage collector data
+ bool PrintGCInfo = false;
+
+ // Enable codegen in SCC order.
+ bool RequiresCodeGenSCCOrder = false;
+
+ // Enable the machine outliner
+ RunOutliner EnableMachineOutliner = RunOutliner::TargetDefault;
+
+ // Register allocator to use
+ RegAllocType RegAlloc = RegAllocType::Default;
+
+ // Experimental option to use CFL-AA in codegen
+ CFLAAType UseCFLAA = CFLAAType::None;
+
+ // Enable abort calls when "global" instruction selection fails to
+ // lower/select an instruction
+ Optional<GlobalISelAbortMode> EnableGlobalISelAbort;
+
+ // Verify generated machine code"
+ Optional<bool> VerifyMachineCode;
+
+ // Enable the "fast" instruction selector
+ Optional<bool> EnableFastISelOption;
+
+ // Enable the "global" instruction selector
+ Optional<bool> EnableGlobalISelOption;
+};
+
+CGPassBuilderOption getCGPassBuilderOption();
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_CGPASSBUILDEROPTION_H
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
new file mode 100644
index 000000000000..0c679eb174b7
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -0,0 +1,1171 @@
+//===- Construction of codegen pass pipelines ------------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Interfaces for registering analysis passes, producing common pass manager
+/// configurations, and parsing of pass pipelines.
+///
+/// TODO: handle addRequiredID where, in legacy PM, one pass require other pass
+/// to run as prerequisite.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_CODEGENPASSBUILDER_H
+#define LLVM_CODEGEN_CODEGENPASSBUILDER_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/CFLAndersAliasAnalysis.h"
+#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
+#include "llvm/Analysis/ScopedNoAliasAA.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/CodeGen/CGPassBuilderOption.h"
+#include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/MIRPrinter.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
+#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCTargetOptions.h"
+#include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+#include <cassert>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace llvm {
+
+// FIXME: Dummy target independent passes definitions that have not yet been
+// ported to new pass manager. Once they do, remove these.
+#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
+ template <typename... Ts> PASS_NAME(Ts &&...) {} \
+ PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \
+ return PreservedAnalyses::all(); \
+ } \
+ };
+#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
+ template <typename... Ts> PASS_NAME(Ts &&...) {} \
+ PreservedAnalyses run(Module &, ModuleAnalysisManager &) { \
+ return PreservedAnalyses::all(); \
+ } \
+ };
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
+ template <typename... Ts> PASS_NAME(Ts &&...) {} \
+ Error run(Module &, MachineFunctionAnalysisManager &) { \
+ return Error::success(); \
+ } \
+ PreservedAnalyses run(MachineFunction &, \
+ MachineFunctionAnalysisManager &) { \
+ llvm_unreachable("this api is to make new PM api happy"); \
+ } \
+ static AnalysisKey Key; \
+ };
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
+ template <typename... Ts> PASS_NAME(Ts &&...) {} \
+ PreservedAnalyses run(MachineFunction &, \
+ MachineFunctionAnalysisManager &) { \
+ return PreservedAnalyses::all(); \
+ } \
+ static AnalysisKey Key; \
+ };
+#include "MachinePassRegistry.def"
+
+/// This class provides access to building LLVM's passes.
+///
+/// Its members provide the baseline state available to passes during their
+/// construction. The \c MachinePassRegistry.def file specifies how to construct
+/// all of the built-in passes, and those may reference these members during
+/// construction.
+template <typename DerivedT> class CodeGenPassBuilder {
+public:
+ explicit CodeGenPassBuilder(LLVMTargetMachine &TM, CGPassBuilderOption Opts,
+ PassInstrumentationCallbacks *PIC)
+ : TM(TM), Opt(Opts), PIC(PIC) {
+ // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
+ // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
+
+ // Target should override TM.Options.EnableIPRA in their target-specific
+ // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
+ if (Opt.EnableIPRA)
+ TM.Options.EnableIPRA = *Opt.EnableIPRA;
+
+ if (Opt.EnableGlobalISelAbort)
+ TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
+
+ if (!Opt.OptimizeRegAlloc)
+ Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOpt::None;
+
+ if (!Opt.VerifyMachineCode) {
+#ifdef EXPENSIVE_CHECKS
+ Opt.VerifyMachineCode = TM->isMachineVerifierClean();
+#else
+ Opt.VerifyMachineCode = false;
+#endif
+ }
+ }
+
+ Expected<std::pair<ModulePassManager, MachineFunctionPassManager>>
+ buildPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType) const;
+
+ void registerModuleAnalyses(ModuleAnalysisManager &) const;
+ void registerFunctionAnalyses(FunctionAnalysisManager &) const;
+ void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const;
+ std::pair<StringRef, bool> getPassNameFromLegacyName(StringRef) const;
+
+ void registerAnalyses(MachineFunctionAnalysisManager &MFAM) const {
+ registerModuleAnalyses(*MFAM.MAM);
+ registerFunctionAnalyses(*MFAM.FAM);
+ registerMachineFunctionAnalyses(MFAM);
+ }
+
+ PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
+ return PIC;
+ }
+
+protected:
+ template <typename PassT> using has_key_t = decltype(PassT::Key);
+
+ template <typename PassT>
+ using is_module_pass_t = decltype(std::declval<PassT &>().run(
+ std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
+
+ template <typename PassT>
+ using is_function_pass_t = decltype(std::declval<PassT &>().run(
+ std::declval<Function &>(), std::declval<FunctionAnalysisManager &>()));
+
+ // Function object to maintain state while adding codegen IR passes.
+ class AddIRPass {
+ public:
+ AddIRPass(bool DebugPM) : MPM(DebugPM), FPM(DebugPM) {
+ AddingFunctionPasses = false;
+ }
+
+ // Add Function Pass
+ template <typename PassT>
+ std::enable_if_t<is_detected<is_function_pass_t, PassT>::value>
+ operator()(PassT &&Pass) {
+ if (!AddingFunctionPasses)
+ AddingFunctionPasses = true;
+ FPM.addPass(std::forward<PassT>(Pass));
+ }
+
+ // Add Module Pass
+ template <typename PassT>
+ std::enable_if_t<is_detected<is_module_pass_t, PassT>::value &&
+ !is_detected<is_function_pass_t, PassT>::value>
+ operator()(PassT &&Pass) {
+ assert((!AddingFunctionPasses) &&
+ "could not add module pass after adding function pass");
+ MPM.addPass(std::forward<PassT>(Pass));
+ }
+
+ ModulePassManager releasePM() {
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ return std::move(MPM);
+ }
+
+ private:
+ ModulePassManager MPM;
+ FunctionPassManager FPM;
+ // The codegen IR pipeline are mostly function passes with the exceptions of
+ // a few loop and module passes. `AddingFunctionPasses` makes sure that
+ // we could only add module passes at the beginning of the pipeline. Once
+ // we begin adding function passes, we could no longer add module passes.
+ // This special-casing introduces less adaptor passes. If we have the need
+ // of adding module passes after function passes, we could change the
+ // implementation to accommodate that.
+ bool AddingFunctionPasses;
+ };
+
+ // Function object to maintain state while adding codegen machine passes.
+ class AddMachinePass {
+ public:
+ AddMachinePass(bool DebugPM, bool RequiresCodeGenSCCOrder,
+ bool VerifyMachineCode)
+ : PM(DebugPM, RequiresCodeGenSCCOrder, VerifyMachineCode) {}
+
+ template <typename PassT> void operator()(PassT &&Pass) {
+ static_assert(
+ is_detected<has_key_t, PassT>::value,
+ "Machine function pass must define a static member variable `Key`.");
+ for (auto &C : BeforeCallbacks) {
+ if (!C(&PassT::Key))
+ return;
+ }
+ PM.addPass(std::forward<PassT>(Pass));
+ for (auto &C : AfterCallbacks)
+ C(&PassT::Key);
+ }
+
+ template <typename PassT> void insertPass(AnalysisKey *ID, PassT Pass) {
+ AfterCallbacks.emplace_back(
+ [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) {
+ if (PassID == ID)
+ this->PM.addPass(std::move(Pass));
+ });
+ }
+
+ void disablePass(AnalysisKey *ID) {
+ BeforeCallbacks.emplace_back(
+ [ID](AnalysisKey *PassID) { return PassID != ID; });
+ }
+
+ MachineFunctionPassManager releasePM() { return std::move(PM); }
+
+ private:
+ MachineFunctionPassManager PM;
+ SmallVector<llvm::unique_function<bool(AnalysisKey *)>, 4> BeforeCallbacks;
+ SmallVector<llvm::unique_function<void(AnalysisKey *)>, 4> AfterCallbacks;
+ };
+
+ LLVMTargetMachine &TM;
+ CGPassBuilderOption Opt;
+ PassInstrumentationCallbacks *PIC;
+
+ /// Target override these hooks to parse target-specific analyses.
+ void registerTargetAnalysis(ModuleAnalysisManager &) const {}
+ void registerTargetAnalysis(FunctionAnalysisManager &) const {}
+ void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {}
+ std::pair<StringRef, bool> getTargetPassNameFromLegacyName(StringRef) const {
+ return {"", false};
+ }
+
+ template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
+ CodeGenOpt::Level getOptLevel() const { return TM.getOptLevel(); }
+
+ /// Check whether or not GlobalISel should abort on error.
+ /// When this is disabled, GlobalISel will fall back on SDISel instead of
+ /// erroring out.
+ bool isGlobalISelAbortEnabled() const {
+ return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
+ }
+
+ /// Check whether or not a diagnostic should be emitted when GlobalISel
+ /// uses the fallback path. In other words, it will emit a diagnostic
+ /// when GlobalISel failed and isGlobalISelAbortEnabled is false.
+ bool reportDiagnosticWhenGlobalISelFallback() const {
+ return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
+ }
+
+ /// addInstSelector - This method should install an instruction selector pass,
+ /// which converts from LLVM code to machine instructions.
+ Error addInstSelector(AddMachinePass &) const {
+ return make_error<StringError>("addInstSelector is not overridden",
+ inconvertibleErrorCode());
+ }
+
+ /// Add passes that optimize instruction level parallelism for out-of-order
+ /// targets. These passes are run while the machine code is still in SSA
+ /// form, so they can use MachineTraceMetrics to control their heuristics.
+ ///
+ /// All passes added here should preserve the MachineDominatorTree,
+ /// MachineLoopInfo, and MachineTraceMetrics analyses.
+ void addILPOpts(AddMachinePass &) const {}
+
+ /// This method may be implemented by targets that want to run passes
+ /// immediately before register allocation.
+ void addPreRegAlloc(AddMachinePass &) const {}
+
+ /// addPreRewrite - Add passes to the optimized register allocation pipeline
+ /// after register allocation is complete, but before virtual registers are
+ /// rewritten to physical registers.
+ ///
+ /// These passes must preserve VirtRegMap and LiveIntervals, and when running
+ /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
+ /// When these passes run, VirtRegMap contains legal physreg assignments for
+ /// all virtual registers.
+ ///
+ /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not
+ /// be honored. This is also not generally used for the the fast variant,
+ /// where the allocation and rewriting are done in one pass.
+ void addPreRewrite(AddMachinePass &) const {}
+
+ /// Add passes to be run immediately after virtual registers are rewritten
+ /// to physical registers.
+ void addPostRewrite(AddMachinePass &) const {}
+
+ /// This method may be implemented by targets that want to run passes after
+ /// register allocation pass pipeline but before prolog-epilog insertion.
+ void addPostRegAlloc(AddMachinePass &) const {}
+
+ /// This method may be implemented by targets that want to run passes after
+ /// prolog-epilog insertion and before the second instruction scheduling pass.
+ void addPreSched2(AddMachinePass &) const {}
+
+ /// This pass may be implemented by targets that want to run passes
+ /// immediately before machine code is emitted.
+ void addPreEmitPass(AddMachinePass &) const {}
+
+ /// Targets may add passes immediately before machine code is emitted in this
+ /// callback. This is called even later than `addPreEmitPass`.
+ // FIXME: Rename `addPreEmitPass` to something more sensible given its actual
+ // position and remove the `2` suffix here as this callback is what
+ // `addPreEmitPass` *should* be but in reality isn't.
+ void addPreEmitPass2(AddMachinePass &) const {}
+
+ /// {{@ For GlobalISel
+ ///
+
+ /// addPreISel - This method should add any "last minute" LLVM->LLVM
+ /// passes (which are run just before instruction selector).
+ void addPreISel(AddIRPass &) const {
+ llvm_unreachable("addPreISel is not overridden");
+ }
+
+ /// This method should install an IR translator pass, which converts from
+ /// LLVM code to machine instructions with possibly generic opcodes.
+ Error addIRTranslator(AddMachinePass &) const {
+ return make_error<StringError>("addIRTranslator is not overridden",
+ inconvertibleErrorCode());
+ }
+
+ /// This method may be implemented by targets that want to run passes
+ /// immediately before legalization.
+ void addPreLegalizeMachineIR(AddMachinePass &) const {}
+
+ /// This method should install a legalize pass, which converts the instruction
+ /// sequence into one that can be selected by the target.
+ Error addLegalizeMachineIR(AddMachinePass &) const {
+ return make_error<StringError>("addLegalizeMachineIR is not overridden",
+ inconvertibleErrorCode());
+ }
+
+ /// This method may be implemented by targets that want to run passes
+ /// immediately before the register bank selection.
+ void addPreRegBankSelect(AddMachinePass &) const {}
+
+ /// This method should install a register bank selector pass, which
+ /// assigns register banks to virtual registers without a register
+ /// class or register banks.
+ Error addRegBankSelect(AddMachinePass &) const {
+ return make_error<StringError>("addRegBankSelect is not overridden",
+ inconvertibleErrorCode());
+ }
+
+ /// This method may be implemented by targets that want to run passes
+ /// immediately before the (global) instruction selection.
+ void addPreGlobalInstructionSelect(AddMachinePass &) const {}
+
+ /// This method should install a (global) instruction selector pass, which
+ /// converts possibly generic instructions to fully target-specific
+ /// instructions, thereby constraining all generic virtual registers to
+ /// register classes.
+ Error addGlobalInstructionSelect(AddMachinePass &) const {
+ return make_error<StringError>(
+ "addGlobalInstructionSelect is not overridden",
+ inconvertibleErrorCode());
+ }
+ /// @}}
+
+ /// High level function that adds all passes necessary to go from llvm IR
+ /// representation to the MI representation.
+ /// Adds IR based lowering and target specific optimization passes and finally
+ /// the core instruction selection passes.
+ /// \returns true if an error occurred, false otherwise.
+ ModulePassManager addISelPasses() const;
+
+ /// Add the actual instruction selection passes. This does not include
+ /// preparation passes on IR.
+ Expected<AddMachinePass> addCoreISelPasses() const;
+
+ /// Add the complete, standard set of LLVM CodeGen passes.
+ /// Fully developed targets will not generally override this.
+ Error addMachinePasses(AddMachinePass &) const;
+
+ /// Add passes to lower exception handling for the code generator.
+ void addPassesToHandleExceptions(AddIRPass &) const;
+
+ /// Add common target configurable passes that perform LLVM IR to IR
+ /// transforms following machine independent optimization.
+ void addIRPasses(AddIRPass &) const;
+
+ /// Add pass to prepare the LLVM IR for code generation. This should be done
+ /// before exception handling preparation passes.
+ void addCodeGenPrepare(AddIRPass &) const;
+
+ /// Add common passes that perform LLVM IR to IR transforms in preparation for
+ /// instruction selection.
+ void addISelPrepare(AddIRPass &) const;
+
+ /// Methods with trivial inline returns are convenient points in the common
+ /// codegen pass pipeline where targets may insert passes. Methods with
+ /// out-of-line standard implementations are major CodeGen stages called by
+ /// addMachinePasses. Some targets may override major stages when inserting
+ /// passes is insufficient, but maintaining overriden stages is more work.
+ ///
+
+ /// addMachineSSAOptimization - Add standard passes that optimize machine
+ /// instructions in SSA form.
+ void addMachineSSAOptimization(AddMachinePass &) const;
+
+ /// addFastRegAlloc - Add the minimum set of target-independent passes that
+ /// are required for fast register allocation.
+ Error addFastRegAlloc(AddMachinePass &) const;
+
+ /// addOptimizedRegAlloc - Add passes related to register allocation.
+ /// LLVMTargetMachine provides standard regalloc passes for most targets.
+ void addOptimizedRegAlloc(AddMachinePass &) const;
+
+ /// Add passes that optimize machine instructions after register allocation.
+ void addMachineLateOptimization(AddMachinePass &) const;
+
+ /// addGCPasses - Add late codegen passes that analyze code for garbage
+ /// collection. This should return true if GC info should be printed after
+ /// these passes.
+ void addGCPasses(AddMachinePass &) const {}
+
+ /// Add standard basic block placement passes.
+ void addBlockPlacement(AddMachinePass &) const;
+
+ using CreateMCStreamer =
+ std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
+ void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
+ llvm_unreachable("addAsmPrinter is not overridden");
+ }
+
+ /// Utilities for targets to add passes to the pass manager.
+ ///
+
+ /// createTargetRegisterAllocator - Create the register allocator pass for
+ /// this target at the current optimization level.
+ void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const;
+
+ /// addMachinePasses helper to create the target-selected or overriden
+ /// regalloc pass.
+ void addRegAllocPass(AddMachinePass &, bool Optimized) const;
+
+ /// Add core register alloator passes which do the actual register assignment
+ /// and rewriting. \returns true if any passes were added.
+ Error addRegAssignmentFast(AddMachinePass &) const;
+ Error addRegAssignmentOptimized(AddMachinePass &) const;
+
+private:
+ DerivedT &derived() { return static_cast<DerivedT &>(*this); }
+ const DerivedT &derived() const {
+ return static_cast<const DerivedT &>(*this);
+ }
+};
+
+template <typename Derived>
+Expected<std::pair<ModulePassManager, MachineFunctionPassManager>>
+CodeGenPassBuilder<Derived>::buildPipeline(raw_pwrite_stream &Out,
+ raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType) const {
+ Expected<AddMachinePass> AddPassOrErr = addCoreISelPasses();
+ if (!AddPassOrErr)
+ return AddPassOrErr.takeError();
+
+ AddMachinePass &addPass = *AddPassOrErr;
+
+ if (auto Err = derived().addMachinePasses(addPass))
+ return std::move(Err);
+
+ derived().addAsmPrinter(
+ addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
+ return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
+ });
+
+ addPass(FreeMachineFunctionPass());
+
+ return std::pair<ModulePassManager, MachineFunctionPassManager>{
+ addISelPasses(), addPass.releasePM()};
+}
+
+static inline AAManager registerAAAnalyses(CFLAAType UseCFLAA) {
+ AAManager AA;
+
+ // The order in which these are registered determines their priority when
+ // being queried.
+
+ switch (UseCFLAA) {
+ case CFLAAType::Steensgaard:
+ AA.registerFunctionAnalysis<CFLSteensAA>();
+ break;
+ case CFLAAType::Andersen:
+ AA.registerFunctionAnalysis<CFLAndersAA>();
+ break;
+ case CFLAAType::Both:
+ AA.registerFunctionAnalysis<CFLAndersAA>();
+ AA.registerFunctionAnalysis<CFLSteensAA>();
+ break;
+ default:
+ break;
+ }
+
+ // Basic AliasAnalysis support.
+ // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
+ // BasicAliasAnalysis wins if they disagree. This is intended to help
+ // support "obvious" type-punning idioms.
+ AA.registerFunctionAnalysis<TypeBasedAA>();
+ AA.registerFunctionAnalysis<ScopedNoAliasAA>();
+ AA.registerFunctionAnalysis<BasicAA>();
+
+ return AA;
+}
+
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::registerModuleAnalyses(
+ ModuleAnalysisManager &MAM) const {
+#define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
+ MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
+#include "MachinePassRegistry.def"
+ derived().registerTargetAnalysis(MAM);
+}
+
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::registerFunctionAnalyses(
+ FunctionAnalysisManager &FAM) const {
+ FAM.registerPass([this] { return registerAAAnalyses(this->Opt.UseCFLAA); });
+
+#define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
+ FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
+#include "MachinePassRegistry.def"
+ derived().registerTargetAnalysis(FAM);
+}
+
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::registerMachineFunctionAnalyses(
+ MachineFunctionAnalysisManager &MFAM) const {
+#define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
+ MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
+#include "MachinePassRegistry.def"
+ derived().registerTargetAnalysis(MFAM);
+}
+
+// FIXME: For new PM, use pass name directly in commandline seems good.
+// Translate stringfied pass name to its old commandline name. Returns the
+// matching legacy name and a boolean value indicating if the pass is a machine
+// pass.
+template <typename Derived>
+std::pair<StringRef, bool>
+CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {
+ std::pair<StringRef, bool> Ret;
+ if (Name.empty())
+ return Ret;
+
+#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, false};
+#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, false};
+#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, false};
+#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, false};
+#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, true};
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, true};
+#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, true};
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ if (Name == NAME) \
+ Ret = {#PASS_NAME, true};
+#include "llvm/CodeGen/MachinePassRegistry.def"
+
+ if (Ret.first.empty())
+ Ret = derived().getTargetPassNameFromLegacyName(Name);
+
+ if (Ret.first.empty())
+ report_fatal_error(Twine('\"') + Twine(Name) +
+ Twine("\" pass could not be found."));
+
+ return Ret;
+}
+
+template <typename Derived>
+ModulePassManager CodeGenPassBuilder<Derived>::addISelPasses() const {
+ AddIRPass addPass(Opt.DebugPM);
+
+ if (TM.useEmulatedTLS())
+ addPass(LowerEmuTLSPass());
+
+ addPass(PreISelIntrinsicLoweringPass());
+
+ derived().addIRPasses(addPass);
+ derived().addCodeGenPrepare(addPass);
+ addPassesToHandleExceptions(addPass);
+ derived().addISelPrepare(addPass);
+ return addPass.releasePM();
+}
+
+/// Add common target configurable passes that perform LLVM IR to IR transforms
+/// following machine independent optimization.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
+ // Before running any passes, run the verifier to determine if the input
+ // coming from the front-end and/or optimizer is valid.
+ if (!Opt.DisableVerify)
+ addPass(VerifierPass());
+
+ // Run loop strength reduction before anything else.
+ if (getOptLevel() != CodeGenOpt::None && !Opt.DisableLoopStrengthReducePass) {
+ addPass(createFunctionToLoopPassAdaptor(
+ LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
+ // FIXME: use -stop-after so we could remove PrintAfterLSR
+ if (Opt.PrintAfterLSR)
+ addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
+ }
+
+ if (getOptLevel() != CodeGenOpt::None) {
+ // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
+ // loads and compares. ExpandMemCmpPass then tries to expand those calls
+ // into optimally-sized loads and compares. The transforms are enabled by a
+ // target lowering hook.
+ if (!Opt.DisableMergeICmpsPass)
+ addPass(MergeICmpsPass());
+ addPass(ExpandMemCmpPass());
+ }
+
+ // Run GC lowering passes for builtin collectors
+ // TODO: add a pass insertion point here
+ addPass(GCLoweringPass());
+ addPass(ShadowStackGCLoweringPass());
+ addPass(LowerConstantIntrinsicsPass());
+
+ // Make sure that no unreachable blocks are instruction selected.
+ addPass(UnreachableBlockElimPass());
+
+ // Prepare expensive constants for SelectionDAG.
+ if (getOptLevel() != CodeGenOpt::None && !Opt.DisableConstantHoistingPass)
+ addPass(ConstantHoistingPass());
+
+ if (getOptLevel() != CodeGenOpt::None &&
+ !Opt.DisablePartiallyInlineLibCallsPass)
+ addPass(PartiallyInlineLibCallsPass());
+
+ // Instrument function entry and exit, e.g. with calls to mcount().
+ addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
+
+ // Add scalarization of target's unsupported masked memory intrinsics pass.
+ // the unsupported intrinsic will be replaced with a chain of basic blocks,
+ // that stores/loads element one-by-one if the appropriate mask bit is set.
+ addPass(ScalarizeMaskedMemIntrinPass());
+
+ // Expand reduction intrinsics into shuffle sequences if the target wants to.
+ addPass(ExpandReductionsPass());
+}
+
+/// Turn exception handling constructs into something the code generators can
+/// handle.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
+ AddIRPass &addPass) const {
+ const MCAsmInfo *MCAI = TM.getMCAsmInfo();
+ assert(MCAI && "No MCAsmInfo");
+ switch (MCAI->getExceptionHandlingType()) {
+ case ExceptionHandling::SjLj:
+ // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
+ // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
+ // catch info can get misplaced when a selector ends up more than one block
+ // removed from the parent invoke(s). This could happen when a landing
+ // pad is shared by multiple invokes and is also a target of a normal
+ // edge from elsewhere.
+ addPass(SjLjEHPreparePass());
+ LLVM_FALLTHROUGH;
+ case ExceptionHandling::DwarfCFI:
+ case ExceptionHandling::ARM:
+ addPass(DwarfEHPass());
+ break;
+ case ExceptionHandling::WinEH:
+ // We support using both GCC-style and MSVC-style exceptions on Windows, so
+ // add both preparation passes. Each pass will only actually run if it
+ // recognizes the personality function.
+ addPass(WinEHPass());
+ addPass(DwarfEHPass());
+ break;
+ case ExceptionHandling::Wasm:
+ // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
+ // on catchpads and cleanuppads because it does not outline them into
+ // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
+ // should remove PHIs there.
+ addPass(WinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
+ addPass(WasmEHPass());
+ break;
+ case ExceptionHandling::None:
+ addPass(LowerInvokePass());
+
+ // The lower invoke pass may create unreachable code. Remove it.
+ addPass(UnreachableBlockElimPass());
+ break;
+ }
+}
+
+/// Add pass to prepare the LLVM IR for code generation. This should be done
+/// before exception handling preparation passes.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
+ if (getOptLevel() != CodeGenOpt::None && !Opt.DisableCodeGenPreparePass)
+ addPass(CodeGenPreparePass());
+ // TODO: Default ctor'd RewriteSymbolPass is no-op.
+ // addPass(RewriteSymbolPass());
+}
+
+/// Add common passes that perform LLVM IR to IR transforms in preparation for
+/// instruction selection.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
+ derived().addPreISel(addPass);
+
+ // Add both the safe stack and the stack protection passes: each of them will
+ // only protect functions that have corresponding attributes.
+ addPass(SafeStackPass());
+ addPass(StackProtectorPass());
+
+ if (Opt.PrintISelInput)
+ addPass(PrintFunctionPass(dbgs(),
+ "\n\n*** Final LLVM Code input to ISel ***\n"));
+
+ // All passes which modify the LLVM IR are now complete; run the verifier
+ // to ensure that the IR is valid.
+ if (!Opt.DisableVerify)
+ addPass(VerifierPass());
+}
+
+template <typename Derived>
+Expected<typename CodeGenPassBuilder<Derived>::AddMachinePass>
+CodeGenPassBuilder<Derived>::addCoreISelPasses() const {
+ // Enable FastISel with -fast-isel, but allow that to be overridden.
+ TM.setO0WantsFastISel(Opt.EnableFastISelOption.getValueOr(true));
+
+ // Determine an instruction selector.
+ enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+ SelectorType Selector;
+
+ if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true)
+ Selector = SelectorType::FastISel;
+ else if ((Opt.EnableGlobalISelOption &&
+ *Opt.EnableGlobalISelOption == true) ||
+ (TM.Options.EnableGlobalISel &&
+ (!Opt.EnableGlobalISelOption ||
+ *Opt.EnableGlobalISelOption == false)))
+ Selector = SelectorType::GlobalISel;
+ else if (TM.getOptLevel() == CodeGenOpt::None && TM.getO0WantsFastISel())
+ Selector = SelectorType::FastISel;
+ else
+ Selector = SelectorType::SelectionDAG;
+
+ // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
+ if (Selector == SelectorType::FastISel) {
+ TM.setFastISel(true);
+ TM.setGlobalISel(false);
+ } else if (Selector == SelectorType::GlobalISel) {
+ TM.setFastISel(false);
+ TM.setGlobalISel(true);
+ }
+
+ AddMachinePass addPass(Opt.DebugPM, Opt.RequiresCodeGenSCCOrder,
+ *Opt.VerifyMachineCode);
+
+ // Add instruction selector passes.
+ if (Selector == SelectorType::GlobalISel) {
+ if (auto Err = derived().addIRTranslator(addPass))
+ return std::move(Err);
+
+ derived().addPreLegalizeMachineIR(addPass);
+
+ if (auto Err = derived().addLegalizeMachineIR(addPass))
+ return std::move(Err);
+
+ // Before running the register bank selector, ask the target if it
+ // wants to run some passes.
+ derived().addPreRegBankSelect(addPass);
+
+ if (auto Err = derived().addRegBankSelect(addPass))
+ return std::move(Err);
+
+ derived().addPreGlobalInstructionSelect(addPass);
+
+ if (auto Err = derived().addGlobalInstructionSelect(addPass))
+ return std::move(Err);
+
+ // Pass to reset the MachineFunction if the ISel failed.
+ addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
+ isGlobalISelAbortEnabled()));
+
+ // Provide a fallback path when we do not want to abort on
+ // not-yet-supported input.
+ if (!isGlobalISelAbortEnabled()) {
+ if (auto Err = derived().addInstSelector(addPass))
+ return std::move(Err);
+ }
+
+ } else if (auto Err = derived().addInstSelector(addPass))
+ return std::move(Err);
+
+ // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
+ // FinalizeISel.
+ addPass(FinalizeISelPass());
+
+ return addPass;
+}
+
+/// Add the complete set of target-independent postISel code generator passes.
+///
+/// This can be read as the standard order of major LLVM CodeGen stages. Stages
+/// with nontrivial configuration or multiple passes are broken out below in
+/// add%Stage routines.
+///
+/// Any CodeGenPassBuilder<Derived>::addXX routine may be overriden by the
+/// Target. The addPre/Post methods with empty header implementations allow
+/// injecting target-specific fixups just before or after major stages.
+/// Additionally, targets have the flexibility to change pass order within a
+/// stage by overriding default implementation of add%Stage routines below. Each
+/// technique has maintainability tradeoffs because alternate pass orders are
+/// not well supported. addPre/Post works better if the target pass is easily
+/// tied to a common pass. But if it has subtle dependencies on multiple passes,
+/// the target should override the stage instead.
+template <typename Derived>
+Error CodeGenPassBuilder<Derived>::addMachinePasses(
+ AddMachinePass &addPass) const {
+ // Add passes that optimize machine instructions in SSA form.
+ if (getOptLevel() != CodeGenOpt::None) {
+ derived().addMachineSSAOptimization(addPass);
+ } else {
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ addPass(LocalStackSlotPass());
+ }
+
+ if (TM.Options.EnableIPRA)
+ addPass(RegUsageInfoPropagationPass());
+
+ // Run pre-ra passes.
+ derived().addPreRegAlloc(addPass);
+
+ // Run register allocation and passes that are tightly coupled with it,
+ // including phi elimination and scheduling.
+ if (*Opt.OptimizeRegAlloc) {
+ derived().addOptimizedRegAlloc(addPass);
+ } else {
+ if (auto Err = derived().addFastRegAlloc(addPass))
+ return Err;
+ }
+
+ // Run post-ra passes.
+ derived().addPostRegAlloc(addPass);
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ if (getOptLevel() != CodeGenOpt::None) {
+ addPass(PostRAMachineSinkingPass());
+ addPass(ShrinkWrapPass());
+ }
+
+ addPass(PrologEpilogInserterPass());
+
+ /// Add passes that optimize machine instructions after register allocation.
+ if (getOptLevel() != CodeGenOpt::None)
+ derived().addMachineLateOptimization(addPass);
+
+ // Expand pseudo instructions before second scheduling pass.
+ addPass(ExpandPostRAPseudosPass());
+
+ // Run pre-sched2 passes.
+ derived().addPreSched2(addPass);
+
+ if (Opt.EnableImplicitNullChecksPass)
+ addPass(ImplicitNullChecksPass());
+
+ // Second pass scheduler.
+ // Let Target optionally insert this pass by itself at some other point.
+ if (getOptLevel() != CodeGenOpt::None &&
+ !TM.targetSchedulesPostRAScheduling()) {
+ if (Opt.EnablePostMachineSchedulerPass)
+ addPass(PostMachineSchedulerPass());
+ else
+ addPass(PostRASchedulerPass());
+ }
+
+ // GC
+ derived().addGCPasses(addPass);
+
+ // Basic block placement.
+ if (getOptLevel() != CodeGenOpt::None)
+ derived().addBlockPlacement(addPass);
+
+ // Insert before XRay Instrumentation.
+ addPass(FEntryInserterPass());
+
+ addPass(XRayInstrumentationPass());
+ addPass(PatchableFunctionPass());
+
+ derived().addPreEmitPass(addPass);
+
+ if (TM.Options.EnableIPRA) {
+ // Collect register usage information and produce a register mask of
+ // clobbered registers, to be used to optimize call sites.
+ addPass(RegUsageInfoCollectorPass());
+ }
+
+ addPass(FuncletLayoutPass());
+
+ addPass(StackMapLivenessPass());
+ addPass(LiveDebugValuesPass());
+
+ if (TM.Options.EnableMachineOutliner && getOptLevel() != CodeGenOpt::None &&
+ Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining;
+ if (AddOutliner)
+ addPass(MachineOutlinerPass(RunOnAllFunctions));
+ }
+
+ // Add passes that directly emit MI after all other MI passes.
+ derived().addPreEmitPass2(addPass);
+
+ return Error::success();
+}
+
+/// Add passes that optimize machine instructions in SSA form.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
+ AddMachinePass &addPass) const {
+ // Pre-ra tail duplication.
+ addPass(EarlyTailDuplicatePass());
+
+ // Optimize PHIs before DCE: removing dead PHI cycles may make more
+ // instructions dead.
+ addPass(OptimizePHIsPass());
+
+ // This pass merges large allocas. StackSlotColoring is a
diff erent pass
+ // which merges spill slots.
+ addPass(StackColoringPass());
+
+ // If the target requests it, assign local variables to stack slots relative
+ // to one another and simplify frame index references where possible.
+ addPass(LocalStackSlotPass());
+
+ // With optimization, dead code should already be eliminated. However
+ // there is one known exception: lowered code for arguments that are only
+ // used by tail calls, where the tail calls reuse the incoming stack
+ // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+ addPass(DeadMachineInstructionElimPass());
+
+ // Allow targets to insert passes that improve instruction level parallelism,
+ // like if-conversion. Such passes will typically need dominator trees and
+ // loop info, just like LICM and CSE below.
+ derived().addILPOpts(addPass);
+
+ addPass(EarlyMachineLICMPass());
+ addPass(MachineCSEPass());
+
+ addPass(MachineSinkingPass());
+
+ addPass(PeepholeOptimizerPass());
+ // Clean-up the dead code that may have been generated by peephole
+ // rewriting.
+ addPass(DeadMachineInstructionElimPass());
+}
+
+//===---------------------------------------------------------------------===//
+/// Register Allocation Pass Configuration
+//===---------------------------------------------------------------------===//
+
+/// Instantiate the default register allocator pass for this target for either
+/// the optimized or unoptimized allocation path. This will be added to the pass
+/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
+/// in the optimized case.
+///
+/// A target that uses the standard regalloc pass order for fast or optimized
+/// allocation may still override this for per-target regalloc
+/// selection. But -regalloc=... always takes precedence.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addTargetRegisterAllocator(
+ AddMachinePass &addPass, bool Optimized) const {
+ if (Optimized)
+ addPass(RAGreedyPass());
+ else
+ addPass(RAFastPass());
+}
+
+/// Find and instantiate the register allocation pass requested by this target
+/// at the current optimization level. Different register allocators are
+/// defined as separate passes because they may require
diff erent analysis.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
+ bool Optimized) const {
+ switch (Opt.RegAlloc) {
+ case RegAllocType::Default:
+ // With no -regalloc= override, ask the target for a regalloc pass.
+ derived().addTargetRegisterAllocator(addPass, Optimized);
+ break;
+ case RegAllocType::Basic:
+ addPass(RABasicPass());
+ break;
+ case RegAllocType::Fast:
+ addPass(RAFastPass());
+ break;
+ case RegAllocType::Greedy:
+ addPass(RAGreedyPass());
+ break;
+ case RegAllocType::PBQP:
+ addPass(RAPBQPPass());
+ break;
+ default:
+ llvm_unreachable("unknonwn register allocator type");
+ }
+}
+
+template <typename Derived>
+Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
+ AddMachinePass &addPass) const {
+ if (Opt.RegAlloc != RegAllocType::Default &&
+ Opt.RegAlloc != RegAllocType::Fast)
+ return make_error<StringError>(
+ "Must use fast (default) register allocator for unoptimized regalloc.",
+ inconvertibleErrorCode());
+
+ addRegAllocPass(addPass, false);
+ return Error::success();
+}
+
+template <typename Derived>
+Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
+ AddMachinePass &addPass) const {
+ // Add the selected register allocation pass.
+ addRegAllocPass(addPass, true);
+
+ // Allow targets to change the register assignments before rewriting.
+ derived().addPreRewrite(addPass);
+
+ // Finally rewrite virtual registers.
+ addPass(VirtRegRewriterPass());
+ // Perform stack slot coloring and post-ra machine LICM.
+ //
+ // FIXME: Re-enable coloring with register when it's capable of adding
+ // kill markers.
+ addPass(StackSlotColoringPass());
+
+ return Error::success();
+}
+
+/// Add the minimum set of target-independent passes that are required for
+/// register allocation. No coalescing or scheduling.
+template <typename Derived>
+Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
+ AddMachinePass &addPass) const {
+ addPass(PHIEliminationPass());
+ addPass(TwoAddressInstructionPass());
+ return derived().addRegAssignmentFast(addPass);
+}
+
+/// Add standard target-independent passes that are tightly coupled with
+/// optimized register allocation, including coalescing, machine instruction
+/// scheduling, and register allocation itself.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
+ AddMachinePass &addPass) const {
+ addPass(DetectDeadLanesPass());
+
+ addPass(ProcessImplicitDefsPass());
+
+ // Edge splitting is smarter with machine loop info.
+ addPass(PHIEliminationPass());
+
+ // Eventually, we want to run LiveIntervals before PHI elimination.
+ if (Opt.EnableLiveIntervalsPass)
+ addPass(LiveIntervalsPass());
+
+ addPass(TwoAddressInstructionPass());
+ addPass(RegisterCoalescerPass());
+
+ // The machine scheduler may accidentally create disconnected components
+ // when moving subregister definitions around, avoid this by splitting them to
+ // separate vregs before. Splitting can also improve reg. allocation quality.
+ addPass(RenameIndependentSubregsPass());
+
+ // PreRA instruction scheduling.
+ addPass(MachineSchedulerPass());
+
+ if (derived().addRegAssignmentOptimized(addPass)) {
+ // Allow targets to expand pseudo instructions depending on the choice of
+ // registers before MachineCopyPropagation.
+ derived().addPostRewrite(addPass);
+
+ // Copy propagate to forward register uses and try to eliminate COPYs that
+ // were not coalesced.
+ addPass(MachineCopyPropagationPass());
+
+ // Run post-ra machine LICM to hoist reloads / remats.
+ //
+ // FIXME: can this move into MachineLateOptimization?
+ addPass(MachineLICMPass());
+ }
+}
+
+//===---------------------------------------------------------------------===//
+/// Post RegAlloc Pass Configuration
+//===---------------------------------------------------------------------===//
+
+/// Add passes that optimize machine instructions after register allocation.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addMachineLateOptimization(
+ AddMachinePass &addPass) const {
+ // Branch folding must be run after regalloc and prolog/epilog insertion.
+ addPass(BranchFolderPass());
+
+ // Tail duplication.
+ // Note that duplicating tail just increases code size and degrades
+ // performance for targets that require Structured Control Flow.
+ // In addition it can also make CFG irreducible. Thus we disable it.
+ if (!TM.requiresStructuredCFG())
+ addPass(TailDuplicatePass());
+
+ // Copy propagation.
+ addPass(MachineCopyPropagationPass());
+}
+
+/// Add standard basic block placement passes.
+template <typename Derived>
+void CodeGenPassBuilder<Derived>::addBlockPlacement(
+ AddMachinePass &addPass) const {
+ addPass(MachineBlockPlacementPass());
+ // Run a separate pass to collect block placement statistics.
+ if (Opt.EnableMachineBlockPlacementStatsPass)
+ addPass(MachineBlockPlacementStatsPass());
+}
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_CODEGENPASSBUILDER_H
diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
new file mode 100644
index 000000000000..734bbebc76de
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
@@ -0,0 +1,195 @@
+//===- MachinePassRegistry.def - Registry of passes -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are for target-independent
+// code generator.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef MODULE_ANALYSIS
+#define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
+#undef MODULE_ANALYSIS
+
+#ifndef MODULE_PASS
+#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ())
+#undef MODULE_PASS
+
+#ifndef FUNCTION_ANALYSIS
+#define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
+FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, (std::move(TM.getTargetIRAnalysis())))
+#undef FUNCTION_ANALYSIS
+
+#ifndef FUNCTION_PASS
+#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+FUNCTION_PASS("mergeicmps", MergeICmpsPass, ())
+FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ())
+FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass, ())
+FUNCTION_PASS("consthoist", ConstantHoistingPass, ())
+FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass, ())
+FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false))
+FUNCTION_PASS("post-inline-ee-instrument", EntryExitInstrumenterPass, (true))
+FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
+FUNCTION_PASS("lowerinvoke", LowerInvokePass, ())
+FUNCTION_PASS("verify", VerifierPass, ())
+#undef FUNCTION_PASS
+
+#ifndef LOOP_PASS
+#define LOOP_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+LOOP_PASS("loop-reduce", LoopStrengthReducePass, ())
+#undef LOOP_PASS
+
+#ifndef MACHINE_MODULE_PASS
+#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+#undef MACHINE_MODULE_PASS
+
+#ifndef MACHINE_FUNCTION_ANALYSIS
+#define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
+// LiveVariables currently requires pure SSA form.
+// FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+// LiveVariables can be removed completely, and LiveIntervals can be directly
+// computed. (We still either need to regenerate kill flags after regalloc, or
+// preferably fix the scavenger to not depend on them).
+// MACHINE_FUNCTION_ANALYSIS("live-vars", LiveVariablesAnalysis())
+
+// MACHINE_FUNCTION_ANALYSIS("live-stacks", LiveStacksPass())
+// MACHINE_FUNCTION_ANALYSIS("slot-indexes", SlotIndexesAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("edge-bundles", EdgeBundlesAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("lazy-machine-bfi", LazyMachineBlockFrequencyInfoAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-bfi", MachineBlockFrequencyInfoAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-loops", MachineLoopInfoAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-dom-frontier", MachineDominanceFrontierAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-dom-tree", MachineDominatorTreeAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-ore", MachineOptimizationRemarkEmitterPassAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-post-dom-tree", MachinePostDominatorTreeAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-region-info", MachineRegionInfoPassAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("machine-trace-metrics", MachineTraceMetricsAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("reaching-def", ReachingDefAnalysisAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("live-reg-matrix", LiveRegMatrixAnalysis())
+// MACHINE_FUNCTION_ANALYSIS("gc-analysis", GCMachineCodeAnalysisPass())
+#undef MACHINE_FUNCTION_ANALYSIS
+
+#ifndef MACHINE_FUNCTION_PASS
+#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+MACHINE_FUNCTION_PASS("mir-printer", PrintMIRPass, ())
+#undef MACHINE_FUNCTION_PASS
+
+// After a pass is converted to new pass manager, its entry should be moved from
+// dummy table to the normal one. For example, for a machine function pass,
+// DUMMY_MACHINE_FUNCTION_PASS to MACHINE_FUNCTION_PASS.
+
+#ifndef DUMMY_FUNCTION_PASS
+#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ())
+DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
+DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
+DUMMY_FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass, ())
+DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ())
+DUMMY_FUNCTION_PASS("dwarfehprepare", DwarfEHPass, ())
+DUMMY_FUNCTION_PASS("winehprepare", WinEHPass, ())
+DUMMY_FUNCTION_PASS("wasmehprepare", WasmEHPass, ())
+DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ())
+DUMMY_FUNCTION_PASS("safe-stack", SafeStackPass, ())
+DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ())
+DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ())
+DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ())
+DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
+DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ())
+DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ())
+DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ())
+#undef DUMMY_FUNCTION_PASS
+
+#ifndef DUMMY_MODULE_PASS
+#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+DUMMY_MODULE_PASS("lower-emutls", LowerEmuTLSPass, ())
+#undef DUMMY_MODULE_PASS
+
+#ifndef DUMMY_MACHINE_MODULE_PASS
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+DUMMY_MACHINE_MODULE_PASS("machine-outliner", MachineOutlinerPass, ())
+#undef DUMMY_MACHINE_MODULE_PASS
+
+#ifndef DUMMY_MACHINE_FUNCTION_PASS
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
+#endif
+DUMMY_MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("shrink-wrap", ShrinkWrapPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("prologepilog", PrologEpilogInserterPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("postrapseudos", ExpandPostRAPseudosPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("postmisched", PostMachineSchedulerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-scheduler", MachineSchedulerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-cp", MachineCopyPropagationPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("post-RA-sched", PostRASchedulerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("fentry-insert", FEntryInserterPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("patchable-function", PatchableFunctionPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("reg-usage-propagation", RegUsageInfoPropagationPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("livedebugvalues", LiveDebugValuesPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass, ())
+DUMMY_MACHINE_FUNCTION_PASS("opt-phis", OptimizePHIsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machinelicm", MachineLICMPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-cse", MachineCSEPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-sink", MachineSinkingPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("postra-machine-sink", PostRAMachineSinkingPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("peephole-opt", PeepholeOptimizerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("regalloc", RegAllocPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("phi-node-elimination", PHIEliminationPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("twoaddressinstruction", TwoAddressInstructionPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("liveintervals", LiveIntervalsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("simple-register-coalescing", RegisterCoalescerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("rename-independent-subregs", RenameIndependentSubregsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("branch-folder", BranchFolderPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass, ())
+DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("block-placement-stats", MachineBlockPlacementStatsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("early-ifcvt", EarlyIfConverterPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-combiner", MachineCombinerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("lrshrink", LiveRangeShrinkPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("break-false-deps", BreakFalseDepsPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("cfi-instr-inserter", CFIInstrInserterPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("cfguard-longjmp", CFGuardLongjmpPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("ra-basic", RABasicPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("ra-fast", RAFastPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("ra-greedy", RAGreedyPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("ra-pbqp", RAPBQPPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("legalizer", LegalizerPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("irtranslator", IRTranslatorPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machineverifier", MachineVerifierPass, ())
+#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 76e217c89974..457eae26fd47 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -28,6 +28,7 @@
namespace llvm {
+class LLVMTargetMachine;
class Module;
class Function;
@@ -140,6 +141,10 @@ class StandardInstrumentations {
TimePassesHandler &getTimePasses() { return TimePasses; }
};
+
+void registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
+ LLVMTargetMachine &);
+
} // namespace llvm
#endif
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 2a422341fdc8..c7673d3e74e4 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -15,9 +15,12 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/CodeGen/CGPassBuilderOption.h"
+#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/Pass.h"
#include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Error.h"
#include "llvm/Target/TargetOptions.h"
#include <string>
@@ -367,6 +370,20 @@ class LLVMTargetMachine : public TargetMachine {
bool DisableVerify = true,
MachineModuleInfoWrapperPass *MMIWP = nullptr) override;
+ virtual Expected<std::pair<ModulePassManager, MachineFunctionPassManager>>
+ buildCodeGenPipeline(raw_pwrite_stream &, raw_pwrite_stream *,
+ CodeGenFileType, CGPassBuilderOption,
+ MachineFunctionAnalysisManager &,
+ PassInstrumentationCallbacks *) {
+ return make_error<StringError>("buildCodeGenPipeline is not overriden",
+ inconvertibleErrorCode());
+ }
+
+ virtual std::pair<StringRef, bool> getPassNameFromLegacyName(StringRef) {
+ llvm_unreachable(
+ "getPassNameFromLegacyName parseMIRPipeline is not overriden");
+ }
+
/// Add passes to the specified pass manager to get machine code emitted with
/// the MCJIT. This method returns true if machine code is not supported. It
/// fills the MCContext Ctx pointer which can be used to build custom
@@ -387,6 +404,10 @@ class LLVMTargetMachine : public TargetMachine {
raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
MCContext &Context);
+ Expected<std::unique_ptr<MCStreamer>>
+ createMCStreamer(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+ CodeGenFileType FileType, MCContext &Ctx);
+
/// True if the target uses physical regs (as nearly all targets do). False
/// for stack machines such as WebAssembly and other virtual-register
/// machines. If true, all vregs must be allocated before PEI. If false, then
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 617692a34792..83b3655441fe 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -14,6 +14,7 @@ add_llvm_component_library(LLVMCodeGen
CFGuardLongjmp.cpp
CFIInstrInserter.cpp
CodeGen.cpp
+ CodeGenPassBuilder.cpp
CodeGenPrepare.cpp
CommandFlags.cpp
CriticalAntiDepBreaker.cpp
diff --git a/llvm/lib/CodeGen/CodeGenPassBuilder.cpp b/llvm/lib/CodeGen/CodeGenPassBuilder.cpp
new file mode 100644
index 000000000000..7f37f2069a3b
--- /dev/null
+++ b/llvm/lib/CodeGen/CodeGenPassBuilder.cpp
@@ -0,0 +1,25 @@
+//===--- CodeGenPassBuilder.cpp --------------------------------------- ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines interfaces to access the target independent code
+// generation passes provided by the LLVM backend.
+//
+//===---------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/CodeGenPassBuilder.h"
+
+using namespace llvm;
+
+namespace llvm {
+#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ AnalysisKey PASS_NAME::Key;
+#include "llvm/CodeGen/MachinePassRegistry.def"
+#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
+ AnalysisKey PASS_NAME::Key;
+#include "llvm/CodeGen/MachinePassRegistry.def"
+} // namespace llvm
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index e94b7ed4de03..e86f25512999 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -118,6 +118,24 @@ bool LLVMTargetMachine::addAsmPrinter(PassManagerBase &PM,
raw_pwrite_stream *DwoOut,
CodeGenFileType FileType,
MCContext &Context) {
+ Expected<std::unique_ptr<MCStreamer>> MCStreamerOrErr =
+ createMCStreamer(Out, DwoOut, FileType, Context);
+ if (auto Err = MCStreamerOrErr.takeError())
+ return true;
+
+ // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
+ FunctionPass *Printer =
+ getTarget().createAsmPrinter(*this, std::move(*MCStreamerOrErr));
+ if (!Printer)
+ return true;
+
+ PM.add(Printer);
+ return false;
+}
+
+Expected<std::unique_ptr<MCStreamer>> LLVMTargetMachine::createMCStreamer(
+ raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+ MCContext &Context) {
if (Options.MCOptions.MCSaveTempLabels)
Context.setAllowTemporaryLabels(false);
@@ -152,10 +170,14 @@ bool LLVMTargetMachine::addAsmPrinter(PassManagerBase &PM,
// Create the code emitter for the target if it exists. If not, .o file
// emission fails.
MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(MII, MRI, Context);
+ if (!MCE)
+ return make_error<StringError>("createMCCodeEmitter failed",
+ inconvertibleErrorCode());
MCAsmBackend *MAB =
getTarget().createMCAsmBackend(STI, MRI, Options.MCOptions);
- if (!MCE || !MAB)
- return true;
+ if (!MAB)
+ return make_error<StringError>("createMCAsmBackend failed",
+ inconvertibleErrorCode());
Triple T(getTargetTriple().str());
AsmStreamer.reset(getTarget().createMCObjectStreamer(
@@ -174,14 +196,7 @@ bool LLVMTargetMachine::addAsmPrinter(PassManagerBase &PM,
break;
}
- // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
- FunctionPass *Printer =
- getTarget().createAsmPrinter(*this, std::move(AsmStreamer));
- if (!Printer)
- return true;
-
- PM.add(Printer);
- return false;
+ return std::move(AsmStreamer);
}
bool LLVMTargetMachine::addPassesToEmitFile(
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 19db8eb480ca..03a567e3d443 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -22,6 +22,7 @@
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/CodeGen/CGPassBuilderOption.h"
#include "llvm/CodeGen/CSEConfigBase.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachinePassRegistry.h"
@@ -29,11 +30,13 @@
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/IR/IRPrintingPasses.h"
#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/PassInstrumentation.h"
#include "llvm/IR/Verifier.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Pass.h"
+#include "llvm/Passes/StandardInstrumentations.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
@@ -120,16 +123,17 @@ static cl::opt<cl::boolOrDefault> DebugifyAndStripAll(
"Debugify MIR before and Strip debug after "
"each pass except those known to be unsafe when debug info is present"),
cl::ZeroOrMore);
-enum RunOutliner { AlwaysOutline, NeverOutline, TargetDefault };
+
// Enable or disable the MachineOutliner.
static cl::opt<RunOutliner> EnableMachineOutliner(
"enable-machine-outliner", cl::desc("Enable the machine outliner"),
- cl::Hidden, cl::ValueOptional, cl::init(TargetDefault),
- cl::values(clEnumValN(AlwaysOutline, "always",
+ cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),
+ cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always",
"Run on all functions guaranteed to be beneficial"),
- clEnumValN(NeverOutline, "never", "Disable all outlining"),
+ clEnumValN(RunOutliner::NeverOutline, "never",
+ "Disable all outlining"),
// Sentinel value for unspecified option.
- clEnumValN(AlwaysOutline, "", "")));
+ clEnumValN(RunOutliner::AlwaysOutline, "", "")));
// Enable or disable FastISel. Both options are needed, because
// FastISel is enabled by default with -fast, and we wish to be
// able to enable or disable fast-isel independently from -O0.
@@ -172,7 +176,6 @@ static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
cl::desc("Run live interval analysis earlier in the pipeline"));
// Experimental option to use CFL-AA in codegen
-enum class CFLAAType { None, Steensgaard, Andersen, Both };
static cl::opt<CFLAAType> UseCFLAA(
"use-cfl-aa-in-codegen", cl::init(CFLAAType::None), cl::Hidden,
cl::desc("Enable the new, experimental CFL alias analysis in CodeGen"),
@@ -404,6 +407,143 @@ void TargetPassConfig::setStartStopPasses() {
Started = (StartAfter == nullptr) && (StartBefore == nullptr);
}
+CGPassBuilderOption llvm::getCGPassBuilderOption() {
+ CGPassBuilderOption Opt;
+
+#define SET_OPTION(Option) \
+ if (Option.getNumOccurrences()) \
+ Opt.Option = Option;
+
+ SET_OPTION(EnableFastISelOption)
+ SET_OPTION(EnableGlobalISelAbort)
+ SET_OPTION(EnableGlobalISelOption)
+ SET_OPTION(EnableIPRA)
+ SET_OPTION(OptimizeRegAlloc)
+ SET_OPTION(VerifyMachineCode)
+
+ Opt.EnableMachineOutliner = EnableMachineOutliner;
+ Opt.UseCFLAA = UseCFLAA;
+ Opt.PrintISelInput = PrintISelInput;
+ Opt.PrintGCInfo = PrintGCInfo;
+ Opt.EnablePostMachineSchedulerPass = MISchedPostRA;
+ Opt.EnableLiveIntervalsPass = EarlyLiveIntervals;
+ Opt.EnableMachineBlockPlacementStatsPass = EnableBlockPlacementStats;
+ Opt.EnableImplicitNullChecksPass = EnableImplicitNullChecks;
+ Opt.DisableLoopStrengthReducePass = DisableLSR;
+ Opt.DisableCodeGenPreparePass = DisableCGP;
+ Opt.DisableMergeICmpsPass = DisableMergeICmps;
+ Opt.DisablePartiallyInlineLibCallsPass = DisablePartialLibcallInlining;
+ Opt.DisableConstantHoistingPass = DisableConstantHoisting;
+ Opt.PrintAfterLSR = PrintLSR;
+
+ return Opt;
+}
+
+static void registerPartialPipelineCallback(PassInstrumentationCallbacks &PIC,
+ LLVMTargetMachine &LLVMTM) {
+ StringRef StartBefore;
+ StringRef StartAfter;
+ StringRef StopBefore;
+ StringRef StopAfter;
+
+ unsigned StartBeforeInstanceNum = 0;
+ unsigned StartAfterInstanceNum = 0;
+ unsigned StopBeforeInstanceNum = 0;
+ unsigned StopAfterInstanceNum = 0;
+
+ std::tie(StartBefore, StartBeforeInstanceNum) =
+ getPassNameAndInstanceNum(StartBeforeOpt);
+ std::tie(StartAfter, StartAfterInstanceNum) =
+ getPassNameAndInstanceNum(StartAfterOpt);
+ std::tie(StopBefore, StopBeforeInstanceNum) =
+ getPassNameAndInstanceNum(StopBeforeOpt);
+ std::tie(StopAfter, StopAfterInstanceNum) =
+ getPassNameAndInstanceNum(StopAfterOpt);
+
+ if (StartBefore.empty() && StartAfter.empty() && StopBefore.empty() &&
+ StopAfter.empty())
+ return;
+
+ std::tie(StartBefore, std::ignore) =
+ LLVMTM.getPassNameFromLegacyName(StartBefore);
+ std::tie(StartAfter, std::ignore) =
+ LLVMTM.getPassNameFromLegacyName(StartAfter);
+ std::tie(StopBefore, std::ignore) =
+ LLVMTM.getPassNameFromLegacyName(StopBefore);
+ std::tie(StopAfter, std::ignore) =
+ LLVMTM.getPassNameFromLegacyName(StopAfter);
+ if (!StartBefore.empty() && !StartAfter.empty())
+ report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
+ Twine(StartAfterOptName) + Twine(" specified!"));
+ if (!StopBefore.empty() && !StopAfter.empty())
+ report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
+ Twine(StopAfterOptName) + Twine(" specified!"));
+
+ PIC.registerBeforePassCallback(
+ [=, EnableCurrent = StartBefore.empty() && StartAfter.empty(),
+ EnableNext = Optional<bool>(), StartBeforeCount = 0u,
+ StartAfterCount = 0u, StopBeforeCount = 0u,
+ StopAfterCount = 0u](StringRef P, Any) mutable {
+ bool StartBeforePass = !StartBefore.empty() && P.contains(StartBefore);
+ bool StartAfterPass = !StartAfter.empty() && P.contains(StartAfter);
+ bool StopBeforePass = !StopBefore.empty() && P.contains(StopBefore);
+ bool StopAfterPass = !StopAfter.empty() && P.contains(StopAfter);
+
+ // Implement -start-after/-stop-after
+ if (EnableNext) {
+ EnableCurrent = *EnableNext;
+ EnableNext.reset();
+ }
+
+ // Using PIC.registerAfterPassCallback won't work because if this
+ // callback returns false, AfterPassCallback is also skipped.
+ if (StartAfterPass && StartAfterCount++ == StartAfterInstanceNum) {
+ assert(!EnableNext && "Error: assign to EnableNext more than once");
+ EnableNext = true;
+ }
+ if (StopAfterPass && StopAfterCount++ == StopAfterInstanceNum) {
+ assert(!EnableNext && "Error: assign to EnableNext more than once");
+ EnableNext = false;
+ }
+
+ if (StartBeforePass && StartBeforeCount++ == StartBeforeInstanceNum)
+ EnableCurrent = true;
+ if (StopBeforePass && StopBeforeCount++ == StopBeforeInstanceNum)
+ EnableCurrent = false;
+ return EnableCurrent;
+ });
+}
+
+void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
+ LLVMTargetMachine &LLVMTM) {
+
+ // Register a callback for disabling passes.
+ PIC.registerBeforePassCallback([](StringRef P, Any) {
+
+#define DISABLE_PASS(Option, Name) \
+ if (Option && P.contains(#Name)) \
+ return false;
+ DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
+ DISABLE_PASS(DisableBranchFold, BranchFolderPass)
+ DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
+ DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterPass)
+ DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
+ DISABLE_PASS(DisableMachineCSE, MachineCSEPass)
+ DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
+ DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
+ DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
+ DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
+ DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
+ DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
+ DISABLE_PASS(DisableSSC, StackSlotColoringPass)
+ DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
+
+ return true;
+ });
+
+ registerPartialPipelineCallback(PIC, LLVMTM);
+}
+
// Out of line constructor provides default values for pass options and
// registers all common codegen passes.
TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
@@ -1012,10 +1152,11 @@ void TargetPassConfig::addMachinePasses() {
addPass(&LiveDebugValuesID, false);
if (TM->Options.EnableMachineOutliner && getOptLevel() != CodeGenOpt::None &&
- EnableMachineOutliner != NeverOutline) {
- bool RunOnAllFunctions = (EnableMachineOutliner == AlwaysOutline);
- bool AddOutliner = RunOnAllFunctions ||
- TM->Options.SupportsDefaultOutlining;
+ EnableMachineOutliner != RunOutliner::NeverOutline) {
+ bool RunOnAllFunctions =
+ (EnableMachineOutliner == RunOutliner::AlwaysOutline);
+ bool AddOutliner =
+ RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
if (AddOutliner)
addPass(createMachineOutlinerPass(RunOnAllFunctions));
}
More information about the llvm-commits
mailing list