[llvm] 2ea4c2c - [BFI] Make BFI information available through loop passes inside LoopStandardAnalysisResults
Wenlei He via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 16:16:42 PDT 2020
Author: Wenlei He
Date: 2020-09-15T16:16:24-07:00
New Revision: 2ea4c2c598b7c6f95b5d4db747bdf72770e586df
URL: https://github.com/llvm/llvm-project/commit/2ea4c2c598b7c6f95b5d4db747bdf72770e586df
DIFF: https://github.com/llvm/llvm-project/commit/2ea4c2c598b7c6f95b5d4db747bdf72770e586df.diff
LOG: [BFI] Make BFI information available through loop passes inside LoopStandardAnalysisResults
~~D65060 uncovered that trying to use BFI in loop passes can lead to non-deterministic behavior when blocks are re-used while retaining old BFI data.~~
~~To make sure BFI is preserved through loop passes a Value Handle (VH) callback is registered on blocks themselves. When a block is freed it now also wipes out the accompanying BFI entry such that stale BFI data can no longer persist resolving the determinism issue. ~~
~~An optimistic approach would be to incrementally update BFI information throughout the loop passes rather than only invalidating them on removed blocks. The issues with that are:~~
~~1. It is not clear how BFI information should be incrementally updated: If a block is duplicated does its BFI information come with? How about if it's split/modified/moved around? ~~
~~2. Assuming we can address these problems the implementation here will be a massive undertaking. ~~
~~There's a known need of BFI in LICM analysis which requires correct but not incrementally updated BFI data. A follow-up change can register BFI in all loop passes so this preserved but potentially lossy data is available to any loop pass that wants it.~~
See: D75341 for an identical implementation of preserving BFI via VH callbacks. The previous statements do still apply but this change no longer has to be in this diff because it's already upstream 😄 .
This diff also moves BFI to be a part of LoopStandardAnalysisResults since the previous method using getCachedResults now (correctly!) statically asserts (D72893) that this data isn't static through the loop passes.
Testing
Ninja check
Reviewed By: asbirlea, nikic
Differential Revision: https://reviews.llvm.org/D86156
Added:
Modified:
llvm/include/llvm/Analysis/LoopAnalysisManager.h
llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
llvm/lib/Passes/PassBuilder.cpp
llvm/lib/Transforms/Scalar/LICM.cpp
llvm/lib/Transforms/Scalar/LoopDistribute.cpp
llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
llvm/lib/Transforms/Utils/LoopVersioning.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Other/opt-O2-pipeline.ll
llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
llvm/test/Other/opt-O3-pipeline.ll
llvm/test/Other/opt-Os-pipeline.ll
llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/llvm/include/llvm/Analysis/LoopAnalysisManager.h
index 0e162e03bde1..11dbd15c8678 100644
--- a/llvm/include/llvm/Analysis/LoopAnalysisManager.h
+++ b/llvm/include/llvm/Analysis/LoopAnalysisManager.h
@@ -57,6 +57,7 @@ struct LoopStandardAnalysisResults {
ScalarEvolution &SE;
TargetLibraryInfo &TLI;
TargetTransformInfo &TTI;
+ BlockFrequencyInfo *BFI;
MemorySSA *MSSA;
};
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 751c1832ba6c..821de6c70aa0 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -41,6 +41,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -233,9 +234,11 @@ class FunctionToLoopPassAdaptor
: public PassInfoMixin<FunctionToLoopPassAdaptor<LoopPassT>> {
public:
explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
+ bool UseBlockFrequencyInfo = false,
bool DebugLogging = false)
: Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging),
- UseMemorySSA(UseMemorySSA) {
+ UseMemorySSA(UseMemorySSA),
+ UseBlockFrequencyInfo(UseBlockFrequencyInfo) {
LoopCanonicalizationFPM.addPass(LoopSimplifyPass());
LoopCanonicalizationFPM.addPass(LCSSAPass());
}
@@ -267,6 +270,9 @@ class FunctionToLoopPassAdaptor
MemorySSA *MSSA = UseMemorySSA
? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA())
: nullptr;
+ BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData()
+ ? (&AM.getResult<BlockFrequencyAnalysis>(F))
+ : nullptr;
LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F),
AM.getResult<AssumptionAnalysis>(F),
AM.getResult<DominatorTreeAnalysis>(F),
@@ -274,6 +280,7 @@ class FunctionToLoopPassAdaptor
AM.getResult<ScalarEvolutionAnalysis>(F),
AM.getResult<TargetLibraryAnalysis>(F),
AM.getResult<TargetIRAnalysis>(F),
+ BFI,
MSSA};
// Setup the loop analysis manager from its proxy. It is important that
@@ -370,6 +377,8 @@ class FunctionToLoopPassAdaptor
PA.preserve<DominatorTreeAnalysis>();
PA.preserve<LoopAnalysis>();
PA.preserve<ScalarEvolutionAnalysis>();
+ if (UseBlockFrequencyInfo && F.hasProfileData())
+ PA.preserve<BlockFrequencyAnalysis>();
if (UseMemorySSA)
PA.preserve<MemorySSAAnalysis>();
// FIXME: What we really want to do here is preserve an AA category, but
@@ -389,6 +398,7 @@ class FunctionToLoopPassAdaptor
FunctionPassManager LoopCanonicalizationFPM;
bool UseMemorySSA = false;
+ bool UseBlockFrequencyInfo = false;
};
/// A function to deduce a loop pass type and wrap it in the templated
@@ -396,9 +406,10 @@ class FunctionToLoopPassAdaptor
template <typename LoopPassT>
FunctionToLoopPassAdaptor<LoopPassT>
createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
+ bool UseBlockFrequencyInfo = false,
bool DebugLogging = false) {
- return FunctionToLoopPassAdaptor<LoopPassT>(std::move(Pass), UseMemorySSA,
- DebugLogging);
+ return FunctionToLoopPassAdaptor<LoopPassT>(
+ std::move(Pass), UseMemorySSA, UseBlockFrequencyInfo, DebugLogging);
}
/// Pass for printing a loop's contents as textual IR.
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 03b31c233361..ddbc7a2fb4d5 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -520,13 +520,15 @@ FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(
FPM.addPass(
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
+ std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
+ DebugLogging));
FPM.addPass(SimplifyCFGPass());
FPM.addPass(InstCombinePass());
// The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
// *All* loop passes must preserve it, in order to be able to use it.
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
+ std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
// Delete small array after loop unroll.
FPM.addPass(SROA());
@@ -677,14 +679,16 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
FPM.addPass(
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
+ std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
+ DebugLogging));
FPM.addPass(SimplifyCFGPass());
FPM.addPass(InstCombinePass());
// The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
// LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
// *All* loop passes must preserve it, in order to be able to use it.
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
+ std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
// Delete small array after loop unroll.
FPM.addPass(SROA());
@@ -721,7 +725,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
FPM.addPass(DSEPass());
FPM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
- EnableMSSALoopDependency, DebugLogging));
+ EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
if (PTO.Coroutines)
FPM.addPass(CoroElidePass());
@@ -799,7 +803,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
FunctionPassManager FPM;
FPM.addPass(createFunctionToLoopPassAdaptor(
- LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
+ LoopRotatePass(), EnableMSSALoopDependency,
+ /*UseBlockFrequencyInfo=*/false, DebugLogging));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
// Add the profile lowering pass.
@@ -1129,7 +1134,8 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
// First rotate loops that may have been un-rotated by prior passes.
OptimizePM.addPass(createFunctionToLoopPassAdaptor(
- LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
+ LoopRotatePass(), EnableMSSALoopDependency,
+ /*UseBlockFrequencyInfo=*/false, DebugLogging));
// Distribute loops to allow partial vectorization. I.e. isolate dependences
// into separate loop that would otherwise inhibit vectorization. This is
@@ -1196,7 +1202,7 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
OptimizePM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
- EnableMSSALoopDependency, DebugLogging));
+ EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
// Now that we've vectorized and unrolled loops, we may have more refined
// alignment information, try to re-derive it here.
@@ -2261,8 +2267,9 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
}
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- MPM.addPass(createModuleToFunctionPassAdaptor( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \
+ MPM.addPass( \
+ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS, false, false, DebugLogging))); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2272,7 +2279,7 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
return Params.takeError(); \
MPM.addPass( \
createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
- CREATE_PASS(Params.get()), false, DebugLogging))); \
+ CREATE_PASS(Params.get()), false, false, DebugLogging))); \
return Error::success(); \
}
#include "PassRegistry.def"
@@ -2373,8 +2380,9 @@ Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
}
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \
+ CGPM.addPass( \
+ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS, false, false, DebugLogging))); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2384,7 +2392,7 @@ Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
return Params.takeError(); \
CGPM.addPass( \
createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
- CREATE_PASS(Params.get()), false, DebugLogging))); \
+ CREATE_PASS(Params.get()), false, false, DebugLogging))); \
return Error::success(); \
}
#include "PassRegistry.def"
@@ -2421,8 +2429,9 @@ Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
return Err;
// Add the nested pass manager with the appropriate adaptor.
bool UseMemorySSA = (Name == "loop-mssa");
- FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
- DebugLogging));
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ std::move(LPM), UseMemorySSA, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
return Error::success();
}
if (auto Count = parseRepeatPassName(Name)) {
@@ -2476,8 +2485,8 @@ Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
// The risk is that it may become obsolete if we're not careful.
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- FPM.addPass( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging)); \
+ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \
+ DebugLogging)); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2486,7 +2495,7 @@ Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
if (!Params) \
return Params.takeError(); \
FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
- false, DebugLogging)); \
+ false, false, DebugLogging)); \
return Error::success(); \
}
#include "PassRegistry.def"
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index b741d36e37bf..841badba0834 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -39,6 +39,7 @@
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/GuardUtils.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -171,8 +172,8 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest,
namespace {
struct LoopInvariantCodeMotion {
bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
- TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
- ScalarEvolution *SE, MemorySSA *MSSA,
+ BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
+ TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
OptimizationRemarkEmitter *ORE);
LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
@@ -208,19 +209,23 @@ struct LegacyLICMPass : public LoopPass {
MemorySSA *MSSA = EnableMSSALoopDependency
? (&getAnalysis<MemorySSAWrapperPass>().getMSSA())
: nullptr;
+ bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
+ BlockFrequencyInfo *BFI =
+ hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
+ : nullptr;
// For the old PM, we can't use OptimizationRemarkEmitter as an analysis
- // pass. Function analyses need to be preserved across loop transformations
+ // pass. Function analyses need to be preserved across loop transformations
// but ORE cannot be preserved (see comment before the pass definition).
OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
- return LICM.runOnLoop(L,
- &getAnalysis<AAResultsWrapperPass>().getAAResults(),
- &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
- &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
- &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
- *L->getHeader()->getParent()),
- &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
- *L->getHeader()->getParent()),
- SE ? &SE->getSE() : nullptr, MSSA, &ORE);
+ return LICM.runOnLoop(
+ L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
+ &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
+ &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
+ &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
+ *L->getHeader()->getParent()),
+ &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
+ *L->getHeader()->getParent()),
+ SE ? &SE->getSE() : nullptr, MSSA, &ORE);
}
/// This transformation requires natural loop information & requires that
@@ -236,6 +241,9 @@ struct LegacyLICMPass : public LoopPass {
}
AU.addRequired<TargetTransformInfoWrapperPass>();
getLoopAnalysisUsage(AU);
+ LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
+ AU.addPreserved<LazyBlockFrequencyInfoPass>();
+ AU.addPreserved<LazyBranchProbabilityInfoPass>();
}
private:
@@ -251,8 +259,8 @@ PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
- if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE,
- AR.MSSA, &ORE))
+ if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
+ &AR.SE, AR.MSSA, &ORE))
return PreservedAnalyses::all();
auto PA = getLoopPassPreservedAnalyses();
@@ -272,6 +280,7 @@ INITIALIZE_PASS_DEPENDENCY(LoopPass)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)
INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,
false)
@@ -286,8 +295,8 @@ Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
/// times on one loop.
bool LoopInvariantCodeMotion::runOnLoop(
Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
- TargetLibraryInfo *TLI, TargetTransformInfo *TTI, ScalarEvolution *SE,
- MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
+ BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
+ ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
bool Changed = false;
assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.");
diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
index 7867a5468891..04b7254e4cdb 100644
--- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -1058,7 +1058,8 @@ PreservedAnalyses LoopDistributePass::run(Function &F,
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
std::function<const LoopAccessInfo &(Loop &)> GetLAA =
[&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, nullptr};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, nullptr};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index e8473d652025..ce010c9bacac 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -720,7 +720,8 @@ PreservedAnalyses LoopLoadEliminationPass::run(Function &F,
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
bool Changed = eliminateLoadsAcrossLoops(
F, LI, DT, BFI, PSI, [&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
});
diff --git a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
index d83b7b05f88b..00b242c16f38 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -32,6 +32,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -217,6 +218,10 @@ namespace {
/// loop preheaders be inserted into the CFG.
///
void getAnalysisUsage(AnalysisUsage &AU) const override {
+ // Lazy BFI and BPI are marked as preserved here so Loop Unswitching
+ // can remain part of the same loop pass as LICM
+ AU.addPreserved<LazyBlockFrequencyInfoPass>();
+ AU.addPreserved<LazyBranchProbabilityInfoPass>();
AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<TargetTransformInfoWrapperPass>();
if (EnableMSSALoopDependency) {
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index b4925064bc6b..fe8fb90d140a 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -357,7 +357,8 @@ PreservedAnalyses LoopVersioningPass::run(Function &F,
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 545540efc284..b203dd88eb3d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8621,7 +8621,8 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
std::function<const LoopAccessInfo &(Loop &)> GetLAA =
[&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll
index e606e7cfac17..58ed6b2a0820 100644
--- a/llvm/test/Other/opt-O2-pipeline.ll
+++ b/llvm/test/Other/opt-O2-pipeline.ll
@@ -111,6 +111,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -168,6 +170,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -270,10 +274,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
diff --git a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
index aaee6f786bac..493957e865d4 100644
--- a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
+++ b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
@@ -116,6 +116,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -173,6 +175,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -282,10 +286,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll
index b2d2f85ae21b..f674dabd5217 100644
--- a/llvm/test/Other/opt-O3-pipeline.ll
+++ b/llvm/test/Other/opt-O3-pipeline.ll
@@ -116,6 +116,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -173,6 +175,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -275,10 +279,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll
index cc91707c4b00..66df666a64c6 100644
--- a/llvm/test/Other/opt-Os-pipeline.ll
+++ b/llvm/test/Other/opt-Os-pipeline.ll
@@ -97,6 +97,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -154,6 +156,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -256,10 +260,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
diff --git a/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp b/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
index 8142eaf90de1..8bec9629c554 100644
--- a/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
+++ b/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
@@ -9,7 +9,10 @@
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -294,6 +297,9 @@ class LoopPassManagerTest : public ::testing::Test {
// those.
FAM.registerPass([&] { return AAManager(); });
FAM.registerPass([&] { return AssumptionAnalysis(); });
+ FAM.registerPass([&] { return BlockFrequencyAnalysis(); });
+ FAM.registerPass([&] { return BranchProbabilityAnalysis(); });
+ FAM.registerPass([&] { return PostDominatorTreeAnalysis(); });
FAM.registerPass([&] { return MemorySSAAnalysis(); });
FAM.registerPass([&] { return ScalarEvolutionAnalysis(); });
FAM.registerPass([&] { return TargetLibraryAnalysis(); });
More information about the llvm-commits
mailing list