[llvm] [GVN] Reorganise GVN.h/GVH.cpp to improve readability and maintainability (NFC) (PR #210327)
Momchil Velikov via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 06:30:35 PDT 2026
https://github.com/momchil-velikov created https://github.com/llvm/llvm-project/pull/210327
None
>From e1784a875479f2bb3fd139c7fff75393a3b91dc5 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 09:48:01 +0000
Subject: [PATCH 1/4] [GVN] Remove the "private" `llvm::gvn` namespace (NFC)
Move `AvailableValue` and `AvailableValueInBlock` into GVNPass, similar
to other helper types.
Retain `llvm::gvn::GVNLegacyPass` as just `llvm::GVNLegacyPass` -
"legacy" is already a sufficent hint and it is not going to become more
"private" by stacking "gvn" prefixes to the name.
Ideally, `GVNLegacyPass` should be defined in an anonymous namespace, but
that is not possible because it is declared as a friend of GVNPass.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 19 ++++++-------------
llvm/lib/Transforms/Scalar/GVN.cpp | 10 ++++++----
2 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 9142defb34de2..8b70d61fd9f3f 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -61,15 +61,6 @@ class PHINode;
class TargetLibraryInfo;
class Value;
class IntrinsicInst;
-/// A private "module" namespace for types and utilities used by GVN. These
-/// are implementation details and should not be used by clients.
-namespace LLVM_LIBRARY_VISIBILITY_NAMESPACE gvn {
-
-struct AvailableValue;
-struct AvailableValueInBlock;
-class GVNLegacyPass;
-
-} // end namespace gvn
/// A set of parameters to control various transforms performed by GVN pass.
// Each of the optional boolean parameters can be set to:
@@ -133,6 +124,8 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
public:
struct Expression;
+ struct AvailableValue;
+ struct AvailableValueInBlock;
GVNPass(GVNOptions Options = {}) : Options(Options) {}
@@ -248,7 +241,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
};
private:
- friend class gvn::GVNLegacyPass;
+ friend class GVNLegacyPass;
friend struct DenseMapInfo<Expression>;
MemoryDependenceResults *MD = nullptr;
@@ -354,7 +347,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
bool InvalidBlockRPONumbers = true;
using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
- using AvailValInBlkVect = SmallVector<gvn::AvailableValueInBlock, 64>;
+ using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
@@ -454,7 +447,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// Given a local dependency (Def or Clobber) determine if a value is
/// available for the load.
- std::optional<gvn::AvailableValue>
+ std::optional<AvailableValue>
AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address);
@@ -462,7 +455,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
/// value is available by finding dominating values for both addresses. If
/// so, the load can be rematerialized as a select of those two values.
- std::optional<gvn::AvailableValue>
+ std::optional<AvailableValue>
AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 1b7bcb10be8f8..dc9c1fb0ba719 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -81,10 +81,12 @@
#include <utility>
using namespace llvm;
-using namespace llvm::gvn;
using namespace llvm::VNCoercion;
using namespace PatternMatch;
+using AvailableValue = GVNPass::AvailableValue;
+using AvailableValueInBlock = GVNPass::AvailableValueInBlock;
+
#define DEBUG_TYPE "gvn"
STATISTIC(NumGVNInstr, "Number of instructions deleted");
@@ -193,7 +195,7 @@ template <> struct llvm::DenseMapInfo<GVNPass::Expression> {
/// Materialization of an AvailableValue never fails. An AvailableValue is
/// implicitly associated with a rematerialization point which is the
/// location of the instruction from which it was formed.
-struct llvm::gvn::AvailableValue {
+struct llvm::GVNPass::AvailableValue {
enum class ValType {
SimpleVal, // A simple offsetted value that is accessed.
LoadVal, // A value produced by a load.
@@ -289,7 +291,7 @@ struct llvm::gvn::AvailableValue {
/// Represents an AvailableValue which can be rematerialized at the end of
/// the associated BasicBlock.
-struct llvm::gvn::AvailableValueInBlock {
+struct llvm::GVNPass::AvailableValueInBlock {
/// BB - The basic block in question.
BasicBlock *BB = nullptr;
@@ -3996,7 +3998,7 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-class llvm::gvn::GVNLegacyPass : public FunctionPass {
+class llvm::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
>From 2293acff88a03097a610c682769ad316d6b1fc0e Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 11:06:39 +0000
Subject: [PATCH 2/4] [GVN] Rename some functions to follow LLVM naming
conventions (NFC)
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 8 ++---
llvm/lib/Transforms/Scalar/GVN.cpp | 38 +++++++++++------------
2 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 8b70d61fd9f3f..ffe07fb2ec7e2 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -448,7 +448,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// Given a local dependency (Def or Clobber) determine if a value is
/// available for the load.
std::optional<AvailableValue>
- AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address);
/// Given a select-dependency for the load (the load address is a select of
@@ -456,13 +456,13 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// value is available by finding dominating values for both addresses. If
/// so, the load can be rematerialized as a select of those two values.
std::optional<AvailableValue>
- AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+ analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From);
/// Given a list of non-local dependencies, determine if a value is
/// available for the load in each specified block. If it is, add it to
/// ValuesPerBlock. If not, add it to UnavailableBlocks.
- void AnalyzeLoadAvailability(LoadInst *Load,
+ void analyzeLoadAvailability(LoadInst *Load,
SmallVectorImpl<ReachingMemVal> &Deps,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
@@ -472,7 +472,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
LoadInst *Load);
- bool PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
/// Try to replace a load which executes on each loop iteraiton with Phi
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index dc9c1fb0ba719..ee050d5a3861c 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -965,7 +965,7 @@ enum class AvailabilityState : char {
/// 1) we know the block *is* fully available.
/// 2) we do not know whether the block is fully available or not, but we are
/// currently speculating that it will be.
-static bool IsValueFullyAvailableInBlock(
+static bool isValueFullyAvailableInBlock(
BasicBlock *BB,
DenseMap<BasicBlock *, AvailabilityState> &FullyAvailableBlocks) {
SmallVector<BasicBlock *, 32> Worklist;
@@ -1100,7 +1100,7 @@ static void replaceValuesPerBlockEntry(
/// construct SSA form, allowing us to eliminate Load. This returns the value
/// that should be used at Load's definition site.
static Value *
-ConstructSSAForLoadSet(LoadInst *Load,
+constructSSAForLoadSet(LoadInst *Load,
SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
GVNPass &GVN) {
// Check for the fully redundant, dominating load case. In this case, we can
@@ -1330,7 +1330,7 @@ static Value *findDominatingValue(const MemoryLocation &Loc, Type *LoadTy,
}
std::optional<AvailableValue>
-GVNPass::AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+GVNPass::analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From) {
assert(TrueAddr->getType() == Load->getPointerOperandType() &&
"Invalid address type of true side of select dependency");
@@ -1352,7 +1352,7 @@ GVNPass::AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
}
std::optional<AvailableValue>
-GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+GVNPass::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address) {
assert(Load->isUnordered() && "rules below are incorrect for ordered access");
assert((Dep.Kind == DepKind::Def || Dep.Kind == DepKind::Clobber) &&
@@ -1479,7 +1479,7 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
// loads and DepInst that may clobber the loads.
if (auto *Sel = dyn_cast<SelectInst>(DepInst)) {
assert(Sel->getType() == Load->getPointerOperandType());
- if (auto AV = AnalyzeSelectAvailability(Load, Sel->getCondition(),
+ if (auto AV = analyzeSelectAvailability(Load, Sel->getCondition(),
Sel->getTrueValue(),
Sel->getFalseValue(), DepInst))
return AV;
@@ -1494,7 +1494,7 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
return std::nullopt;
}
-void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
+void GVNPass::analyzeLoadAvailability(LoadInst *Load,
SmallVectorImpl<ReachingMemVal> &Deps,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
@@ -1521,7 +1521,7 @@ void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
// load as a select of the two reaching values (one per side). The values
// are searched for at the end of DepBB.
if (Dep.Kind == DepKind::Select) {
- if (auto AV = AnalyzeSelectAvailability(
+ if (auto AV = analyzeSelectAvailability(
Load, const_cast<Value *>(Dep.SelCond),
const_cast<Value *>(Dep.SelTrueAddr),
const_cast<Value *>(Dep.SelFalseAddr), DepBB->getTerminator())) {
@@ -1537,7 +1537,7 @@ void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
// the pointer operand of the load if PHI translation occurs. Make sure
// to consider the right address.
if (auto AV =
- AnalyzeLoadAvailability(Load, Dep, const_cast<Value *>(Dep.Addr))) {
+ analyzeLoadAvailability(Load, Dep, const_cast<Value *>(Dep.Addr))) {
// subtlety: because we know this was a non-local dependency, we know
// it's safe to materialize anywhere between the instruction within
// DepInfo and the end of it's block.
@@ -1609,7 +1609,7 @@ LoadInst *GVNPass::findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
// If an identical load doesn't depends on any local instructions, it can
// be safely moved to PredBB.
// Also check for the implicit control flow instructions. See the comments
- // in PerformLoadPRE for details.
+ // in performLoadPRE for details.
if (!HasLocalDep && !ICF->isDominatedByICFIFromSameBlock(&Inst))
return cast<LoadInst>(&Inst);
@@ -1693,8 +1693,8 @@ void GVNPass::eliminatePartiallyRedundantLoad(
}
// Perform PHI construction.
- Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // ConstructSSAForLoadSet is responsible for combining metadata.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ // constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
if (isa<PHINode>(V))
@@ -1710,7 +1710,7 @@ void GVNPass::eliminatePartiallyRedundantLoad(
salvageAndRemoveInstruction(Load);
}
-bool GVNPass::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
// Okay, we have *some* definitions of the value. This means that the value
// is available in some of our (transitive) predecessors. Lets think about
@@ -1793,7 +1793,7 @@ bool GVNPass::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
return false;
}
- if (IsValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
+ if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
continue;
}
@@ -2122,7 +2122,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
// Step 1: Analyze the availability of the load.
AvailValInBlkVect ValuesPerBlock;
UnavailBlkVect UnavailableBlocks;
- AnalyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
+ analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
// If we have no predecessors that produce a known value for this load, exit
// early.
@@ -2138,8 +2138,8 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
// Perform PHI construction.
- Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // ConstructSSAForLoadSet is responsible for combining metadata.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ // constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
@@ -2166,7 +2166,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
return Changed;
if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
- PerformLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
+ performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
return true;
return Changed;
@@ -2595,7 +2595,7 @@ void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
/// Given as input a load instruction, the function computes the set of reaching
-/// memory values, one per predecessor path, that AnalyzeLoadAvailability can
+/// memory values, one per predecessor path, that analyzeLoadAvailability can
/// later use to establish whether the load may be eliminated. A reaching value
/// may be of the following descriptor kind:
/// * Def: a precise instruction that produces the exact bits the load would
@@ -2835,7 +2835,7 @@ bool GVNPass::processLoad(LoadInst *L) {
return false;
}
- auto AV = AnalyzeLoadAvailability(L, MemVal, L->getPointerOperand());
+ auto AV = analyzeLoadAvailability(L, MemVal, L->getPointerOperand());
if (!AV)
return false;
>From 67ad2e492b9f7a5ead697bfd8ec1e21aef2973cf Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 14:33:47 +0000
Subject: [PATCH 3/4] [GVN] Remove unused debug helper (NFC)
The `GVNPass::dump` method is not used anywhere. Moreover, there's
no `GVNPass` state that corresponds to its parameter type. Even if a
`GVNPass::dump` method could be useful, this one wasn't it.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 1 -
llvm/lib/Transforms/Scalar/GVN.cpp | 11 -----------
2 files changed, 12 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index ffe07fb2ec7e2..0b9c8dc88fe14 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -491,7 +491,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
// Other helper routines.
bool processInstruction(Instruction *I);
bool processBlock(BasicBlock *BB);
- void dump(DenseMap<uint32_t, Value *> &Map) const;
bool iterateOnFunction(Function &F);
bool performPRE(Function &F);
bool performScalarPRE(Instruction *I);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index ee050d5a3861c..155308f0e7f71 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -934,17 +934,6 @@ void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
removeInstruction(I);
}
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void GVNPass::dump(DenseMap<uint32_t, Value *> &Map) const {
- errs() << "{\n";
- for (const auto &[Num, Exp] : Map) {
- errs() << Num << "\n";
- Exp->dump();
- }
- errs() << "}\n";
-}
-#endif
-
enum class AvailabilityState : char {
/// We know the block *is not* fully available. This is a fixpoint.
Unavailable = 0,
>From 29556c748d9f533bb8bdf1a98f3bf279ebedaddb Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 8 Jul 2026 10:50:27 +0000
Subject: [PATCH 4/4] [GVN] Reorganise GVN.h/GVH.cpp to improve readability and
maintainability (NFC)
Over the years GVN.h/GVN.cpp has grown in size and complexity, and the order of
member functions and definitions has become somewhat arbitrary. This commit
reorganises the code to improve readability and maintainability.
* in `GVNPass` class, put private member variables first, followed by public
member functions, and then private member functions
* in `GVNPass` class: private type definitions are placed in front of the
logically related member variables (except `ValueTable` which need to be
public)
* definitions of `GVNPass::ValueTable` methods are grouped and reordered to
match the order of their declarations
* The following `GVNPass` member functions were made `private` and `LLVM_API`
removed: `getDominatorTree`, `getAliasAnalysis`, `getMemDep`,
`isScalarPREEnabled`, `isLoadPREEnabled`, `isLoadInLoopPREEnabled`,
`isLoadPRESplitBackedgeEnabled`, `isMemDepEnabled`, `isMemorySSAEnabled`,
and `salvageAndRemoveInstruction`
* `constructSSAForLoadSet` changed to take a `Dominator &`, in order to not
require access to the (now) private `getDominatorTree`
* member functions of `GVNPass` rearranged into a more logical order:
- starting with the main pass entry pount (`runImpl`) put utility member
functions in front of their callers, in order of calling (where it matters),
for example `runImpl` -> `iterateOnFunction` -> `perfromPRE`
- group functions of the same "theme" together, for example
`iterateOnFunction` + `processBlock` + `processInstrution`, or another
example, `performLoadPRE` + `performLoopLoadPRE`
- put miscelaneous helper member functions at the end
* rearrange definitions in `GVN.cpp` to match the order of declarations in
`GVN.h`
* place `static` helper functions close and in front of their callers
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 191 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 3045 ++++++++++-----------
2 files changed, 1629 insertions(+), 1607 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 0b9c8dc88fe14..46c54363298a2 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -120,37 +120,10 @@ struct GVNOptions {
/// FIXME: We should have a good summary of the GVN algorithm implemented by
/// this particular pass here.
class GVNPass : public OptionalPassInfoMixin<GVNPass> {
- GVNOptions Options;
-
public:
struct Expression;
struct AvailableValue;
struct AvailableValueInBlock;
-
- GVNPass(GVNOptions Options = {}) : Options(Options) {}
-
- /// Run the pass over the function.
- LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-
- LLVM_ABI void
- printPipeline(raw_ostream &OS,
- function_ref<StringRef(StringRef)> MapClassName2PassName);
-
- /// This removes the specified instruction from
- /// our various maps and marks it for deletion.
- LLVM_ABI void salvageAndRemoveInstruction(Instruction *I);
-
- DominatorTree &getDominatorTree() const { return *DT; }
- AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
- MemoryDependenceResults &getMemDep() const { return *MD; }
-
- LLVM_ABI bool isScalarPREEnabled() const;
- LLVM_ABI bool isLoadPREEnabled() const;
- LLVM_ABI bool isLoadInLoopPREEnabled() const;
- LLVM_ABI bool isLoadPRESplitBackedgeEnabled() const;
- LLVM_ABI bool isMemDepEnabled() const;
- LLVM_ABI bool isMemorySSAEnabled() const;
-
/// This class holds the mapping between values and value numbers. It is used
/// as an efficient mechanism to determine the expression-wise equivalence of
/// two values.
@@ -210,6 +183,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI ~ValueTable();
LLVM_ABI ValueTable &operator=(const ValueTable &Arg);
+ LLVM_ABI void add(Value *V, uint32_t Num);
LLVM_ABI uint32_t lookupOrAdd(MemoryAccess *MA);
LLVM_ABI uint32_t lookupOrAdd(Value *V);
LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const;
@@ -222,7 +196,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
const BasicBlock &CurrBlock);
LLVM_ABI bool exists(Value *V) const;
- LLVM_ABI void add(Value *V, uint32_t Num);
LLVM_ABI void clear();
LLVM_ABI void erase(Value *V);
void setAliasAnalysis(AAResults *A) { AA = A; }
@@ -244,6 +217,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
friend class GVNLegacyPass;
friend struct DenseMapInfo<Expression>;
+ GVNOptions Options;
MemoryDependenceResults *MD = nullptr;
DominatorTree *DT = nullptr;
const TargetLibraryInfo *TLI = nullptr;
@@ -254,7 +228,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoopInfo *LI = nullptr;
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
-
ValueTable VN;
/// A mapping from value numbers to lists of Value*'s that
@@ -346,18 +319,35 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
// of BlockRPONumber prior to accessing the contents of BlockRPONumber.
bool InvalidBlockRPONumbers = true;
+ // List of critical edges to be split between iterations.
+ SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
+
+public:
+ GVNPass(GVNOptions Options = {}) : Options(Options) {}
+
+ /// Run the pass over the function.
+ LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+ LLVM_ABI void
+ printPipeline(raw_ostream &OS,
+ function_ref<StringRef(StringRef)> MapClassName2PassName);
+
+private:
+ DominatorTree &getDominatorTree() const { return *DT; }
+ AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
+ MemoryDependenceResults &getMemDep() const { return *MD; }
+
+ bool isScalarPREEnabled() const;
+ bool isLoadPREEnabled() const;
+ bool isLoadInLoopPREEnabled() const;
+ bool isLoadPRESplitBackedgeEnabled() const;
+ bool isMemDepEnabled() const;
+ bool isMemorySSAEnabled() const;
+
using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
- bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
- const TargetLibraryInfo &RunTLI, AAResults &RunAA,
- MemoryDependenceResults *RunMD, LoopInfo &LI,
- OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
-
- // List of critical edges to be split between iterations.
- SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
-
enum class DepKind {
Other = 0, // Unknown value.
Def, // Exactly overlapping locations.
@@ -416,6 +406,41 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
+ /// Given a select-dependency for the load (the load address is a select of
+ /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
+ /// value is available by finding dominating values for both addresses. If
+ /// so, the load can be rematerialized as a select of those two values.
+ std::optional<AvailableValue>
+ analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+ Value *FalseAddr, Instruction *From);
+
+ /// Given a local dependency (Def or Clobber) determine if a value is
+ /// available for the load.
+ std::optional<AvailableValue>
+ analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ Value *Address);
+
+ /// Given a list of non-local dependencies, determine if a value is
+ /// available for the load in each specified block. If it is, add it to
+ /// ValuesPerBlock. If not, add it to UnavailableBlocks.
+ void analyzeLoadAvailability(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks);
+
+ /// Given a critical edge from Pred to LoadBB, find a load instruction
+ /// which is identical to Load from another successor of Pred.
+ LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
+ LoadInst *Load);
+
+ /// Eliminates partially redundant \p Load, replacing it with \p
+ /// AvailableLoads (connected by Phis if needed).
+ void eliminatePartiallyRedundantLoad(
+ LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ MapVector<BasicBlock *, Value *> &AvailableLoads,
+ MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
+
+ // Helper functions for d etermining load dependencies.
std::optional<GVNPass::ReachingMemVal> scanMemoryAccessesUsers(
const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
@@ -438,40 +463,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
SmallVectorImpl<ReachingMemVal> &Values,
MemorySSA &MSSA, AAResults &AA);
- // Helper functions of redundant load elimination.
- bool processLoad(LoadInst *L);
- bool processMaskedLoad(IntrinsicInst *I);
- bool processNonLocalLoad(LoadInst *L);
- bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
- bool processAssumeIntrinsic(AssumeInst *II);
-
- /// Given a local dependency (Def or Clobber) determine if a value is
- /// available for the load.
- std::optional<AvailableValue>
- analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
- Value *Address);
-
- /// Given a select-dependency for the load (the load address is a select of
- /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
- /// value is available by finding dominating values for both addresses. If
- /// so, the load can be rematerialized as a select of those two values.
- std::optional<AvailableValue>
- analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
- Value *FalseAddr, Instruction *From);
-
- /// Given a list of non-local dependencies, determine if a value is
- /// available for the load in each specified block. If it is, add it to
- /// ValuesPerBlock. If not, add it to UnavailableBlocks.
- void analyzeLoadAvailability(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks);
-
- /// Given a critical edge from Pred to LoadBB, find a load instruction
- /// which is identical to Load from another successor of Pred.
- LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
- LoadInst *Load);
-
bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
@@ -481,31 +472,63 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
- /// Eliminates partially redundant \p Load, replacing it with \p
- /// AvailableLoads (connected by Phis if needed).
- void eliminatePartiallyRedundantLoad(
- LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- MapVector<BasicBlock *, Value *> &AvailableLoads,
- MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
+ // Try to eliminate redundent loades with non-local dependencies.
+ bool processNonLocalLoad(LoadInst *L);
+ bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
- // Other helper routines.
+ /// Add any blocks determined to be unreachable by a conditional branch with a
+ /// constant condition to the dead blocks.
+ bool processFoldableCondBr(CondBrInst *BI);
+
+ /// Propagate equalities derived from llvm.assume intrinsics.
+ bool processAssumeIntrinsic(AssumeInst *II);
+
+ /// Try to eliminate redundant loads.
+ bool processLoad(LoadInst *L);
+
+ /// Try to eliminate masked loads which have loaded from
+ /// masked stores with the same mask.
+ bool processMaskedLoad(IntrinsicInst *I);
+
+ /// Propagate value of a condition to blocks dominated by "then" and "else"
+ /// edges, as well as certains derived equalities.
+ bool
+ propagateEquality(Value *LHS, Value *RHS,
+ const std::variant<BasicBlockEdge, Instruction *> &Root);
+
+ // Pass iteration helper functions.
bool processInstruction(Instruction *I);
bool processBlock(BasicBlock *BB);
bool iterateOnFunction(Function &F);
- bool performPRE(Function &F);
- bool performScalarPRE(Instruction *I);
+
+ // Scalar PRE helper functions
bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
BasicBlock *Curr, unsigned int ValNo);
+ bool performScalarPRE(Instruction *I);
+ bool performPRE(Function &F);
+
+ /// Main entry point for the GVN pass. Also used by the GVNLegacyPass.
+ bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+ const TargetLibraryInfo &RunTLI, AAResults &RunAA,
+ MemoryDependenceResults *RunMD, LoopInfo &LI,
+ OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
+
+ // Other helper routines.
+
Value *findLeader(const BasicBlock *BB, uint32_t Num);
void cleanupGlobalSets();
+
void removeInstruction(Instruction *I);
+
+ /// This removes the specified instruction from
+ /// our various maps and marks it for deletion.
+ void salvageAndRemoveInstruction(Instruction *I);
+
void verifyRemoved(const Instruction *I) const;
+
bool splitCriticalEdges();
BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
- bool
- propagateEquality(Value *LHS, Value *RHS,
- const std::variant<BasicBlockEdge, Instruction *> &Root);
- bool processFoldableCondBr(CondBrInst *BI);
+
void addDeadBlock(BasicBlock *BB);
void assignValNumForDeadCode();
void assignBlockRPONumber(Function &F);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 155308f0e7f71..f16d1fe9ca893 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -289,6 +289,72 @@ struct llvm::GVNPass::AvailableValue {
Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const;
};
+Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
+ Instruction *InsertPt) const {
+ Value *Res;
+ Type *LoadTy = Load->getType();
+ const DataLayout &DL = Load->getDataLayout();
+ if (isSimpleValue()) {
+ Res = getSimpleValue();
+ if (Res->getType() != LoadTy) {
+ Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
+
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
+ << " " << *getSimpleValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isCoercedLoadValue()) {
+ LoadInst *CoercedLoad = getCoercedLoadValue();
+ if (CoercedLoad->getType() == LoadTy && Offset == 0) {
+ Res = CoercedLoad;
+ combineMetadataForCSE(CoercedLoad, Load, false);
+ } else {
+ Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
+ Load->getFunction());
+ // We are adding a new user for this load, for which the original
+ // metadata may not hold. Additionally, the new load may have a different
+ // size and type, so their metadata cannot be combined in any
+ // straightforward way.
+ // Drop all metadata that is not known to cause immediate UB on violation,
+ // unless the load has !noundef, in which case all metadata violations
+ // will be promoted to UB.
+ // !noalias and !alias.scope are kept: the load is not moved and still
+ // accesses the same memory, and these are independent of the load type
+ // and offset, so they remain valid for the coerced result.
+ if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
+ CoercedLoad->dropUnknownNonDebugMetadata(
+ {LLVMContext::MD_dereferenceable,
+ LLVMContext::MD_dereferenceable_or_null,
+ LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
+ LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
+ << " " << *getCoercedLoadValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isMemIntrinValue()) {
+ Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, InsertPt,
+ DL);
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
+ << " " << *getMemIntrinValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ } else if (isSelectValue()) {
+ // Introduce a new value select for a load from an eligible pointer select.
+ Value *Cond = getSelectCondition();
+ assert(V1 && V2 && "both value operands of the select must be present");
+ Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
+ // We use the DebugLoc from the original load here, as this instruction
+ // materializes the value that would previously have been loaded.
+ cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
+ } else {
+ llvm_unreachable("Should not materialize value from dead block");
+ }
+ assert(Res && "failed to materialize?");
+ return Res;
+}
+
/// Represents an AvailableValue which can be rematerialized at the end of
/// the associated BasicBlock.
struct llvm::GVNPass::AvailableValueInBlock {
@@ -452,37 +518,6 @@ GVNPass::Expression GVNPass::ValueTable::createGEPExpr(GetElementPtrInst *GEP) {
return E;
}
-//===----------------------------------------------------------------------===//
-// ValueTable External Functions
-//===----------------------------------------------------------------------===//
-
-GVNPass::ValueTable::ValueTable() = default;
-GVNPass::ValueTable::ValueTable(const ValueTable &) = default;
-GVNPass::ValueTable::ValueTable(ValueTable &&) = default;
-GVNPass::ValueTable::~ValueTable() = default;
-GVNPass::ValueTable &
-GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default;
-
-/// add - Insert a value into the table with a specified value number.
-void GVNPass::ValueTable::add(Value *V, uint32_t Num) {
- ValueNumbering.insert(std::make_pair(V, Num));
- if (PHINode *PN = dyn_cast<PHINode>(V))
- NumberingPhi[Num] = PN;
-}
-
-/// Include the incoming memory state into the hash of the expression for the
-/// given instruction. If the incoming memory state is:
-/// * LiveOnEntry, add the value number of the entry block,
-/// * a MemoryPhi, add the value number of the basic block corresponding to that
-/// MemoryPhi,
-/// * a MemoryDef, add the value number of the memory setting instruction.
-void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
- assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
- assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
- MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
- Exp.VarArgs.push_back(lookupOrAdd(MA));
-}
-
uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) {
// FIXME: Currently the calls which may access the thread id may
// be considered as not accessing the memory. But this is
@@ -634,133 +669,326 @@ uint32_t GVNPass::ValueTable::computeLoadStoreVN(Instruction *I) {
return V;
}
-/// Returns true if a value number exists for the specified value.
-bool GVNPass::ValueTable::exists(Value *V) const {
- return ValueNumbering.contains(V);
-}
+/// Translate value number \p Num using phis, so that it has the values of
+/// the phis in BB.
+uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN) {
+ // See if we can refine the value number by looking at the PN incoming value
+ // for the given predecessor.
+ if (PHINode *PN = NumberingPhi[Num]) {
+ if (PN->getParent() != PhiBlock)
+ return Num;
+ for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) {
+ if (PN->getIncomingBlock(I) != Pred)
+ continue;
+ if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false))
+ return TransVal;
+ }
+ return Num;
+ }
-uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
- return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
- ? lookupOrAdd(MA->getBlock())
- : lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
-}
+ if (BasicBlock *BB = NumberingBB[Num]) {
+ assert(MSSA && "NumberingBB is non-empty only when using MemorySSA");
+ // Value numbers of basic blocks are used to represent memory state in
+ // load/store instructions and read-only function calls when said state is
+ // set by a MemoryPhi.
+ if (BB != PhiBlock)
+ return Num;
+ MemoryPhi *MPhi = MSSA->getMemoryAccess(BB);
+ for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) {
+ if (MPhi->getIncomingBlock(i) != Pred)
+ continue;
+ MemoryAccess *MA = MPhi->getIncomingValue(i);
+ if (auto *PredPhi = dyn_cast<MemoryPhi>(MA))
+ return lookupOrAdd(PredPhi->getBlock());
+ if (MSSA->isLiveOnEntryDef(MA))
+ return lookupOrAdd(&BB->getParent()->getEntryBlock());
+ return lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
+ }
+ llvm_unreachable(
+ "CFG/MemorySSA mismatch: predecessor not found among incoming blocks");
+ }
-/// lookupOrAdd - Returns the value number for the specified value, assigning
-/// it a new number if it did not have one before.
-uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
- auto VI = ValueNumbering.find(V);
- if (VI != ValueNumbering.end())
- return VI->second;
+ // If there is any value related with Num is defined in a BB other than
+ // PhiBlock, it cannot depend on a phi in PhiBlock without going through
+ // a backedge. We can do an early exit in that case to save compile time.
+ if (!areAllValsInBB(Num, PhiBlock, GVN))
+ return Num;
- auto *I = dyn_cast<Instruction>(V);
- if (!I) {
- ValueNumbering[V] = NextValueNumber;
- if (isa<BasicBlock>(V))
- NumberingBB[NextValueNumber] = cast<BasicBlock>(V);
- return NextValueNumber++;
+ if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
+ return Num;
+ Expression Exp = Expressions[ExprIdx[Num]];
+
+ for (unsigned I = 0; I < Exp.VarArgs.size(); I++) {
+ // For InsertValue and ExtractValue, some varargs are index numbers
+ // instead of value numbers. Those index numbers should not be
+ // translated.
+ if ((I > 1 && Exp.Opcode == Instruction::InsertValue) ||
+ (I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
+ (I > 1 && Exp.Opcode == Instruction::ShuffleVector))
+ continue;
+ Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN);
}
- Expression Exp;
- switch (I->getOpcode()) {
- case Instruction::Call:
- return lookupOrAddCall(cast<CallInst>(I));
- case Instruction::FNeg:
- case Instruction::Add:
- case Instruction::FAdd:
- case Instruction::Sub:
- case Instruction::FSub:
- case Instruction::Mul:
- case Instruction::FMul:
- case Instruction::UDiv:
- case Instruction::SDiv:
- case Instruction::FDiv:
- case Instruction::URem:
- case Instruction::SRem:
- case Instruction::FRem:
- case Instruction::Shl:
- case Instruction::LShr:
- case Instruction::AShr:
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor:
- case Instruction::ICmp:
- case Instruction::FCmp:
- case Instruction::Trunc:
- case Instruction::ZExt:
- case Instruction::SExt:
- case Instruction::FPToUI:
- case Instruction::FPToSI:
- case Instruction::UIToFP:
- case Instruction::SIToFP:
- case Instruction::FPTrunc:
- case Instruction::FPExt:
- case Instruction::PtrToInt:
- case Instruction::PtrToAddr:
- case Instruction::IntToPtr:
- case Instruction::AddrSpaceCast:
- case Instruction::BitCast:
- case Instruction::Select:
- case Instruction::Freeze:
- case Instruction::ExtractElement:
- case Instruction::InsertElement:
- case Instruction::ShuffleVector:
- case Instruction::InsertValue:
- Exp = createExpr(I);
- break;
- case Instruction::GetElementPtr:
- Exp = createGEPExpr(cast<GetElementPtrInst>(I));
- break;
- case Instruction::ExtractValue:
- Exp = createExtractvalueExpr(cast<ExtractValueInst>(I));
- break;
- case Instruction::PHI:
- ValueNumbering[V] = NextValueNumber;
- NumberingPhi[NextValueNumber] = cast<PHINode>(V);
- return NextValueNumber++;
- case Instruction::Load:
- case Instruction::Store:
- return computeLoadStoreVN(I);
- default:
- ValueNumbering[V] = NextValueNumber;
- return NextValueNumber++;
+ if (Exp.Commutative) {
+ assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!");
+ if (Exp.VarArgs[0] > Exp.VarArgs[1]) {
+ std::swap(Exp.VarArgs[0], Exp.VarArgs[1]);
+ uint32_t Opcode = Exp.Opcode >> 8;
+ if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
+ Exp.Opcode = (Opcode << 8) |
+ CmpInst::getSwappedPredicate(
+ static_cast<CmpInst::Predicate>(Exp.Opcode & 255));
+ }
}
- uint32_t E = assignExpNewValueNum(Exp).first;
- ValueNumbering[V] = E;
- return E;
+ if (uint32_t NewNum = ExpressionNumbering[Exp]) {
+ if (Exp.Opcode == Instruction::Call && NewNum != Num)
+ return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num;
+ return NewNum;
+ }
+ return Num;
}
-/// Returns the value number of the specified value. Fails if
-/// the value has not yet been numbered.
-uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
- auto VI = ValueNumbering.find(V);
- if (Verify) {
- assert(VI != ValueNumbering.end() && "Value not numbered?");
- return VI->second;
+// Return true if the value number \p Num and NewNum have equal value.
+// Return false if the result is unknown.
+bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
+ const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ GVNPass &GVN) {
+ CallInst *Call = nullptr;
+ auto Leaders = GVN.LeaderTable.getLeaders(Num);
+ for (const auto &Entry : Leaders) {
+ Call = dyn_cast<CallInst>(&*Entry.Val);
+ if (Call && Call->getParent() == PhiBlock)
+ break;
}
- return (VI != ValueNumbering.end()) ? VI->second : 0;
-}
-/// Returns the value number of the given comparison,
-/// assigning it a new number if it did not have one before. Useful when
-/// we deduced the result of a comparison, but don't immediately have an
-/// instruction realizing that comparison to hand.
-uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode,
- CmpInst::Predicate Predicate,
- Value *LHS, Value *RHS) {
- Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
- return assignExpNewValueNum(Exp).first;
-}
+ if (AA->doesNotAccessMemory(Call))
+ return true;
-/// Returns the value number of ptrtoint \p Ptr to \Ty.
-uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
- Expression Exp(Instruction::PtrToInt);
- Exp.Ty = Ty;
- Exp.VarArgs.push_back(lookupOrAdd(Ptr));
- return ExpressionNumbering.lookup(Exp);
+ if (!MD || !AA->onlyReadsMemory(Call))
+ return false;
+
+ MemDepResult LocalDep = MD->getDependency(Call);
+ if (!LocalDep.isNonLocal())
+ return false;
+
+ const MemoryDependenceResults::NonLocalDepInfo &Deps =
+ MD->getNonLocalCallDependency(Call);
+
+ // Check to see if the Call has no function local clobber.
+ for (const NonLocalDepEntry &D : Deps) {
+ if (D.getResult().isNonFuncLocal())
+ return true;
+ }
+ return false;
}
-/// Remove all entries from the ValueTable.
+/// Return a pair the first field showing the value number of \p Exp and the
+/// second field showing whether it is a value number newly created.
+std::pair<uint32_t, bool>
+GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
+ uint32_t &E = ExpressionNumbering[Exp];
+ bool CreateNewValNum = !E;
+ if (CreateNewValNum) {
+ Expressions.push_back(Exp);
+ if (ExprIdx.size() < NextValueNumber + 1)
+ ExprIdx.resize(NextValueNumber * 2);
+ E = NextValueNumber;
+ ExprIdx[NextValueNumber++] = NextExprNumber++;
+ }
+ return {E, CreateNewValNum};
+}
+
+/// Return whether all the values related with the same \p num are
+/// defined in \p BB.
+bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
+ GVNPass &GVN) {
+ return all_of(
+ GVN.LeaderTable.getLeaders(Num),
+ [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
+}
+
+/// Include the incoming memory state into the hash of the expression for the
+/// given instruction. If the incoming memory state is:
+/// * LiveOnEntry, add the value number of the entry block,
+/// * a MemoryPhi, add the value number of the basic block corresponding to that
+/// MemoryPhi,
+/// * a MemoryDef, add the value number of the memory setting instruction.
+void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
+ assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
+ assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
+ MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
+ Exp.VarArgs.push_back(lookupOrAdd(MA));
+}
+
+//===----------------------------------------------------------------------===//
+// ValueTable External Functions
+//===----------------------------------------------------------------------===//
+
+GVNPass::ValueTable::ValueTable() = default;
+GVNPass::ValueTable::ValueTable(const ValueTable &) = default;
+GVNPass::ValueTable::ValueTable(ValueTable &&) = default;
+GVNPass::ValueTable::~ValueTable() = default;
+GVNPass::ValueTable &
+GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default;
+
+/// add - Insert a value into the table with a specified value number.
+void GVNPass::ValueTable::add(Value *V, uint32_t Num) {
+ ValueNumbering.insert(std::make_pair(V, Num));
+ if (PHINode *PN = dyn_cast<PHINode>(V))
+ NumberingPhi[Num] = PN;
+}
+
+uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
+ return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
+ ? lookupOrAdd(MA->getBlock())
+ : lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
+}
+
+/// lookupOrAdd - Returns the value number for the specified value, assigning
+/// it a new number if it did not have one before.
+uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
+ auto VI = ValueNumbering.find(V);
+ if (VI != ValueNumbering.end())
+ return VI->second;
+
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I) {
+ ValueNumbering[V] = NextValueNumber;
+ if (isa<BasicBlock>(V))
+ NumberingBB[NextValueNumber] = cast<BasicBlock>(V);
+ return NextValueNumber++;
+ }
+
+ Expression Exp;
+ switch (I->getOpcode()) {
+ case Instruction::Call:
+ return lookupOrAddCall(cast<CallInst>(I));
+ case Instruction::FNeg:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ case Instruction::Trunc:
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::UIToFP:
+ case Instruction::SIToFP:
+ case Instruction::FPTrunc:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::PtrToAddr:
+ case Instruction::IntToPtr:
+ case Instruction::AddrSpaceCast:
+ case Instruction::BitCast:
+ case Instruction::Select:
+ case Instruction::Freeze:
+ case Instruction::ExtractElement:
+ case Instruction::InsertElement:
+ case Instruction::ShuffleVector:
+ case Instruction::InsertValue:
+ Exp = createExpr(I);
+ break;
+ case Instruction::GetElementPtr:
+ Exp = createGEPExpr(cast<GetElementPtrInst>(I));
+ break;
+ case Instruction::ExtractValue:
+ Exp = createExtractvalueExpr(cast<ExtractValueInst>(I));
+ break;
+ case Instruction::PHI:
+ ValueNumbering[V] = NextValueNumber;
+ NumberingPhi[NextValueNumber] = cast<PHINode>(V);
+ return NextValueNumber++;
+ case Instruction::Load:
+ case Instruction::Store:
+ return computeLoadStoreVN(I);
+ default:
+ ValueNumbering[V] = NextValueNumber;
+ return NextValueNumber++;
+ }
+
+ uint32_t E = assignExpNewValueNum(Exp).first;
+ ValueNumbering[V] = E;
+ return E;
+}
+
+/// Returns the value number of the specified value. Fails if
+/// the value has not yet been numbered.
+uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
+ auto VI = ValueNumbering.find(V);
+ if (Verify) {
+ assert(VI != ValueNumbering.end() && "Value not numbered?");
+ return VI->second;
+ }
+ return (VI != ValueNumbering.end()) ? VI->second : 0;
+}
+
+/// Returns the value number of the given comparison,
+/// assigning it a new number if it did not have one before. Useful when
+/// we deduced the result of a comparison, but don't immediately have an
+/// instruction realizing that comparison to hand.
+uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode,
+ CmpInst::Predicate Predicate,
+ Value *LHS, Value *RHS) {
+ Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
+ return assignExpNewValueNum(Exp).first;
+}
+
+/// Returns the value number of ptrtoint \p Ptr to \Ty.
+uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
+ Expression Exp(Instruction::PtrToInt);
+ Exp.Ty = Ty;
+ Exp.VarArgs.push_back(lookupOrAdd(Ptr));
+ return ExpressionNumbering.lookup(Exp);
+}
+
+/// Wrap phiTranslateImpl to provide caching functionality.
+uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN) {
+ auto FindRes = PhiTranslateTable.find({Num, Pred});
+ if (FindRes != PhiTranslateTable.end())
+ return FindRes->second;
+ uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN);
+ PhiTranslateTable.insert({{Num, Pred}, NewNum});
+ return NewNum;
+}
+
+/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
+/// again.
+void GVNPass::ValueTable::eraseTranslateCacheEntry(
+ uint32_t Num, const BasicBlock &CurrBlock) {
+ for (const BasicBlock *Pred : predecessors(&CurrBlock))
+ PhiTranslateTable.erase({Num, Pred});
+}
+
+/// Returns true if a value number exists for the specified value.
+bool GVNPass::ValueTable::exists(Value *V) const {
+ return ValueNumbering.contains(V);
+}
+
+/// Remove all entries from the ValueTable.
void GVNPass::ValueTable::clear() {
ValueNumbering.clear();
ExpressionNumbering.clear();
@@ -851,31 +1079,6 @@ void GVNPass::LeaderMap::erase(uint32_t N, Instruction *I,
// GVN Pass
//===----------------------------------------------------------------------===//
-bool GVNPass::isScalarPREEnabled() const {
- return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE);
-}
-
-bool GVNPass::isLoadPREEnabled() const {
- return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE);
-}
-
-bool GVNPass::isLoadInLoopPREEnabled() const {
- return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE);
-}
-
-bool GVNPass::isLoadPRESplitBackedgeEnabled() const {
- return Options.AllowLoadPRESplitBackedge.value_or(
- GVNEnableSplitBackedgeInLoadPRE);
-}
-
-bool GVNPass::isMemDepEnabled() const {
- return Options.AllowMemDep.value_or(GVNEnableMemDep);
-}
-
-bool GVNPass::isMemorySSAEnabled() const {
- return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA);
-}
-
PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
// FIXME: The order of evaluation of these 'getResult' calls is very
// significant! Re-ordering these variables will cause GVN when run alone to
@@ -928,10 +1131,29 @@ void GVNPass::printPipeline(
OS << '>';
}
-void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
- salvageKnowledge(I, AC);
- salvageDebugInfo(*I);
- removeInstruction(I);
+bool GVNPass::isScalarPREEnabled() const {
+ return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE);
+}
+
+bool GVNPass::isLoadPREEnabled() const {
+ return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE);
+}
+
+bool GVNPass::isLoadInLoopPREEnabled() const {
+ return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE);
+}
+
+bool GVNPass::isLoadPRESplitBackedgeEnabled() const {
+ return Options.AllowLoadPRESplitBackedge.value_or(
+ GVNEnableSplitBackedgeInLoadPRE);
+}
+
+bool GVNPass::isMemDepEnabled() const {
+ return Options.AllowMemDep.value_or(GVNEnableMemDep);
+}
+
+bool GVNPass::isMemorySSAEnabled() const {
+ return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA);
}
enum class AvailabilityState : char {
@@ -1091,12 +1313,11 @@ static void replaceValuesPerBlockEntry(
static Value *
constructSSAForLoadSet(LoadInst *Load,
SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
- GVNPass &GVN) {
+ DominatorTree &DT) {
// Check for the fully redundant, dominating load case. In this case, we can
// just use the dominating value directly.
if (ValuesPerBlock.size() == 1 &&
- GVN.getDominatorTree().properlyDominates(ValuesPerBlock[0].BB,
- Load->getParent())) {
+ DT.properlyDominates(ValuesPerBlock[0].BB, Load->getParent())) {
assert(!ValuesPerBlock[0].AV.isUndefValue() &&
"Dead BB dominate this block");
return ValuesPerBlock[0].MaterializeAdjustedValue(Load);
@@ -1132,110 +1353,44 @@ constructSSAForLoadSet(LoadInst *Load,
return SSAUpdate.GetValueInMiddleOfBlock(Load->getParent());
}
-Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
- Instruction *InsertPt) const {
- Value *Res;
- Type *LoadTy = Load->getType();
- const DataLayout &DL = Load->getDataLayout();
- if (isSimpleValue()) {
- Res = getSimpleValue();
- if (Res->getType() != LoadTy) {
- Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
+static bool isLifetimeStart(const Instruction *Inst) {
+ if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))
+ return II->getIntrinsicID() == Intrinsic::lifetime_start;
+ return false;
+}
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
- << " " << *getSimpleValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isCoercedLoadValue()) {
- LoadInst *CoercedLoad = getCoercedLoadValue();
- if (CoercedLoad->getType() == LoadTy && Offset == 0) {
- Res = CoercedLoad;
- combineMetadataForCSE(CoercedLoad, Load, false);
- } else {
- Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
- Load->getFunction());
- // We are adding a new user for this load, for which the original
- // metadata may not hold. Additionally, the new load may have a different
- // size and type, so their metadata cannot be combined in any
- // straightforward way.
- // Drop all metadata that is not known to cause immediate UB on violation,
- // unless the load has !noundef, in which case all metadata violations
- // will be promoted to UB.
- // !noalias and !alias.scope are kept: the load is not moved and still
- // accesses the same memory, and these are independent of the load type
- // and offset, so they remain valid for the coerced result.
- if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
- CoercedLoad->dropUnknownNonDebugMetadata(
- {LLVMContext::MD_dereferenceable,
- LLVMContext::MD_dereferenceable_or_null,
- LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
- LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
- << " " << *getCoercedLoadValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isMemIntrinValue()) {
- Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy,
- InsertPt, DL);
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
- << " " << *getMemIntrinValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- } else if (isSelectValue()) {
- // Introduce a new value select for a load from an eligible pointer select.
- Value *Cond = getSelectCondition();
- assert(V1 && V2 && "both value operands of the select must be present");
- Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
- // We use the DebugLoc from the original load here, as this instruction
- // materializes the value that would previously have been loaded.
- cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
- } else {
- llvm_unreachable("Should not materialize value from dead block");
- }
- assert(Res && "failed to materialize?");
- return Res;
-}
-
-static bool isLifetimeStart(const Instruction *Inst) {
- if (const IntrinsicInst* II = dyn_cast<IntrinsicInst>(Inst))
- return II->getIntrinsicID() == Intrinsic::lifetime_start;
- return false;
-}
-
-/// Assuming To can be reached from both From and Between, does Between lie on
-/// every path from From to To?
-static bool liesBetween(const Instruction *From, Instruction *Between,
- const Instruction *To, const DominatorTree *DT) {
- if (From->getParent() == Between->getParent())
- return DT->dominates(From, Between);
- SmallPtrSet<BasicBlock *, 1> Exclusion;
- Exclusion.insert(Between->getParent());
- return !isPotentiallyReachable(From, To, &Exclusion, DT);
-}
-
-static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
- const DominatorTree *DT) {
- Value *PtrOp = Load->getPointerOperand();
- if (!PtrOp->hasUseList())
- return nullptr;
-
- Instruction *OtherAccess = nullptr;
-
- for (auto *U : PtrOp->users()) {
- if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
- auto *I = cast<Instruction>(U);
- if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
- // Use the most immediately dominating value.
- if (OtherAccess) {
- if (DT->dominates(OtherAccess, I))
- OtherAccess = I;
- else
- assert(U == OtherAccess || DT->dominates(I, OtherAccess));
- } else
- OtherAccess = I;
- }
+/// Assuming To can be reached from both From and Between, does Between lie on
+/// every path from From to To?
+static bool liesBetween(const Instruction *From, Instruction *Between,
+ const Instruction *To, const DominatorTree *DT) {
+ if (From->getParent() == Between->getParent())
+ return DT->dominates(From, Between);
+ SmallPtrSet<BasicBlock *, 1> Exclusion;
+ Exclusion.insert(Between->getParent());
+ return !isPotentiallyReachable(From, To, &Exclusion, DT);
+}
+
+static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
+ const DominatorTree *DT) {
+ Value *PtrOp = Load->getPointerOperand();
+ if (!PtrOp->hasUseList())
+ return nullptr;
+
+ Instruction *OtherAccess = nullptr;
+
+ for (auto *U : PtrOp->users()) {
+ if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
+ auto *I = cast<Instruction>(U);
+ if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
+ // Use the most immediately dominating value.
+ if (OtherAccess) {
+ if (DT->dominates(OtherAccess, I))
+ OtherAccess = I;
+ else
+ assert(U == OtherAccess || DT->dominates(I, OtherAccess));
+ } else
+ OtherAccess = I;
+ }
}
}
@@ -1682,7 +1837,7 @@ void GVNPass::eliminatePartiallyRedundantLoad(
}
// Perform PHI construction.
- Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
// constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
@@ -1699,1076 +1854,1010 @@ void GVNPass::eliminatePartiallyRedundantLoad(
salvageAndRemoveInstruction(Load);
}
-bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
- // Okay, we have *some* definitions of the value. This means that the value
- // is available in some of our (transitive) predecessors. Lets think about
- // doing PRE of this load. This will involve inserting a new load into the
- // predecessor when it's not available. We could do this in general, but
- // prefer to not increase code size. As such, we only do this when we know
- // that we only have to insert *one* load (which means we're basically moving
- // the load, not inserting a new one).
-
- SmallPtrSet<BasicBlock *, 4> Blockers(llvm::from_range, UnavailableBlocks);
+static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
+ OptimizationRemarkEmitter *ORE) {
+ using namespace ore;
- // Let's find the first basic block with more than one predecessor. Walk
- // backwards through predecessors if needed.
- BasicBlock *LoadBB = Load->getParent();
- BasicBlock *TmpBB = LoadBB;
+ ORE->emit([&]() {
+ return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
+ << "load of type " << NV("Type", Load->getType()) << " eliminated"
+ << setExtraArgs() << " in favor of "
+ << NV("InfavorOfValue", AvailableValue);
+ });
+}
- // Check that there is no implicit control flow instructions above our load in
- // its block. If there is an instruction that doesn't always pass the
- // execution to the following instruction, then moving through it may become
- // invalid. For example:
- //
- // int arr[LEN];
- // int index = ???;
- // ...
- // guard(0 <= index && index < LEN);
- // use(arr[index]);
- //
- // It is illegal to move the array access to any point above the guard,
- // because if the index is out of bounds we should deoptimize rather than
- // access the array.
- // Check that there is no guard in this block above our instruction.
- bool MustEnsureSafetyOfSpeculativeExecution =
- ICF->isDominatedByICFIFromSameBlock(Load);
+/// If a load has !invariant.group, try to find the most-dominating instruction
+/// with the same metadata and equivalent pointer (modulo bitcasts and zero
+/// GEPs). If one is found that dominates the load, its value can be reused.
+static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) {
+ Value *PointerOperand = L->getPointerOperand()->stripPointerCasts();
- while (TmpBB->getSinglePredecessor()) {
- TmpBB = TmpBB->getSinglePredecessor();
- if (TmpBB == LoadBB) // Infinite (unreachable) loop.
- return false;
- if (Blockers.count(TmpBB))
- return false;
+ // It's not safe to walk the use list of a global value because function
+ // passes aren't allowed to look outside their functions.
+ // FIXME: this could be fixed by filtering instructions from outside of
+ // current function.
+ if (isa<Constant>(PointerOperand))
+ return nullptr;
- // If any of these blocks has more than one successor (i.e. if the edge we
- // just traversed was critical), then there are other paths through this
- // block along which the load may not be anticipated. Hoisting the load
- // above this block would be adding the load to execution paths along
- // which it was not previously executed.
- if (TmpBB->getTerminator()->getNumSuccessors() != 1)
- return false;
+ // Queue to process all pointers that are equivalent to load operand.
+ SmallVector<Value *, 8> PointerUsesQueue;
+ PointerUsesQueue.push_back(PointerOperand);
- // Check that there is no implicit control flow in a block above.
- MustEnsureSafetyOfSpeculativeExecution =
- MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB);
- }
+ Instruction *MostDominatingInstruction = L;
- assert(TmpBB);
- LoadBB = TmpBB;
+ // FIXME: This loop is potentially O(n^2) due to repeated dominates checks.
+ while (!PointerUsesQueue.empty()) {
+ Value *Ptr = PointerUsesQueue.pop_back_val();
+ assert(Ptr && !isa<GlobalValue>(Ptr) &&
+ "Null or GlobalValue should not be inserted");
- // Check to see how many predecessors have the loaded value fully
- // available.
- MapVector<BasicBlock *, Value *> PredLoads;
- DenseMap<BasicBlock *, AvailabilityState> FullyAvailableBlocks;
- for (const AvailableValueInBlock &AV : ValuesPerBlock)
- FullyAvailableBlocks[AV.BB] = AvailabilityState::Available;
- for (BasicBlock *UnavailableBB : UnavailableBlocks)
- FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable;
+ for (User *U : Ptr->users()) {
+ auto *I = dyn_cast<Instruction>(U);
+ if (!I || I == L || !DT.dominates(I, MostDominatingInstruction))
+ continue;
- // The edge from Pred to LoadBB is a critical edge will be splitted.
- SmallVector<BasicBlock *, 4> CriticalEdgePredSplit;
- // The edge from Pred to LoadBB is a critical edge, another successor of Pred
- // contains a load can be moved to Pred. This data structure maps the Pred to
- // the movable load.
- MapVector<BasicBlock *, LoadInst *> CriticalEdgePredAndLoad;
- for (BasicBlock *Pred : predecessors(LoadBB)) {
- // If any predecessor block is an EH pad that does not allow non-PHI
- // instructions before the terminator, we can't PRE the load.
- if (Pred->getTerminator()->isEHPad()) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+ // Add bitcasts and zero GEPs to queue.
+ // TODO: Should drop bitcast?
+ if (isa<BitCastInst>(I) ||
+ (isa<GetElementPtrInst>(I) &&
+ cast<GetElementPtrInst>(I)->hasAllZeroIndices())) {
+ PointerUsesQueue.push_back(I);
+ continue;
+ }
- if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
- continue;
+ // If we hit a load/store with an invariant.group metadata and the same
+ // pointer operand, we can assume that value pointed to by the pointer
+ // operand didn't change.
+ if (I->hasMetadata(LLVMContext::MD_invariant_group) &&
+ Ptr == getLoadStorePointerOperand(I) && !I->isVolatile())
+ MostDominatingInstruction = I;
}
+ }
- if (Pred->getTerminator()->getNumSuccessors() != 1) {
- if (isa<IndirectBrInst>(Pred->getTerminator())) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
-
- if (LoadBB->isEHPad()) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+ return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr;
+}
- // Do not split backedge as it will break the canonical loop form.
- if (!isLoadPRESplitBackedgeEnabled())
- if (DT->dominates(LoadBB, Pred)) {
- LLVM_DEBUG(
- dbgs()
- << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+/// Return the memory location accessed by the (masked) load/store instruction
+/// `I`, if the instruction could potentially provide a useful value for
+/// eliminating the load.
+static std::optional<MemoryLocation>
+maybeLoadStoreLocation(Instruction *I, bool AllowStores,
+ const TargetLibraryInfo *TLI) {
+ if (auto *LI = dyn_cast<LoadInst>(I))
+ return MemoryLocation::get(LI);
- if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load))
- CriticalEdgePredAndLoad[Pred] = LI;
- else
- CriticalEdgePredSplit.push_back(Pred);
- } else {
- // Only add the predecessors that will not be split for now.
- PredLoads[Pred] = nullptr;
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::masked_load:
+ return MemoryLocation::getForArgument(II, 0, TLI);
+ case Intrinsic::masked_store:
+ if (AllowStores)
+ return MemoryLocation::getForArgument(II, 1, TLI);
+ return std::nullopt;
+ default:
+ break;
}
}
- // Decide whether PRE is profitable for this load.
- unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size();
- unsigned NumUnavailablePreds = NumInsertPreds +
- CriticalEdgePredAndLoad.size();
- assert(NumUnavailablePreds != 0 &&
- "Fully available value should already be eliminated!");
- (void)NumUnavailablePreds;
-
- // If we need to insert new load in multiple predecessors, reject it.
- // FIXME: If we could restructure the CFG, we could make a common pred with
- // all the preds that don't have an available Load and insert a new load into
- // that one block.
- if (NumInsertPreds > 1)
- return false;
-
- // Now we know where we will insert load. We must ensure that it is safe
- // to speculatively execute the load at that points.
- if (MustEnsureSafetyOfSpeculativeExecution) {
- if (CriticalEdgePredSplit.size())
- if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC,
- DT))
- return false;
- for (auto &PL : PredLoads)
- if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC,
- DT))
- return false;
- for (auto &CEP : CriticalEdgePredAndLoad)
- if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC,
- DT))
- return false;
- }
-
- // Split critical edges, and update the unavailable predecessors accordingly.
- for (BasicBlock *OrigPred : CriticalEdgePredSplit) {
- BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB);
- assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!");
- PredLoads[NewPred] = nullptr;
- LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->"
- << LoadBB->getName() << '\n');
- }
-
- for (auto &CEP : CriticalEdgePredAndLoad)
- PredLoads[CEP.first] = nullptr;
+ if (!AllowStores)
+ return std::nullopt;
- // Check if the load can safely be moved to all the unavailable predecessors.
- bool CanDoPRE = true;
- const DataLayout &DL = Load->getDataLayout();
- SmallVector<Instruction*, 8> NewInsts;
- for (auto &PredLoad : PredLoads) {
- BasicBlock *UnavailablePred = PredLoad.first;
+ if (auto *SI = dyn_cast<StoreInst>(I))
+ return MemoryLocation::get(SI);
+ return std::nullopt;
+}
- // Do PHI translation to get its value in the predecessor if necessary. The
- // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
- // We do the translation for each edge we skipped by going from Load's block
- // to LoadBB, otherwise we might miss pieces needing translation.
+/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
+/// looking for memory reads whose location aliases `Loc` and dominates our
+/// load.
+std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
+ const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
+ const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
+ BatchAAResults &AA, LoadInst *L) {
- // If all preds have a single successor, then we know it is safe to insert
- // the load on the pred (?!?), so we can insert code to materialize the
- // pointer if it is not available.
- Value *LoadPtr = Load->getPointerOperand();
- BasicBlock *Cur = Load->getParent();
- while (Cur != LoadBB) {
- PHITransAddr Address(LoadPtr, DL, AC);
- LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(),
- *DT, NewInsts);
- if (!LoadPtr) {
- CanDoPRE = false;
- break;
- }
- Cur = Cur->getSinglePredecessor();
+ // Prefer a candidate that is closer to the load within the same block.
+ auto UpdateChoice = [&](std::optional<ReachingMemVal> &Choice,
+ AliasResult &AR, Instruction *Candidate) {
+ if (!Choice) {
+ if (AR == AliasResult::PartialAlias)
+ Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset());
+ else
+ Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate);
+ return;
}
+ if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst),
+ MSSA.getMemoryAccess(Candidate)))
+ return;
- if (LoadPtr) {
- PHITransAddr Address(LoadPtr, DL, AC);
- LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT,
- NewInsts);
- }
- // If we couldn't find or insert a computation of this phi translated value,
- // we fail PRE.
- if (!LoadPtr) {
- LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
- << *Load->getPointerOperand() << "\n");
- CanDoPRE = false;
- break;
+ if (AR == AliasResult::PartialAlias) {
+ Choice->Kind = DepKind::Clobber;
+ Choice->Offset = AR.getOffset();
+ } else {
+ Choice->Kind = DepKind::Def;
+ Choice->Offset = -1;
}
- PredLoad.second = LoadPtr;
- }
+ Choice->Inst = Candidate;
+ Choice->Block = Candidate->getParent();
+ };
- if (!CanDoPRE) {
- while (!NewInsts.empty()) {
- // Erase instructions generated by the failed PHI translation before
- // trying to number them. PHI translation might insert instructions
- // in basic blocks other than the current one, and we delete them
- // directly, as salvageAndRemoveInstruction only allows removing from the
- // current basic block.
- NewInsts.pop_back_val()->eraseFromParent();
- }
- // HINT: Don't revert the edge-splitting as following transformation may
- // also need to split these critical edges.
- return !CriticalEdgePredSplit.empty();
- }
+ std::optional<ReachingMemVal> ReachingVal;
+ for (MemoryAccess *MA : ClobbersList) {
+ unsigned Scanned = 0;
+ for (User *U : MA->users()) {
+ if (++Scanned >= ScanUsersLimit)
+ return ReachingMemVal::getUnknown(BB, Loc.Ptr);
- // Okay, we can eliminate this load by inserting a reload in the predecessor
- // and using PHI construction to get the value in the other predecessors, do
- // it.
- LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
- LLVM_DEBUG(if (!NewInsts.empty()) dbgs() << "INSERTED " << NewInsts.size()
- << " INSTS: " << *NewInsts.back()
- << '\n');
+ auto *UseOrDef = dyn_cast<MemoryUseOrDef>(U);
+ if (!UseOrDef || UseOrDef->getBlock() != BB)
+ continue;
- // Assign value numbers to the new instructions.
- for (Instruction *I : NewInsts) {
- // Instructions that have been inserted in predecessor(s) to materialize
- // the load address do not retain their original debug locations. Doing
- // so could lead to confusing (but correct) source attributions.
- I->updateLocationAfterHoist();
+ Instruction *MemI = UseOrDef->getMemoryInst();
+ if (MemI == L ||
+ (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L))))
+ continue;
- // FIXME: We really _ought_ to insert these value numbers into their
- // parent's availability map. However, in doing so, we risk getting into
- // ordering issues. If a block hasn't been processed yet, we would be
- // marking a value as AVAIL-IN, which isn't what we intend.
- VN.lookupOrAdd(I);
+ if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) {
+ AliasResult AR = AA.alias(*MaybeLoc, Loc);
+ // If the locations do not certainly alias, we cannot possibly infer the
+ // following load loads the same value.
+ if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias)
+ continue;
+
+ // Locations partially overlap, but neither is a subset of the other, or
+ // the second location is before the first.
+ if (AR == AliasResult::PartialAlias &&
+ (!AR.hasOffset() || AR.getOffset() < 0))
+ continue;
+
+ // Found candidate, the new load memory location and the given location
+ // must alias: precise overlap, or subset with non-negative offset.
+ UpdateChoice(ReachingVal, AR, MemI);
+ }
+ }
+ if (ReachingVal)
+ break;
}
- eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads,
- &CriticalEdgePredAndLoad);
- ++NumPRELoad;
- return true;
+ return ReachingVal;
}
-bool GVNPass::performLoopLoadPRE(LoadInst *Load,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
- const Loop *L = LI->getLoopFor(Load->getParent());
- // TODO: Generalize to other loop blocks that dominate the latch.
- if (!L || L->getHeader() != Load->getParent())
- return false;
-
- BasicBlock *Preheader = L->getLoopPreheader();
- BasicBlock *Latch = L->getLoopLatch();
- if (!Preheader || !Latch)
- return false;
+/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
+/// given location. Returns a ReachingMemVal describing the dependency.
+std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
+ MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad,
+ BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
+ assert(ClobberMA->getBlock() == BB);
- Value *LoadPtr = Load->getPointerOperand();
- // Must be available in preheader.
- if (!L->isLoopInvariant(LoadPtr))
- return false;
+ // If the clobbering access is the entry memory state, we cannot say anything
+ // about the content of the memory, except when we are accessing a local
+ // object, which can be turned later into producing `undef`.
+ if (MSSA.isLiveOnEntryDef(ClobberMA)) {
+ if (auto *Alloc = dyn_cast<AllocaInst>(getUnderlyingObject(Loc.Ptr)))
+ if (Alloc->getParent() == BB)
+ return ReachingMemVal::getDef(Loc.Ptr, const_cast<AllocaInst *>(Alloc));
+ return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ }
- // We plan to hoist the load to preheader without introducing a new fault.
- // In order to do it, we need to prove that we cannot side-exit the loop
- // once loop header is first entered before execution of the load.
- if (ICF->isDominatedByICFIFromSameBlock(Load))
- return false;
+ // Loads from "constant" memory can't be clobbered.
+ if (IsInvariantLoad || AA.pointsToConstantMemory(Loc))
+ return std::nullopt;
- BasicBlock *LoopBlock = nullptr;
- for (auto *Blocker : UnavailableBlocks) {
- // Blockers from outside the loop are handled in preheader.
- if (!L->contains(Blocker))
- continue;
+ auto GetOrdering = [](const Instruction *I) {
+ if (auto *L = dyn_cast<LoadInst>(I))
+ return L->getOrdering();
+ return cast<StoreInst>(I)->getOrdering();
+ };
+ Instruction *ClobberI = cast<MemoryDef>(ClobberMA)->getMemoryInst();
- // Only allow one loop block. Loop header is not less frequently executed
- // than each loop block, and likely it is much more frequently executed. But
- // in case of multiple loop blocks, we need extra information (such as block
- // frequency info) to understand whether it is profitable to PRE into
- // multiple loop blocks.
- if (LoopBlock)
- return false;
+ // Check if the clobbering access is a load or a store that we can reuse.
+ if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) {
+ AliasResult AR = AA.alias(*MaybeLoc, Loc);
+ if (AR == AliasResult::MustAlias)
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
- // Do not sink into inner loops. This may be non-profitable.
- if (L != LI->getLoopFor(Blocker))
- return false;
+ if (AR == AliasResult::NoAlias) {
+ // If the locations do not alias we may still be able to skip over the
+ // clobbering instruction, even if it is atomic.
+ // The original load is either non-atomic or unordered. We can reorder
+ // these across non-atomic, unordered or monotonic loads or across any
+ // store.
+ if (!ClobberI->isAtomic() ||
+ !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) ||
+ isa<StoreInst>(ClobberI))
+ return std::nullopt;
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ }
- // Blocks that dominate the latch execute on every single iteration, maybe
- // except the last one. So PREing into these blocks doesn't make much sense
- // in most cases. But the blocks that do not necessarily execute on each
- // iteration are sometimes much colder than the header, and this is when
- // PRE is potentially profitable.
- if (DT->dominates(Blocker, Latch))
- return false;
+ // Skip over volatile loads (the original load is non-volatile, non-atomic).
+ if (!ClobberI->isAtomic() && isa<LoadInst>(ClobberI))
+ return std::nullopt;
- // Make sure that the terminator itself doesn't clobber.
- if (Blocker->getTerminator()->mayWriteToMemory())
- return false;
+ if (AR == AliasResult::MayAlias ||
+ (AR == AliasResult::PartialAlias &&
+ (!AR.hasOffset() || AR.getOffset() < 0)))
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
- LoopBlock = Blocker;
+ // The only option left is a store of the superset of the required bits.
+ assert(AR == AliasResult::PartialAlias && AR.hasOffset() &&
+ AR.getOffset() > 0 &&
+ "Must be the superset/partial overlap case with positive offset");
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset());
}
- if (!LoopBlock)
- return false;
-
- // Make sure the memory at this pointer cannot be freed, therefore we can
- // safely reload from it after clobber.
- if (LoadPtr->canBeFreed())
- return false;
+ if (auto *II = dyn_cast<IntrinsicInst>(ClobberI)) {
+ if (isa<DbgInfoIntrinsic>(II))
+ return std::nullopt;
+ if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI);
+ if (AA.isMustAlias(IIObjLoc, Loc))
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ return std::nullopt;
+ }
+ }
- // TODO: Support critical edge splitting if blocker has more than 1 successor.
- MapVector<BasicBlock *, Value *> AvailableLoads;
- AvailableLoads[LoopBlock] = LoadPtr;
- AvailableLoads[Preheader] = LoadPtr;
+ // If we are at a malloc-like function call, we can turn the load into `undef`
+ // or zero.
+ if (isNoAliasCall(ClobberI)) {
+ const Value *Obj = getUnderlyingObject(Loc.Ptr);
+ if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr))
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ }
- LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
- eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads,
- /*CriticalEdgePredAndLoad*/ nullptr);
- ++NumPRELoopLoad;
- return true;
-}
+ // Can reorder loads across a release fence.
+ if (auto *FI = dyn_cast<FenceInst>(ClobberI))
+ if (FI->getOrdering() == AtomicOrdering::Release)
+ return std::nullopt;
-static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
- OptimizationRemarkEmitter *ORE) {
- using namespace ore;
+ // See if the clobber instruction (e.g., a generic call) may modify the
+ // location.
+ ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc);
+ // If may modify the location, analyze deeper, to exclude accesses to
+ // non-escaping local allocations.
+ if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref)
+ return std::nullopt;
- ORE->emit([&]() {
- return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
- << "load of type " << NV("Type", Load->getType()) << " eliminated"
- << setExtraArgs() << " in favor of "
- << NV("InfavorOfValue", AvailableValue);
- });
+ // Conservatively assume the clobbering memory access may overwrite the
+ // location.
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
}
-/// Attempt to eliminate a load whose dependencies are
-/// non-local by performing PHI construction.
-bool GVNPass::processNonLocalLoad(LoadInst *Load) {
- // Non-local speculations are not allowed under asan.
- if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
- Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+/// Collect the predecessors of block, while doing phi-translation of the memory
+/// address and the memory clobber. Return false if the block should be marked
+/// as clobbering the memory location in an unknown way.
+bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
+ MemoryAccess *ClobberMA,
+ DependencyBlockSet &Blocks,
+ SmallVectorImpl<BasicBlock *> &Worklist) {
+ if (Addr.needsPHITranslationFromBlock(BB) &&
+ !Addr.isPotentiallyPHITranslatable())
return false;
- // Find the non-local dependencies of the load.
- LoadDepVect Deps;
- MD->getNonLocalPointerDependency(Load, Deps);
+ auto *MPhi =
+ ClobberMA->getBlock() == BB ? dyn_cast<MemoryPhi>(ClobberMA) : nullptr;
+ SmallVector<std::pair<BasicBlock *, DependencyBlockInfo>, 8> Preds;
+ for (BasicBlock *Pred : predecessors(BB)) {
+ // Skip unreachable predecessors.
+ if (!DT->isReachableFromEntry(Pred))
+ continue;
- // If we had to process more than one hundred blocks to find the
- // dependencies, this load isn't worth worrying about. Optimizing
- // it will be too expensive.
- unsigned NumDeps = Deps.size();
- if (NumDeps > MaxNumDeps)
- return false;
+ // Skip already visited predecessors.
+ if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; }))
+ continue;
- SmallVector<ReachingMemVal, 64> MemVals;
- MemVals.reserve(Deps.size());
+ PHITransAddr TransAddr = Addr;
+ if (TransAddr.needsPHITranslationFromBlock(BB))
+ TransAddr.translateValue(BB, Pred, DT, false);
- for (const NonLocalDepResult &Dep : Deps) {
- const auto &R = Dep.getResult();
- SelectAddr SelAddr = Dep.getAddress();
- BasicBlock *BB = Dep.getBB();
- Instruction *Inst = R.getInst();
- if (R.isSelect()) {
- auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs();
- MemVals.emplace_back(
- ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second));
+ auto It = Blocks.find(Pred);
+ if (It != Blocks.end()) {
+ // If we reach a visited block with a different address, set the
+ // current block as clobbering the memory location in an unknown way
+ // (by returning false).
+ if (It->second.Addr.getAddr() != TransAddr.getAddr())
+ return false;
+ // Otherwise, just stop the traversal.
continue;
}
- Value *Address = SelAddr.getAddr();
- if (R.isClobber())
- MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst));
- else if (R.isDef())
- MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst));
- else
- MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst));
- }
- return processNonLocalLoad(Load, MemVals);
-}
-
-bool GVNPass::processNonLocalLoad(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps) {
- // If we had a phi translation failure, we'll have a single entry which is a
- // clobber in the current block. Reject this early.
- if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
- LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
- dbgs() << " has unknown dependencies\n";);
- return false;
+ Preds.emplace_back(
+ Pred, DependencyBlockInfo(TransAddr,
+ MPhi ? MPhi->getIncomingValueForBlock(Pred)
+ : ClobberMA));
}
- bool Changed = false;
- // This is a limited form of scalar PRE for load indices. If this load follows
- // a GEP, see if we can PRE the indices before analyzing.
- if (isScalarPREEnabled()) {
- if (GetElementPtrInst *GEP =
- dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
- for (Use &U : GEP->indices())
- if (Instruction *I = dyn_cast<Instruction>(U.get()))
- Changed |= performScalarPRE(I);
- }
+ // We collected the predecessors and stored them in Preds. Now, populate the
+ // worklist with the predecessors found, and cache the eventual translated
+ // address for each block.
+ for (auto &P : Preds) {
+ [[maybe_unused]] auto It =
+ Blocks.try_emplace(P.first, std::move(P.second)).first;
+ Worklist.push_back(P.first);
}
- // Step 1: Analyze the availability of the load.
- AvailValInBlkVect ValuesPerBlock;
- UnavailBlkVect UnavailableBlocks;
- analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
-
- // If we have no predecessors that produce a known value for this load, exit
- // early.
- if (ValuesPerBlock.empty())
- return Changed;
+ return true;
+}
- // Step 2: Eliminate fully redundancy.
- //
- // If all of the instructions we depend on produce a known value for this
- // load, then it is fully redundant and we can use PHI insertion to compute
- // its value. Insert PHIs and remove the fully redundant value now.
- if (UnavailableBlocks.empty()) {
- LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
+/// Build a list of MemoryAccesses whose users could potentially alias the
+/// memory location being queried. Starts from StartInfo's initial clobber,
+/// walk the use-def chain to the final clobber. If the chain extends beyond
+/// `BB`, continue into that block but only if it is in the previously collected
+/// set.
+void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
+ BasicBlock *BB,
+ const DependencyBlockInfo &StartInfo,
+ const DependencyBlockSet &Blocks,
+ MemorySSA &MSSA) {
+ MemoryAccess *MA = StartInfo.InitialClobberMA;
+ MemoryAccess *LastMA = StartInfo.ClobberMA;
- // Perform PHI construction.
- Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // constructSSAForLoadSet is responsible for combining metadata.
- ICF->removeUsersOf(Load);
- Load->replaceAllUsesWith(V);
+ for (;;) {
+ while (MA != LastMA) {
+ Clobbers.push_back(MA);
+ MA = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
+ }
+ Clobbers.push_back(MA);
- if (isa<PHINode>(V))
- V->takeName(Load);
- if (Instruction *I = dyn_cast<Instruction>(V))
- // If instruction I has debug info, then we should not update it.
- // Also, if I has a null DebugLoc, then it is still potentially incorrect
- // to propagate Load's DebugLoc because Load may not post-dominate I.
- if (Load->getDebugLoc() && Load->getParent() == I->getParent())
- I->setDebugLoc(Load->getDebugLoc());
- if (MD && V->getType()->isPtrOrPtrVectorTy())
- MD->invalidateCachedPointerInfo(V);
- ++NumGVNLoad;
- reportLoadElim(Load, V, ORE);
- salvageAndRemoveInstruction(Load);
- return true;
- }
+ if (MSSA.isLiveOnEntryDef(MA) ||
+ (MA->getBlock() == BB && !isa<MemoryPhi>(MA)))
+ break;
- // Step 3: Eliminate partial redundancy.
- if (!isLoadPREEnabled())
- return Changed;
- if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
- return Changed;
+ // If the final clobber in the current block is a MemoryPhi, go to the
+ // immediate dominator; otherwise, just get to the block containing the
+ // final clobber.
+ if (MA->getBlock() == BB)
+ BB = DT->getNode(BB)->getIDom()->getBlock();
+ else
+ BB = MA->getBlock();
- if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
- performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
- return true;
+ auto It = Blocks.find(BB);
+ if (It == Blocks.end())
+ break;
- return Changed;
+ MA = It->second.InitialClobberMA;
+ LastMA = It->second.ClobberMA;
+ if (MA == Clobbers.back())
+ Clobbers.pop_back();
+ }
}
-bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
- Value *V = IntrinsicI->getArgOperand(0);
-
- if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
- if (Cond->isZero()) {
- Type *Int8Ty = Type::getInt8Ty(V->getContext());
- Type *PtrTy = PointerType::get(V->getContext(), 0);
- // Insert a new store to null instruction before the load to indicate that
- // this code is not reachable. FIXME: We could insert unreachable
- // instruction directly because we can modify the CFG.
- auto *NewS =
- new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy),
- IntrinsicI->getIterator());
- if (MSSAU) {
- const MemoryUseOrDef *FirstNonDom = nullptr;
- const auto *AL =
- MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent());
-
- // If there are accesses in the current basic block, find the first one
- // that does not come before NewS. The new memory access is inserted
- // after the found access or before the terminator if no such access is
- // found.
- if (AL) {
- for (const auto &Acc : *AL) {
- if (auto *Current = dyn_cast<MemoryUseOrDef>(&Acc))
- if (!Current->getMemoryInst()->comesBefore(NewS)) {
- FirstNonDom = Current;
- break;
- }
- }
- }
-
- auto *NewDef =
- FirstNonDom ? MSSAU->createMemoryAccessBefore(
- NewS, nullptr,
- const_cast<MemoryUseOrDef *>(FirstNonDom))
- : MSSAU->createMemoryAccessInBB(
- NewS, nullptr,
- NewS->getParent(), MemorySSA::BeforeTerminator);
+/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
+/// Given as input a load instruction, the function computes the set of reaching
+/// memory values, one per predecessor path, that analyzeLoadAvailability can
+/// later use to establish whether the load may be eliminated. A reaching value
+/// may be of the following descriptor kind:
+/// * Def: a precise instruction that produces the exact bits the load would
+/// read (e.g., an equivalent load or a MustAlias store);
+/// * Clobber: a write that clobbers a superset of the bits the load would read
+/// (e.g., a memset over a larger region);
+/// * Other: we know which block defines the memory location in some way, but
+/// could not identify a precise instruction (e.g., memory already live at
+/// function entry).
+bool GVNPass::findReachingValuesForLoad(LoadInst *L,
+ SmallVectorImpl<ReachingMemVal> &Values,
+ MemorySSA &MSSA, AAResults &AAR) {
+ EarliestEscapeAnalysis EA(*DT, LI);
+ BatchAAResults AA(AAR, &EA);
+ BasicBlock *StartBlock = L->getParent();
+ bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load);
+ // TODO: Simplify later work by just getClobberingMemoryAccess().
+ MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess();
+ const MemoryLocation Loc = MemoryLocation::get(L);
- MSSAU->insertDef(cast<MemoryDef>(NewDef), /*RenameUses=*/false);
- }
- }
- if (isAssumeWithEmptyBundle(*IntrinsicI)) {
- salvageAndRemoveInstruction(IntrinsicI);
+ // Fast path for load tagged with !invariant.group.
+ if (L->hasMetadata(LLVMContext::MD_invariant_group)) {
+ if (Instruction *G = findInvariantGroupValue(L, *DT)) {
+ Values.emplace_back(
+ ReachingMemVal::getDef(getLoadStorePointerOperand(G), G));
return true;
}
- return false;
- }
-
- if (isa<Constant>(V)) {
- // If it's not false, and constant, it must evaluate to true. This means our
- // assume is assume(true), and thus, pointless, and we don't want to do
- // anything more here.
- return false;
}
- Constant *True = ConstantInt::getTrue(V->getContext());
- return propagateEquality(V, True, IntrinsicI);
-}
+ // Phase 1. First off, look for a local dependency to avoid having to
+ // disambiguate between before the load and after the load of the starting
+ // block (as the load may be visited from a backedge).
+ do {
+ // Scan users of the clobbering memory access.
+ if (auto RMV = scanMemoryAccessesUsers(
+ Loc, IsInvariantLoad, StartBlock,
+ SmallVector<MemoryAccess *, 1>{ClobberMA}, MSSA, AA, L)) {
+ Values.emplace_back(*RMV);
+ return true;
+ }
-static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
- patchReplacementInstruction(I, Repl);
- I->replaceAllUsesWith(Repl);
-}
+ // Exit from here, and proceed visiting predecessors if the clobbering
+ // access is non-local or is a MemoryPhi.
+ if (ClobberMA->getBlock() != StartBlock || isa<MemoryPhi>(ClobberMA))
+ break;
-/// If a load has !invariant.group, try to find the most-dominating instruction
-/// with the same metadata and equivalent pointer (modulo bitcasts and zero
-/// GEPs). If one is found that dominates the load, its value can be reused.
-static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) {
- Value *PointerOperand = L->getPointerOperand()->stripPointerCasts();
+ // Check if the clobber actually aliases the load location.
+ if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad,
+ StartBlock, MSSA, AA)) {
+ Values.emplace_back(*RMV);
+ return true;
+ }
- // It's not safe to walk the use list of a global value because function
- // passes aren't allowed to look outside their functions.
- // FIXME: this could be fixed by filtering instructions from outside of
- // current function.
- if (isa<Constant>(PointerOperand))
- return nullptr;
+ // It may happen that the clobbering memory access does not actually
+ // clobber our load location, transition to its defining memory access.
+ ClobberMA = cast<MemoryUseOrDef>(ClobberMA)->getDefiningAccess();
+ } while (ClobberMA->getBlock() == StartBlock);
- // Queue to process all pointers that are equivalent to load operand.
- SmallVector<Value *, 8> PointerUsesQueue;
- PointerUsesQueue.push_back(PointerOperand);
+ // Non-local speculations are not allowed under ASan.
+ if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
+ L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ return false;
- Instruction *MostDominatingInstruction = L;
+ // Phase 2. Walk backwards through the CFG, collecting all the blocks that
+ // contain an instruction that modifies the load memory location, or that lie
+ // on a path between a clobbering block and our load. Start off by collecting
+ // the predecessors of `StartBlock`. All the visited blocks are stored in a
+ // the set `Blocks`. If possible, the memory address maintained for the block
+ // visited does get phi-translated.
+ DependencyBlockSet Blocks;
+ SmallVector<BasicBlock *, 16> InitialWorklist;
+ const DataLayout &DL = L->getModule()->getDataLayout();
+ if (!collectPredecessors(StartBlock,
+ PHITransAddr(L->getPointerOperand(), DL, AC),
+ ClobberMA, Blocks, InitialWorklist))
+ return false;
- // FIXME: This loop is potentially O(n^2) due to repeated dominates checks.
- while (!PointerUsesQueue.empty()) {
- Value *Ptr = PointerUsesQueue.pop_back_val();
- assert(Ptr && !isa<GlobalValue>(Ptr) &&
- "Null or GlobalValue should not be inserted");
+ // Do a bottom-up DFS.
+ auto Worklist = InitialWorklist;
+ while (!Worklist.empty()) {
+ auto *BB = Worklist.pop_back_val();
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
- for (User *U : Ptr->users()) {
- auto *I = dyn_cast<Instruction>(U);
- if (!I || I == L || !DT.dominates(I, MostDominatingInstruction))
- continue;
+ // Phi-translation may have failed.
+ if (!Info.Addr.getAddr())
+ continue;
- // Add bitcasts and zero GEPs to queue.
- // TODO: Should drop bitcast?
- if (isa<BitCastInst>(I) ||
- (isa<GetElementPtrInst>(I) &&
- cast<GetElementPtrInst>(I)->hasAllZeroIndices())) {
- PointerUsesQueue.push_back(I);
+ // If the clobbering memory access is in the current block and it indeed
+ // clobbers our load location, record the dependency and do not visit the
+ // predecessors of this block further, continue with the blocks in the
+ // worklist.
+ if (Info.ClobberMA->getBlock() == BB && !isa<MemoryPhi>(Info.ClobberMA)) {
+ if (auto RMV = accessMayModifyLocation(
+ Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()),
+ IsInvariantLoad, BB, MSSA, AA)) {
+ Info.MemVal = RMV;
continue;
}
+ assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) &&
+ "LiveOnEntry aliases everything");
- // If we hit a load/store with an invariant.group metadata and the same
- // pointer operand, we can assume that value pointed to by the pointer
- // operand didn't change.
- if (I->hasMetadata(LLVMContext::MD_invariant_group) &&
- Ptr == getLoadStorePointerOperand(I) && !I->isVolatile())
- MostDominatingInstruction = I;
+ // If, however, the clobbering memory access does not actually clobber
+ // our load location, transition to its defining memory access, but
+ // keep examining the same basic block.
+ Info.ClobberMA =
+ cast<MemoryUseOrDef>(Info.ClobberMA)->getDefiningAccess();
+ Worklist.emplace_back(BB);
+ continue;
}
- }
-
- return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr;
-}
-
-/// Return the memory location accessed by the (masked) load/store instruction
-/// `I`, if the instruction could potentially provide a useful value for
-/// eliminating the load.
-static std::optional<MemoryLocation>
-maybeLoadStoreLocation(Instruction *I, bool AllowStores,
- const TargetLibraryInfo *TLI) {
- if (auto *LI = dyn_cast<LoadInst>(I))
- return MemoryLocation::get(LI);
- if (auto *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::masked_load:
- return MemoryLocation::getForArgument(II, 0, TLI);
- case Intrinsic::masked_store:
- if (AllowStores)
- return MemoryLocation::getForArgument(II, 1, TLI);
- return std::nullopt;
- default:
- break;
+ // At this point we know the current block is "transparent", i.e. the memory
+ // location is not modified when execution goes through this block.
+ // Continue to its predecessors, unless a predecessor has already been
+ // visited with a different address. We currently cannot represent such a
+ // dependency.
+ if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) {
+ Info.ForceUnknown = true;
+ continue;
}
+ if (BB != StartBlock &&
+ !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist))
+ Info.ForceUnknown = true;
}
- if (!AllowStores)
- return std::nullopt;
-
- if (auto *SI = dyn_cast<StoreInst>(I))
- return MemoryLocation::get(SI);
- return std::nullopt;
-}
+ // Phase 3. We have collected all the blocks that either write a value to the
+ // memory location of the load, or there exists a path to the load, along
+ // which the memory location is not modified. Perform a second DFS to find
+ // load-to-load dependencies; namely, look at the dominating memory reads,
+ // that alias our load. These are the MemoryUses that are users of the
+ // MemoryDefs we previously identified. If no memory read is encountered,
+ // either confirm the clobbering write found before or set to unknown.
+ Worklist = InitialWorklist;
+ for (BasicBlock *BB : Worklist) {
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
+ Info.Visited = true;
+ }
-/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
-/// looking for memory reads whose location aliases `Loc` and dominates our
-/// load.
-std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
- const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
- const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
- BatchAAResults &AA, LoadInst *L) {
+ SmallVector<MemoryAccess *> Clobbers;
+ while (!Worklist.empty()) {
+ auto *BB = Worklist.pop_back_val();
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
- // Prefer a candidate that is closer to the load within the same block.
- auto UpdateChoice = [&](std::optional<ReachingMemVal> &Choice,
- AliasResult &AR, Instruction *Candidate) {
- if (!Choice) {
- if (AR == AliasResult::PartialAlias)
- Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset());
- else
- Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate);
- return;
+ // If phi-translation failed, assume the memory location is modified in
+ // unknown way.
+ if (!Info.Addr.getAddr()) {
+ Values.push_back(ReachingMemVal::getUnknown(BB, nullptr));
+ continue;
}
- if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst),
- MSSA.getMemoryAccess(Candidate)))
- return;
- if (AR == AliasResult::PartialAlias) {
- Choice->Kind = DepKind::Clobber;
- Choice->Offset = AR.getOffset();
- } else {
- Choice->Kind = DepKind::Def;
- Choice->Offset = -1;
+ Clobbers.clear();
+ collectClobberList(Clobbers, BB, Info, Blocks, MSSA);
+ if (auto RMV =
+ scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()),
+ IsInvariantLoad, BB, Clobbers, MSSA, AA)) {
+ Values.push_back(*RMV);
+ continue;
}
- Choice->Inst = Candidate;
- Choice->Block = Candidate->getParent();
- };
+ // If no reusable memory use was found, and the current block is not
+ // transparent, use the already established memory def.
+ if (Info.MemVal) {
+ Values.push_back(*Info.MemVal);
+ continue;
+ }
- std::optional<ReachingMemVal> ReachingVal;
- for (MemoryAccess *MA : ClobbersList) {
- unsigned Scanned = 0;
- for (User *U : MA->users()) {
- if (++Scanned >= ScanUsersLimit)
- return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ if (Info.ForceUnknown) {
+ Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr()));
+ continue;
+ }
- auto *UseOrDef = dyn_cast<MemoryUseOrDef>(U);
- if (!UseOrDef || UseOrDef->getBlock() != BB)
+ // If the current block is transparent, continue to its predecessors.
+ for (BasicBlock *Pred : predecessors(BB)) {
+ auto It = Blocks.find(Pred);
+ if (It == Blocks.end())
continue;
-
- Instruction *MemI = UseOrDef->getMemoryInst();
- if (MemI == L ||
- (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L))))
+ DependencyBlockInfo &PredInfo = It->second;
+ if (PredInfo.Visited)
continue;
+ PredInfo.Visited = true;
+ Worklist.push_back(Pred);
+ }
+ }
- if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) {
- AliasResult AR = AA.alias(*MaybeLoc, Loc);
- // If the locations do not certainly alias, we cannot possibly infer the
- // following load loads the same value.
- if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias)
- continue;
+ return true;
+}
- // Locations partially overlap, but neither is a subset of the other, or
- // the second location is before the first.
- if (AR == AliasResult::PartialAlias &&
- (!AR.hasOffset() || AR.getOffset() < 0))
- continue;
+bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
+ // Okay, we have *some* definitions of the value. This means that the value
+ // is available in some of our (transitive) predecessors. Lets think about
+ // doing PRE of this load. This will involve inserting a new load into the
+ // predecessor when it's not available. We could do this in general, but
+ // prefer to not increase code size. As such, we only do this when we know
+ // that we only have to insert *one* load (which means we're basically moving
+ // the load, not inserting a new one).
- // Found candidate, the new load memory location and the given location
- // must alias: precise overlap, or subset with non-negative offset.
- UpdateChoice(ReachingVal, AR, MemI);
- }
- }
- if (ReachingVal)
- break;
- }
+ SmallPtrSet<BasicBlock *, 4> Blockers(llvm::from_range, UnavailableBlocks);
- return ReachingVal;
-}
+ // Let's find the first basic block with more than one predecessor. Walk
+ // backwards through predecessors if needed.
+ BasicBlock *LoadBB = Load->getParent();
+ BasicBlock *TmpBB = LoadBB;
-/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
-/// given location. Returns a ReachingMemVal describing the dependency.
-std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
- MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad,
- BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
- assert(ClobberMA->getBlock() == BB);
+ // Check that there is no implicit control flow instructions above our load in
+ // its block. If there is an instruction that doesn't always pass the
+ // execution to the following instruction, then moving through it may become
+ // invalid. For example:
+ //
+ // int arr[LEN];
+ // int index = ???;
+ // ...
+ // guard(0 <= index && index < LEN);
+ // use(arr[index]);
+ //
+ // It is illegal to move the array access to any point above the guard,
+ // because if the index is out of bounds we should deoptimize rather than
+ // access the array.
+ // Check that there is no guard in this block above our instruction.
+ bool MustEnsureSafetyOfSpeculativeExecution =
+ ICF->isDominatedByICFIFromSameBlock(Load);
- // If the clobbering access is the entry memory state, we cannot say anything
- // about the content of the memory, except when we are accessing a local
- // object, which can be turned later into producing `undef`.
- if (MSSA.isLiveOnEntryDef(ClobberMA)) {
- if (auto *Alloc = dyn_cast<AllocaInst>(getUnderlyingObject(Loc.Ptr)))
- if (Alloc->getParent() == BB)
- return ReachingMemVal::getDef(Loc.Ptr, const_cast<AllocaInst *>(Alloc));
- return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ while (TmpBB->getSinglePredecessor()) {
+ TmpBB = TmpBB->getSinglePredecessor();
+ if (TmpBB == LoadBB) // Infinite (unreachable) loop.
+ return false;
+ if (Blockers.count(TmpBB))
+ return false;
+
+ // If any of these blocks has more than one successor (i.e. if the edge we
+ // just traversed was critical), then there are other paths through this
+ // block along which the load may not be anticipated. Hoisting the load
+ // above this block would be adding the load to execution paths along
+ // which it was not previously executed.
+ if (TmpBB->getTerminator()->getNumSuccessors() != 1)
+ return false;
+
+ // Check that there is no implicit control flow in a block above.
+ MustEnsureSafetyOfSpeculativeExecution =
+ MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB);
}
- // Loads from "constant" memory can't be clobbered.
- if (IsInvariantLoad || AA.pointsToConstantMemory(Loc))
- return std::nullopt;
+ assert(TmpBB);
+ LoadBB = TmpBB;
- auto GetOrdering = [](const Instruction *I) {
- if (auto *L = dyn_cast<LoadInst>(I))
- return L->getOrdering();
- return cast<StoreInst>(I)->getOrdering();
- };
- Instruction *ClobberI = cast<MemoryDef>(ClobberMA)->getMemoryInst();
+ // Check to see how many predecessors have the loaded value fully
+ // available.
+ MapVector<BasicBlock *, Value *> PredLoads;
+ DenseMap<BasicBlock *, AvailabilityState> FullyAvailableBlocks;
+ for (const AvailableValueInBlock &AV : ValuesPerBlock)
+ FullyAvailableBlocks[AV.BB] = AvailabilityState::Available;
+ for (BasicBlock *UnavailableBB : UnavailableBlocks)
+ FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable;
- // Check if the clobbering access is a load or a store that we can reuse.
- if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) {
- AliasResult AR = AA.alias(*MaybeLoc, Loc);
- if (AR == AliasResult::MustAlias)
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ // The edge from Pred to LoadBB is a critical edge will be splitted.
+ SmallVector<BasicBlock *, 4> CriticalEdgePredSplit;
+ // The edge from Pred to LoadBB is a critical edge, another successor of Pred
+ // contains a load can be moved to Pred. This data structure maps the Pred to
+ // the movable load.
+ MapVector<BasicBlock *, LoadInst *> CriticalEdgePredAndLoad;
+ for (BasicBlock *Pred : predecessors(LoadBB)) {
+ // If any predecessor block is an EH pad that does not allow non-PHI
+ // instructions before the terminator, we can't PRE the load.
+ if (Pred->getTerminator()->isEHPad()) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (AR == AliasResult::NoAlias) {
- // If the locations do not alias we may still be able to skip over the
- // clobbering instruction, even if it is atomic.
- // The original load is either non-atomic or unordered. We can reorder
- // these across non-atomic, unordered or monotonic loads or across any
- // store.
- if (!ClobberI->isAtomic() ||
- !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) ||
- isa<StoreInst>(ClobberI))
- return std::nullopt;
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
+ continue;
}
- // Skip over volatile loads (the original load is non-volatile, non-atomic).
- if (!ClobberI->isAtomic() && isa<LoadInst>(ClobberI))
- return std::nullopt;
+ if (Pred->getTerminator()->getNumSuccessors() != 1) {
+ if (isa<IndirectBrInst>(Pred->getTerminator())) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (AR == AliasResult::MayAlias ||
- (AR == AliasResult::PartialAlias &&
- (!AR.hasOffset() || AR.getOffset() < 0)))
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ if (LoadBB->isEHPad()) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- // The only option left is a store of the superset of the required bits.
- assert(AR == AliasResult::PartialAlias && AR.hasOffset() &&
- AR.getOffset() > 0 &&
- "Must be the superset/partial overlap case with positive offset");
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset());
- }
+ // Do not split backedge as it will break the canonical loop form.
+ if (!isLoadPRESplitBackedgeEnabled())
+ if (DT->dominates(LoadBB, Pred)) {
+ LLVM_DEBUG(
+ dbgs()
+ << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (auto *II = dyn_cast<IntrinsicInst>(ClobberI)) {
- if (isa<DbgInfoIntrinsic>(II))
- return std::nullopt;
- if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
- MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI);
- if (AA.isMustAlias(IIObjLoc, Loc))
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
- return std::nullopt;
+ if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load))
+ CriticalEdgePredAndLoad[Pred] = LI;
+ else
+ CriticalEdgePredSplit.push_back(Pred);
+ } else {
+ // Only add the predecessors that will not be split for now.
+ PredLoads[Pred] = nullptr;
}
}
- // If we are at a malloc-like function call, we can turn the load into `undef`
- // or zero.
- if (isNoAliasCall(ClobberI)) {
- const Value *Obj = getUnderlyingObject(Loc.Ptr);
- if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr))
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ // Decide whether PRE is profitable for this load.
+ unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size();
+ unsigned NumUnavailablePreds =
+ NumInsertPreds + CriticalEdgePredAndLoad.size();
+ assert(NumUnavailablePreds != 0 &&
+ "Fully available value should already be eliminated!");
+ (void)NumUnavailablePreds;
+
+ // If we need to insert new load in multiple predecessors, reject it.
+ // FIXME: If we could restructure the CFG, we could make a common pred with
+ // all the preds that don't have an available Load and insert a new load into
+ // that one block.
+ if (NumInsertPreds > 1)
+ return false;
+
+ // Now we know where we will insert load. We must ensure that it is safe
+ // to speculatively execute the load at that points.
+ if (MustEnsureSafetyOfSpeculativeExecution) {
+ if (CriticalEdgePredSplit.size())
+ if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC,
+ DT))
+ return false;
+ for (auto &PL : PredLoads)
+ if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC,
+ DT))
+ return false;
+ for (auto &CEP : CriticalEdgePredAndLoad)
+ if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC,
+ DT))
+ return false;
}
- // Can reorder loads across a release fence.
- if (auto *FI = dyn_cast<FenceInst>(ClobberI))
- if (FI->getOrdering() == AtomicOrdering::Release)
- return std::nullopt;
+ // Split critical edges, and update the unavailable predecessors accordingly.
+ for (BasicBlock *OrigPred : CriticalEdgePredSplit) {
+ BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB);
+ assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!");
+ PredLoads[NewPred] = nullptr;
+ LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->"
+ << LoadBB->getName() << '\n');
+ }
- // See if the clobber instruction (e.g., a generic call) may modify the
- // location.
- ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc);
- // If may modify the location, analyze deeper, to exclude accesses to
- // non-escaping local allocations.
- if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref)
- return std::nullopt;
+ for (auto &CEP : CriticalEdgePredAndLoad)
+ PredLoads[CEP.first] = nullptr;
- // Conservatively assume the clobbering memory access may overwrite the
- // location.
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
-}
+ // Check if the load can safely be moved to all the unavailable predecessors.
+ bool CanDoPRE = true;
+ const DataLayout &DL = Load->getDataLayout();
+ SmallVector<Instruction *, 8> NewInsts;
+ for (auto &PredLoad : PredLoads) {
+ BasicBlock *UnavailablePred = PredLoad.first;
-/// Collect the predecessors of block, while doing phi-translation of the memory
-/// address and the memory clobber. Return false if the block should be marked
-/// as clobbering the memory location in an unknown way.
-bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
- MemoryAccess *ClobberMA,
- DependencyBlockSet &Blocks,
- SmallVectorImpl<BasicBlock *> &Worklist) {
- if (Addr.needsPHITranslationFromBlock(BB) &&
- !Addr.isPotentiallyPHITranslatable())
- return false;
+ // Do PHI translation to get its value in the predecessor if necessary. The
+ // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
+ // We do the translation for each edge we skipped by going from Load's block
+ // to LoadBB, otherwise we might miss pieces needing translation.
- auto *MPhi =
- ClobberMA->getBlock() == BB ? dyn_cast<MemoryPhi>(ClobberMA) : nullptr;
- SmallVector<std::pair<BasicBlock *, DependencyBlockInfo>, 8> Preds;
- for (BasicBlock *Pred : predecessors(BB)) {
- // Skip unreachable predecessors.
- if (!DT->isReachableFromEntry(Pred))
- continue;
+ // If all preds have a single successor, then we know it is safe to insert
+ // the load on the pred (?!?), so we can insert code to materialize the
+ // pointer if it is not available.
+ Value *LoadPtr = Load->getPointerOperand();
+ BasicBlock *Cur = Load->getParent();
+ while (Cur != LoadBB) {
+ PHITransAddr Address(LoadPtr, DL, AC);
+ LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(),
+ *DT, NewInsts);
+ if (!LoadPtr) {
+ CanDoPRE = false;
+ break;
+ }
+ Cur = Cur->getSinglePredecessor();
+ }
- // Skip already visited predecessors.
- if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; }))
- continue;
+ if (LoadPtr) {
+ PHITransAddr Address(LoadPtr, DL, AC);
+ LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT,
+ NewInsts);
+ }
+ // If we couldn't find or insert a computation of this phi translated value,
+ // we fail PRE.
+ if (!LoadPtr) {
+ LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
+ << *Load->getPointerOperand() << "\n");
+ CanDoPRE = false;
+ break;
+ }
- PHITransAddr TransAddr = Addr;
- if (TransAddr.needsPHITranslationFromBlock(BB))
- TransAddr.translateValue(BB, Pred, DT, false);
+ PredLoad.second = LoadPtr;
+ }
- auto It = Blocks.find(Pred);
- if (It != Blocks.end()) {
- // If we reach a visited block with a different address, set the
- // current block as clobbering the memory location in an unknown way
- // (by returning false).
- if (It->second.Addr.getAddr() != TransAddr.getAddr())
- return false;
- // Otherwise, just stop the traversal.
- continue;
+ if (!CanDoPRE) {
+ while (!NewInsts.empty()) {
+ // Erase instructions generated by the failed PHI translation before
+ // trying to number them. PHI translation might insert instructions
+ // in basic blocks other than the current one, and we delete them
+ // directly, as salvageAndRemoveInstruction only allows removing from the
+ // current basic block.
+ NewInsts.pop_back_val()->eraseFromParent();
}
-
- Preds.emplace_back(
- Pred, DependencyBlockInfo(TransAddr,
- MPhi ? MPhi->getIncomingValueForBlock(Pred)
- : ClobberMA));
+ // HINT: Don't revert the edge-splitting as following transformation may
+ // also need to split these critical edges.
+ return !CriticalEdgePredSplit.empty();
}
- // We collected the predecessors and stored them in Preds. Now, populate the
- // worklist with the predecessors found, and cache the eventual translated
- // address for each block.
- for (auto &P : Preds) {
- [[maybe_unused]] auto It =
- Blocks.try_emplace(P.first, std::move(P.second)).first;
- Worklist.push_back(P.first);
+ // Okay, we can eliminate this load by inserting a reload in the predecessor
+ // and using PHI construction to get the value in the other predecessors, do
+ // it.
+ LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
+ LLVM_DEBUG(if (!NewInsts.empty()) dbgs()
+ << "INSERTED " << NewInsts.size() << " INSTS: " << *NewInsts.back()
+ << '\n');
+
+ // Assign value numbers to the new instructions.
+ for (Instruction *I : NewInsts) {
+ // Instructions that have been inserted in predecessor(s) to materialize
+ // the load address do not retain their original debug locations. Doing
+ // so could lead to confusing (but correct) source attributions.
+ I->updateLocationAfterHoist();
+
+ // FIXME: We really _ought_ to insert these value numbers into their
+ // parent's availability map. However, in doing so, we risk getting into
+ // ordering issues. If a block hasn't been processed yet, we would be
+ // marking a value as AVAIL-IN, which isn't what we intend.
+ VN.lookupOrAdd(I);
}
+ eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads,
+ &CriticalEdgePredAndLoad);
+ ++NumPRELoad;
return true;
}
-/// Build a list of MemoryAccesses whose users could potentially alias the
-/// memory location being queried. Starts from StartInfo's initial clobber,
-/// walk the use-def chain to the final clobber. If the chain extends beyond
-/// `BB`, continue into that block but only if it is in the previously collected
-/// set.
-void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
- BasicBlock *BB,
- const DependencyBlockInfo &StartInfo,
- const DependencyBlockSet &Blocks,
- MemorySSA &MSSA) {
- MemoryAccess *MA = StartInfo.InitialClobberMA;
- MemoryAccess *LastMA = StartInfo.ClobberMA;
-
- for (;;) {
- while (MA != LastMA) {
- Clobbers.push_back(MA);
- MA = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
- }
- Clobbers.push_back(MA);
-
- if (MSSA.isLiveOnEntryDef(MA) ||
- (MA->getBlock() == BB && !isa<MemoryPhi>(MA)))
- break;
+bool GVNPass::performLoopLoadPRE(LoadInst *Load,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
+ const Loop *L = LI->getLoopFor(Load->getParent());
+ // TODO: Generalize to other loop blocks that dominate the latch.
+ if (!L || L->getHeader() != Load->getParent())
+ return false;
- // If the final clobber in the current block is a MemoryPhi, go to the
- // immediate dominator; otherwise, just get to the block containing the
- // final clobber.
- if (MA->getBlock() == BB)
- BB = DT->getNode(BB)->getIDom()->getBlock();
- else
- BB = MA->getBlock();
+ BasicBlock *Preheader = L->getLoopPreheader();
+ BasicBlock *Latch = L->getLoopLatch();
+ if (!Preheader || !Latch)
+ return false;
- auto It = Blocks.find(BB);
- if (It == Blocks.end())
- break;
+ Value *LoadPtr = Load->getPointerOperand();
+ // Must be available in preheader.
+ if (!L->isLoopInvariant(LoadPtr))
+ return false;
- MA = It->second.InitialClobberMA;
- LastMA = It->second.ClobberMA;
- if (MA == Clobbers.back())
- Clobbers.pop_back();
- }
-}
+ // We plan to hoist the load to preheader without introducing a new fault.
+ // In order to do it, we need to prove that we cannot side-exit the loop
+ // once loop header is first entered before execution of the load.
+ if (ICF->isDominatedByICFIFromSameBlock(Load))
+ return false;
-/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
-/// Given as input a load instruction, the function computes the set of reaching
-/// memory values, one per predecessor path, that analyzeLoadAvailability can
-/// later use to establish whether the load may be eliminated. A reaching value
-/// may be of the following descriptor kind:
-/// * Def: a precise instruction that produces the exact bits the load would
-/// read (e.g., an equivalent load or a MustAlias store);
-/// * Clobber: a write that clobbers a superset of the bits the load would read
-/// (e.g., a memset over a larger region);
-/// * Other: we know which block defines the memory location in some way, but
-/// could not identify a precise instruction (e.g., memory already live at
-/// function entry).
-bool GVNPass::findReachingValuesForLoad(LoadInst *L,
- SmallVectorImpl<ReachingMemVal> &Values,
- MemorySSA &MSSA, AAResults &AAR) {
- EarliestEscapeAnalysis EA(*DT, LI);
- BatchAAResults AA(AAR, &EA);
- BasicBlock *StartBlock = L->getParent();
- bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load);
- // TODO: Simplify later work by just getClobberingMemoryAccess().
- MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess();
- const MemoryLocation Loc = MemoryLocation::get(L);
+ BasicBlock *LoopBlock = nullptr;
+ for (auto *Blocker : UnavailableBlocks) {
+ // Blockers from outside the loop are handled in preheader.
+ if (!L->contains(Blocker))
+ continue;
- // Fast path for load tagged with !invariant.group.
- if (L->hasMetadata(LLVMContext::MD_invariant_group)) {
- if (Instruction *G = findInvariantGroupValue(L, *DT)) {
- Values.emplace_back(
- ReachingMemVal::getDef(getLoadStorePointerOperand(G), G));
- return true;
- }
- }
+ // Only allow one loop block. Loop header is not less frequently executed
+ // than each loop block, and likely it is much more frequently executed. But
+ // in case of multiple loop blocks, we need extra information (such as block
+ // frequency info) to understand whether it is profitable to PRE into
+ // multiple loop blocks.
+ if (LoopBlock)
+ return false;
- // Phase 1. First off, look for a local dependency to avoid having to
- // disambiguate between before the load and after the load of the starting
- // block (as the load may be visited from a backedge).
- do {
- // Scan users of the clobbering memory access.
- if (auto RMV = scanMemoryAccessesUsers(
- Loc, IsInvariantLoad, StartBlock,
- SmallVector<MemoryAccess *, 1>{ClobberMA}, MSSA, AA, L)) {
- Values.emplace_back(*RMV);
- return true;
- }
+ // Do not sink into inner loops. This may be non-profitable.
+ if (L != LI->getLoopFor(Blocker))
+ return false;
- // Exit from here, and proceed visiting predecessors if the clobbering
- // access is non-local or is a MemoryPhi.
- if (ClobberMA->getBlock() != StartBlock || isa<MemoryPhi>(ClobberMA))
- break;
+ // Blocks that dominate the latch execute on every single iteration, maybe
+ // except the last one. So PREing into these blocks doesn't make much sense
+ // in most cases. But the blocks that do not necessarily execute on each
+ // iteration are sometimes much colder than the header, and this is when
+ // PRE is potentially profitable.
+ if (DT->dominates(Blocker, Latch))
+ return false;
- // Check if the clobber actually aliases the load location.
- if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad,
- StartBlock, MSSA, AA)) {
- Values.emplace_back(*RMV);
- return true;
- }
+ // Make sure that the terminator itself doesn't clobber.
+ if (Blocker->getTerminator()->mayWriteToMemory())
+ return false;
- // It may happen that the clobbering memory access does not actually
- // clobber our load location, transition to its defining memory access.
- ClobberMA = cast<MemoryUseOrDef>(ClobberMA)->getDefiningAccess();
- } while (ClobberMA->getBlock() == StartBlock);
+ LoopBlock = Blocker;
+ }
- // Non-local speculations are not allowed under ASan.
- if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
- L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ if (!LoopBlock)
return false;
- // Phase 2. Walk backwards through the CFG, collecting all the blocks that
- // contain an instruction that modifies the load memory location, or that lie
- // on a path between a clobbering block and our load. Start off by collecting
- // the predecessors of `StartBlock`. All the visited blocks are stored in a
- // the set `Blocks`. If possible, the memory address maintained for the block
- // visited does get phi-translated.
- DependencyBlockSet Blocks;
- SmallVector<BasicBlock *, 16> InitialWorklist;
- const DataLayout &DL = L->getModule()->getDataLayout();
- if (!collectPredecessors(StartBlock,
- PHITransAddr(L->getPointerOperand(), DL, AC),
- ClobberMA, Blocks, InitialWorklist))
+ // Make sure the memory at this pointer cannot be freed, therefore we can
+ // safely reload from it after clobber.
+ if (LoadPtr->canBeFreed())
return false;
- // Do a bottom-up DFS.
- auto Worklist = InitialWorklist;
- while (!Worklist.empty()) {
- auto *BB = Worklist.pop_back_val();
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
+ // TODO: Support critical edge splitting if blocker has more than 1 successor.
+ MapVector<BasicBlock *, Value *> AvailableLoads;
+ AvailableLoads[LoopBlock] = LoadPtr;
+ AvailableLoads[Preheader] = LoadPtr;
+
+ LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
+ eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads,
+ /*CriticalEdgePredAndLoad*/ nullptr);
+ ++NumPRELoopLoad;
+ return true;
+}
+
+/// Attempt to eliminate a load whose dependencies are
+/// non-local by performing PHI construction.
+bool GVNPass::processNonLocalLoad(LoadInst *Load) {
+ // Non-local speculations are not allowed under asan.
+ if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
+ Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ return false;
- // Phi-translation may have failed.
- if (!Info.Addr.getAddr())
- continue;
+ // Find the non-local dependencies of the load.
+ LoadDepVect Deps;
+ MD->getNonLocalPointerDependency(Load, Deps);
- // If the clobbering memory access is in the current block and it indeed
- // clobbers our load location, record the dependency and do not visit the
- // predecessors of this block further, continue with the blocks in the
- // worklist.
- if (Info.ClobberMA->getBlock() == BB && !isa<MemoryPhi>(Info.ClobberMA)) {
- if (auto RMV = accessMayModifyLocation(
- Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()),
- IsInvariantLoad, BB, MSSA, AA)) {
- Info.MemVal = RMV;
- continue;
- }
- assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) &&
- "LiveOnEntry aliases everything");
+ // If we had to process more than one hundred blocks to find the
+ // dependencies, this load isn't worth worrying about. Optimizing
+ // it will be too expensive.
+ unsigned NumDeps = Deps.size();
+ if (NumDeps > MaxNumDeps)
+ return false;
- // If, however, the clobbering memory access does not actually clobber
- // our load location, transition to its defining memory access, but
- // keep examining the same basic block.
- Info.ClobberMA =
- cast<MemoryUseOrDef>(Info.ClobberMA)->getDefiningAccess();
- Worklist.emplace_back(BB);
- continue;
- }
+ SmallVector<ReachingMemVal, 64> MemVals;
+ MemVals.reserve(Deps.size());
- // At this point we know the current block is "transparent", i.e. the memory
- // location is not modified when execution goes through this block.
- // Continue to its predecessors, unless a predecessor has already been
- // visited with a different address. We currently cannot represent such a
- // dependency.
- if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) {
- Info.ForceUnknown = true;
+ for (const NonLocalDepResult &Dep : Deps) {
+ const auto &R = Dep.getResult();
+ SelectAddr SelAddr = Dep.getAddress();
+ BasicBlock *BB = Dep.getBB();
+ Instruction *Inst = R.getInst();
+ if (R.isSelect()) {
+ auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs();
+ MemVals.emplace_back(
+ ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second));
continue;
}
- if (BB != StartBlock &&
- !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist))
- Info.ForceUnknown = true;
+ Value *Address = SelAddr.getAddr();
+ if (R.isClobber())
+ MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst));
+ else if (R.isDef())
+ MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst));
+ else
+ MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst));
}
- // Phase 3. We have collected all the blocks that either write a value to the
- // memory location of the load, or there exists a path to the load, along
- // which the memory location is not modified. Perform a second DFS to find
- // load-to-load dependencies; namely, look at the dominating memory reads,
- // that alias our load. These are the MemoryUses that are users of the
- // MemoryDefs we previously identified. If no memory read is encountered,
- // either confirm the clobbering write found before or set to unknown.
- Worklist = InitialWorklist;
- for (BasicBlock *BB : Worklist) {
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
- Info.Visited = true;
- }
+ return processNonLocalLoad(Load, MemVals);
+}
- SmallVector<MemoryAccess *> Clobbers;
- while (!Worklist.empty()) {
- auto *BB = Worklist.pop_back_val();
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
+bool GVNPass::processNonLocalLoad(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps) {
+ // If we had a phi translation failure, we'll have a single entry which is a
+ // clobber in the current block. Reject this early.
+ if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
+ LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
+ dbgs() << " has unknown dependencies\n";);
+ return false;
+ }
- // If phi-translation failed, assume the memory location is modified in
- // unknown way.
- if (!Info.Addr.getAddr()) {
- Values.push_back(ReachingMemVal::getUnknown(BB, nullptr));
- continue;
+ bool Changed = false;
+ // This is a limited form of scalar PRE for load indices. If this load follows
+ // a GEP, see if we can PRE the indices before analyzing.
+ if (isScalarPREEnabled()) {
+ if (GetElementPtrInst *GEP =
+ dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
+ for (Use &U : GEP->indices())
+ if (Instruction *I = dyn_cast<Instruction>(U.get()))
+ Changed |= performScalarPRE(I);
}
+ }
- Clobbers.clear();
- collectClobberList(Clobbers, BB, Info, Blocks, MSSA);
- if (auto RMV =
- scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()),
- IsInvariantLoad, BB, Clobbers, MSSA, AA)) {
- Values.push_back(*RMV);
- continue;
- }
+ // Step 1: Analyze the availability of the load.
+ AvailValInBlkVect ValuesPerBlock;
+ UnavailBlkVect UnavailableBlocks;
+ analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
- // If no reusable memory use was found, and the current block is not
- // transparent, use the already established memory def.
- if (Info.MemVal) {
- Values.push_back(*Info.MemVal);
- continue;
- }
+ // If we have no predecessors that produce a known value for this load, exit
+ // early.
+ if (ValuesPerBlock.empty())
+ return Changed;
- if (Info.ForceUnknown) {
- Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr()));
- continue;
- }
+ // Step 2: Eliminate fully redundancy.
+ //
+ // If all of the instructions we depend on produce a known value for this
+ // load, then it is fully redundant and we can use PHI insertion to compute
+ // its value. Insert PHIs and remove the fully redundant value now.
+ if (UnavailableBlocks.empty()) {
+ LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
- // If the current block is transparent, continue to its predecessors.
- for (BasicBlock *Pred : predecessors(BB)) {
- auto It = Blocks.find(Pred);
- if (It == Blocks.end())
- continue;
- DependencyBlockInfo &PredInfo = It->second;
- if (PredInfo.Visited)
- continue;
- PredInfo.Visited = true;
- Worklist.push_back(Pred);
- }
+ // Perform PHI construction.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
+ // constructSSAForLoadSet is responsible for combining metadata.
+ ICF->removeUsersOf(Load);
+ Load->replaceAllUsesWith(V);
+
+ if (isa<PHINode>(V))
+ V->takeName(Load);
+ if (Instruction *I = dyn_cast<Instruction>(V))
+ // If instruction I has debug info, then we should not update it.
+ // Also, if I has a null DebugLoc, then it is still potentially incorrect
+ // to propagate Load's DebugLoc because Load may not post-dominate I.
+ if (Load->getDebugLoc() && Load->getParent() == I->getParent())
+ I->setDebugLoc(Load->getDebugLoc());
+ if (MD && V->getType()->isPtrOrPtrVectorTy())
+ MD->invalidateCachedPointerInfo(V);
+ ++NumGVNLoad;
+ reportLoadElim(Load, V, ORE);
+ salvageAndRemoveInstruction(Load);
+ return true;
}
- return true;
+ // Step 3: Eliminate partial redundancy.
+ if (!isLoadPREEnabled())
+ return Changed;
+ if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
+ return Changed;
+
+ if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
+ performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
+ return true;
+
+ return Changed;
}
/// Attempt to eliminate a load, first by eliminating it
@@ -2874,188 +2963,99 @@ bool GVNPass::processMaskedLoad(IntrinsicInst *I) {
return true;
}
-/// Return a pair the first field showing the value number of \p Exp and the
-/// second field showing whether it is a value number newly created.
-std::pair<uint32_t, bool>
-GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
- uint32_t &E = ExpressionNumbering[Exp];
- bool CreateNewValNum = !E;
- if (CreateNewValNum) {
- Expressions.push_back(Exp);
- if (ExprIdx.size() < NextValueNumber + 1)
- ExprIdx.resize(NextValueNumber * 2);
- E = NextValueNumber;
- ExprIdx[NextValueNumber++] = NextExprNumber++;
- }
- return {E, CreateNewValNum};
-}
-
-/// Return whether all the values related with the same \p num are
-/// defined in \p BB.
-bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
- GVNPass &GVN) {
- return all_of(
- GVN.LeaderTable.getLeaders(Num),
- [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
-}
-
-/// Wrap phiTranslateImpl to provide caching functionality.
-uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
- auto FindRes = PhiTranslateTable.find({Num, Pred});
- if (FindRes != PhiTranslateTable.end())
- return FindRes->second;
- uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN);
- PhiTranslateTable.insert({{Num, Pred}, NewNum});
- return NewNum;
-}
-
-// Return true if the value number \p Num and NewNum have equal value.
-// Return false if the result is unknown.
-bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
- const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- GVNPass &GVN) {
- CallInst *Call = nullptr;
- auto Leaders = GVN.LeaderTable.getLeaders(Num);
- for (const auto &Entry : Leaders) {
- Call = dyn_cast<CallInst>(&*Entry.Val);
- if (Call && Call->getParent() == PhiBlock)
- break;
- }
-
- if (AA->doesNotAccessMemory(Call))
- return true;
+// If the given branch is recognized as a foldable branch (i.e. conditional
+// branch with constant condition), it will perform following analyses and
+// transformation.
+// 1) If the dead out-coming edge is a critical-edge, split it. Let
+// R be the target of the dead out-coming edge.
+// 1) Identify the set of dead blocks implied by the branch's dead outcoming
+// edge. The result of this step will be {X| X is dominated by R}
+// 2) Identify those blocks which haves at least one dead predecessor. The
+// result of this step will be dominance-frontier(R).
+// 3) Update the PHIs in DF(R) by replacing the operands corresponding to
+// dead blocks with "UndefVal" in an hope these PHIs will optimized away.
+//
+// Return true iff *NEW* dead code are found.
+bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
+ // If a branch has two identical successors, we cannot declare either dead.
+ if (BI->getSuccessor(0) == BI->getSuccessor(1))
+ return false;
- if (!MD || !AA->onlyReadsMemory(Call))
+ ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
+ if (!Cond)
return false;
- MemDepResult LocalDep = MD->getDependency(Call);
- if (!LocalDep.isNonLocal())
+ BasicBlock *DeadRoot =
+ Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0);
+ if (DeadBlocks.count(DeadRoot))
return false;
- const MemoryDependenceResults::NonLocalDepInfo &Deps =
- MD->getNonLocalCallDependency(Call);
+ if (!DeadRoot->getSinglePredecessor())
+ DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot);
- // Check to see if the Call has no function local clobber.
- for (const NonLocalDepEntry &D : Deps) {
- if (D.getResult().isNonFuncLocal())
- return true;
- }
- return false;
+ addDeadBlock(DeadRoot);
+ return true;
}
-/// Translate value number \p Num using phis, so that it has the values of
-/// the phis in BB.
-uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
- // See if we can refine the value number by looking at the PN incoming value
- // for the given predecessor.
- if (PHINode *PN = NumberingPhi[Num]) {
- if (PN->getParent() != PhiBlock)
- return Num;
- for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) {
- if (PN->getIncomingBlock(I) != Pred)
- continue;
- if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false))
- return TransVal;
- }
- return Num;
- }
-
- if (BasicBlock *BB = NumberingBB[Num]) {
- assert(MSSA && "NumberingBB is non-empty only when using MemorySSA");
- // Value numbers of basic blocks are used to represent memory state in
- // load/store instructions and read-only function calls when said state is
- // set by a MemoryPhi.
- if (BB != PhiBlock)
- return Num;
- MemoryPhi *MPhi = MSSA->getMemoryAccess(BB);
- for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) {
- if (MPhi->getIncomingBlock(i) != Pred)
- continue;
- MemoryAccess *MA = MPhi->getIncomingValue(i);
- if (auto *PredPhi = dyn_cast<MemoryPhi>(MA))
- return lookupOrAdd(PredPhi->getBlock());
- if (MSSA->isLiveOnEntryDef(MA))
- return lookupOrAdd(&BB->getParent()->getEntryBlock());
- return lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
- }
- llvm_unreachable(
- "CFG/MemorySSA mismatch: predecessor not found among incoming blocks");
- }
+bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
+ Value *V = IntrinsicI->getArgOperand(0);
- // If there is any value related with Num is defined in a BB other than
- // PhiBlock, it cannot depend on a phi in PhiBlock without going through
- // a backedge. We can do an early exit in that case to save compile time.
- if (!areAllValsInBB(Num, PhiBlock, GVN))
- return Num;
+ if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
+ if (Cond->isZero()) {
+ Type *Int8Ty = Type::getInt8Ty(V->getContext());
+ Type *PtrTy = PointerType::get(V->getContext(), 0);
+ // Insert a new store to null instruction before the load to indicate that
+ // this code is not reachable. FIXME: We could insert unreachable
+ // instruction directly because we can modify the CFG.
+ auto *NewS =
+ new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy),
+ IntrinsicI->getIterator());
+ if (MSSAU) {
+ const MemoryUseOrDef *FirstNonDom = nullptr;
+ const auto *AL =
+ MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent());
- if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
- return Num;
- Expression Exp = Expressions[ExprIdx[Num]];
+ // If there are accesses in the current basic block, find the first one
+ // that does not come before NewS. The new memory access is inserted
+ // after the found access or before the terminator if no such access is
+ // found.
+ if (AL) {
+ for (const auto &Acc : *AL) {
+ if (auto *Current = dyn_cast<MemoryUseOrDef>(&Acc))
+ if (!Current->getMemoryInst()->comesBefore(NewS)) {
+ FirstNonDom = Current;
+ break;
+ }
+ }
+ }
- for (unsigned I = 0; I < Exp.VarArgs.size(); I++) {
- // For InsertValue and ExtractValue, some varargs are index numbers
- // instead of value numbers. Those index numbers should not be
- // translated.
- if ((I > 1 && Exp.Opcode == Instruction::InsertValue) ||
- (I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
- (I > 1 && Exp.Opcode == Instruction::ShuffleVector))
- continue;
- Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN);
- }
+ auto *NewDef =
+ FirstNonDom
+ ? MSSAU->createMemoryAccessBefore(
+ NewS, nullptr, const_cast<MemoryUseOrDef *>(FirstNonDom))
+ : MSSAU->createMemoryAccessInBB(NewS, nullptr,
+ NewS->getParent(),
+ MemorySSA::BeforeTerminator);
- if (Exp.Commutative) {
- assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!");
- if (Exp.VarArgs[0] > Exp.VarArgs[1]) {
- std::swap(Exp.VarArgs[0], Exp.VarArgs[1]);
- uint32_t Opcode = Exp.Opcode >> 8;
- if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
- Exp.Opcode = (Opcode << 8) |
- CmpInst::getSwappedPredicate(
- static_cast<CmpInst::Predicate>(Exp.Opcode & 255));
+ MSSAU->insertDef(cast<MemoryDef>(NewDef), /*RenameUses=*/false);
+ }
}
+ if (isAssumeWithEmptyBundle(*IntrinsicI)) {
+ salvageAndRemoveInstruction(IntrinsicI);
+ return true;
+ }
+ return false;
}
- if (uint32_t NewNum = ExpressionNumbering[Exp]) {
- if (Exp.Opcode == Instruction::Call && NewNum != Num)
- return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num;
- return NewNum;
- }
- return Num;
-}
-
-/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
-/// again.
-void GVNPass::ValueTable::eraseTranslateCacheEntry(
- uint32_t Num, const BasicBlock &CurrBlock) {
- for (const BasicBlock *Pred : predecessors(&CurrBlock))
- PhiTranslateTable.erase({Num, Pred});
-}
-
-// In order to find a leader for a given value number at a
-// specific basic block, we first obtain the list of all Values for that number,
-// and then scan the list to find one whose block dominates the block in
-// question. This is fast because dominator tree queries consist of only
-// a few comparisons of DFS numbers.
-Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
- auto Leaders = LeaderTable.getLeaders(Num);
- if (Leaders.empty())
- return nullptr;
-
- Value *Val = nullptr;
- for (const auto &Entry : Leaders) {
- if (DT->dominates(Entry.BB, BB)) {
- Val = Entry.Val;
- if (isa<Constant>(Val))
- return Val;
- }
+ if (isa<Constant>(V)) {
+ // If it's not false, and constant, it must evaluate to true. This means our
+ // assume is assume(true), and thus, pointless, and we don't want to do
+ // anything more here.
+ return false;
}
- return Val;
+ Constant *True = ConstantInt::getTrue(V->getContext());
+ return propagateEquality(V, True, IntrinsicI);
}
/// There is an edge from 'Src' to 'Dst'. Return
@@ -3074,15 +3074,6 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E,
return Pred != nullptr;
}
-void GVNPass::assignBlockRPONumber(Function &F) {
- BlockRPONumber.clear();
- uint32_t NextBlockNumber = 1;
- ReversePostOrderTraversal<Function *> RPOT(&F);
- for (BasicBlock *BB : RPOT)
- BlockRPONumber[BB] = NextBlockNumber++;
- InvalidBlockRPONumbers = false;
-}
-
/// The given values are known to be equal in every use
/// dominated by 'Root'. Exploit this, for example by replacing 'LHS' with
/// 'RHS' everywhere in the scope. Returns whether a change was made.
@@ -3290,6 +3281,11 @@ bool GVNPass::propagateEquality(
return Changed;
}
+static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
+ patchReplacementInstruction(I, Repl);
+ I->replaceAllUsesWith(Repl);
+}
+
/// When calculating availability, handle an instruction
/// by inserting it into the appropriate sets.
bool GVNPass::processInstruction(Instruction *I) {
@@ -3433,94 +3429,18 @@ bool GVNPass::processInstruction(Instruction *I) {
return false;
}
- if (Repl == I) {
- // If I was the result of a shortcut PRE, it might already be in the table
- // and the best replacement for itself. Nothing to do.
- return false;
- }
-
- // Remove it!
- patchAndReplaceAllUsesWith(I, Repl);
- if (MD && Repl->getType()->isPtrOrPtrVectorTy())
- MD->invalidateCachedPointerInfo(Repl);
- salvageAndRemoveInstruction(I);
- return true;
-}
-
-/// runOnFunction - This is the main transformation entry point for a function.
-bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
- const TargetLibraryInfo &RunTLI, AAResults &RunAA,
- MemoryDependenceResults *RunMD, LoopInfo &LI,
- OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
- AC = &RunAC;
- DT = &RunDT;
- VN.setDomTree(DT);
- TLI = &RunTLI;
- AA = &RunAA;
- VN.setAliasAnalysis(&RunAA);
- MD = RunMD;
- ImplicitControlFlowTracking ImplicitCFT;
- ICF = &ImplicitCFT;
- this->LI = &LI;
- VN.setMemDep(MD);
- // Propagate the MSSA-enabled flag so the value-numbering paths in
- // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether
- // IsMSSAEnabled is turned on.
- VN.setMemorySSA(MSSA, isMemorySSAEnabled());
- ORE = RunORE;
- InvalidBlockRPONumbers = true;
- MemorySSAUpdater Updater(MSSA);
- MSSAU = MSSA ? &Updater : nullptr;
-
- bool Changed = false;
- bool ShouldContinue = true;
-
- DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
- // Merge unconditional branches, allowing PRE to catch more
- // optimization opportunities.
- for (BasicBlock &BB : make_early_inc_range(F)) {
- bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
- if (RemovedBlock)
- ++NumGVNBlocks;
-
- Changed |= RemovedBlock;
- }
- DTU.flush();
-
- unsigned Iteration = 0;
- while (ShouldContinue) {
- LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n");
- (void) Iteration;
- ShouldContinue = iterateOnFunction(F);
- Changed |= ShouldContinue;
- ++Iteration;
- }
-
- if (isScalarPREEnabled()) {
- // Fabricate val-num for dead-code in order to suppress assertion in
- // performPRE().
- assignValNumForDeadCode();
- bool PREChanged = true;
- while (PREChanged) {
- PREChanged = performPRE(F);
- Changed |= PREChanged;
- }
- }
-
- // FIXME: Should perform GVN again after PRE does something. PRE can move
- // computations into blocks where they become fully redundant. Note that
- // we can't do this until PRE's critical edge splitting updates memdep.
- // Actually, when this happens, we should just fully integrate PRE into GVN.
-
- cleanupGlobalSets();
- // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each
- // iteration.
- DeadBlocks.clear();
-
- if (MSSA && VerifyMemorySSA)
- MSSA->verifyMemorySSA();
+ if (Repl == I) {
+ // If I was the result of a shortcut PRE, it might already be in the table
+ // and the best replacement for itself. Nothing to do.
+ return false;
+ }
- return Changed;
+ // Remove it!
+ patchAndReplaceAllUsesWith(I, Repl);
+ if (MD && Repl->getType()->isPtrOrPtrVectorTy())
+ MD->invalidateCachedPointerInfo(Repl);
+ salvageAndRemoveInstruction(I);
+ return true;
}
bool GVNPass::processBlock(BasicBlock *BB) {
@@ -3543,6 +3463,23 @@ bool GVNPass::processBlock(BasicBlock *BB) {
return ChangedFunction;
}
+/// Executes one iteration of GVN.
+bool GVNPass::iterateOnFunction(Function &F) {
+ cleanupGlobalSets();
+
+ // Top-down walk of the dominator tree.
+ bool Changed = false;
+ // Needed for value numbering with phi construction to work.
+ // RPOT walks the graph in its constructor and will not be invalidated during
+ // processBlock.
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+
+ for (BasicBlock *BB : RPOT)
+ Changed |= processBlock(BB);
+
+ return Changed;
+}
+
// Instantiate an expression in a predecessor that lacked it.
bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
BasicBlock *Curr, unsigned int ValNo) {
@@ -3780,60 +3717,104 @@ bool GVNPass::performPRE(Function &F) {
return Changed;
}
-/// Split the critical edge connecting the given two blocks, and return
-/// the block inserted to the critical edge.
-BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
- // GVN does not require loop-simplify, do not try to preserve it if it is not
- // possible.
- BasicBlock *BB = SplitCriticalEdge(
- Pred, Succ,
- CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify());
- if (BB) {
- if (MD)
- MD->invalidateCachedPredecessors();
- InvalidBlockRPONumbers = true;
+/// runOnFunction - This is the main transformation entry point for a function.
+bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+ const TargetLibraryInfo &RunTLI, AAResults &RunAA,
+ MemoryDependenceResults *RunMD, LoopInfo &LI,
+ OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
+ AC = &RunAC;
+ DT = &RunDT;
+ VN.setDomTree(DT);
+ TLI = &RunTLI;
+ AA = &RunAA;
+ VN.setAliasAnalysis(&RunAA);
+ MD = RunMD;
+ ImplicitControlFlowTracking ImplicitCFT;
+ ICF = &ImplicitCFT;
+ this->LI = &LI;
+ VN.setMemDep(MD);
+ // Propagate the MSSA-enabled flag so the value-numbering paths in
+ // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether
+ // IsMSSAEnabled is turned on.
+ VN.setMemorySSA(MSSA, isMemorySSAEnabled());
+ ORE = RunORE;
+ InvalidBlockRPONumbers = true;
+ MemorySSAUpdater Updater(MSSA);
+ MSSAU = MSSA ? &Updater : nullptr;
+
+ bool Changed = false;
+ bool ShouldContinue = true;
+
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+ // Merge unconditional branches, allowing PRE to catch more
+ // optimization opportunities.
+ for (BasicBlock &BB : make_early_inc_range(F)) {
+ bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
+ if (RemovedBlock)
+ ++NumGVNBlocks;
+
+ Changed |= RemovedBlock;
}
- return BB;
-}
+ DTU.flush();
-/// Split critical edges found during the previous
-/// iteration that may enable further optimization.
-bool GVNPass::splitCriticalEdges() {
- if (ToSplit.empty())
- return false;
+ unsigned Iteration = 0;
+ while (ShouldContinue) {
+ LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n");
+ (void)Iteration;
+ ShouldContinue = iterateOnFunction(F);
+ Changed |= ShouldContinue;
+ ++Iteration;
+ }
- bool Changed = false;
- do {
- std::pair<Instruction *, unsigned> Edge = ToSplit.pop_back_val();
- Changed |= SplitCriticalEdge(Edge.first, Edge.second,
- CriticalEdgeSplittingOptions(DT, LI, MSSAU)) !=
- nullptr;
- } while (!ToSplit.empty());
- if (Changed) {
- if (MD)
- MD->invalidateCachedPredecessors();
- InvalidBlockRPONumbers = true;
+ if (isScalarPREEnabled()) {
+ // Fabricate val-num for dead-code in order to suppress assertion in
+ // performPRE().
+ assignValNumForDeadCode();
+ bool PREChanged = true;
+ while (PREChanged) {
+ PREChanged = performPRE(F);
+ Changed |= PREChanged;
+ }
}
- return Changed;
-}
-/// Executes one iteration of GVN.
-bool GVNPass::iterateOnFunction(Function &F) {
- cleanupGlobalSets();
+ // FIXME: Should perform GVN again after PRE does something. PRE can move
+ // computations into blocks where they become fully redundant. Note that
+ // we can't do this until PRE's critical edge splitting updates memdep.
+ // Actually, when this happens, we should just fully integrate PRE into GVN.
- // Top-down walk of the dominator tree.
- bool Changed = false;
- // Needed for value numbering with phi construction to work.
- // RPOT walks the graph in its constructor and will not be invalidated during
- // processBlock.
- ReversePostOrderTraversal<Function *> RPOT(&F);
+ cleanupGlobalSets();
+ // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each
+ // iteration.
+ DeadBlocks.clear();
- for (BasicBlock *BB : RPOT)
- Changed |= processBlock(BB);
+ if (MSSA && VerifyMemorySSA)
+ MSSA->verifyMemorySSA();
return Changed;
}
+// In order to find a leader for a given value number at a
+// specific basic block, we first obtain the list of all Values for that number,
+// and then scan the list to find one whose block dominates the block in
+// question. This is fast because dominator tree queries consist of only
+// a few comparisons of DFS numbers.
+Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
+ auto Leaders = LeaderTable.getLeaders(Num);
+ if (Leaders.empty())
+ return nullptr;
+
+ Value *Val = nullptr;
+ for (const auto &Entry : Leaders) {
+ if (DT->dominates(Entry.BB, BB)) {
+ Val = Entry.Val;
+ if (isa<Constant>(Val))
+ return Val;
+ }
+ }
+
+ return Val;
+}
+
void GVNPass::cleanupGlobalSets() {
VN.clear();
LeaderTable.clear();
@@ -3855,12 +3836,55 @@ void GVNPass::removeInstruction(Instruction *I) {
++NumGVNInstr;
}
+void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
+ salvageKnowledge(I, AC);
+ salvageDebugInfo(*I);
+ removeInstruction(I);
+}
+
/// Verify that the specified instruction does not occur in our
/// internal data structures.
void GVNPass::verifyRemoved(const Instruction *Inst) const {
VN.verifyRemoved(Inst);
}
+/// Split critical edges found during the previous
+/// iteration that may enable further optimization.
+bool GVNPass::splitCriticalEdges() {
+ if (ToSplit.empty())
+ return false;
+
+ bool Changed = false;
+ do {
+ std::pair<Instruction *, unsigned> Edge = ToSplit.pop_back_val();
+ Changed |= SplitCriticalEdge(Edge.first, Edge.second,
+ CriticalEdgeSplittingOptions(DT, LI, MSSAU)) !=
+ nullptr;
+ } while (!ToSplit.empty());
+ if (Changed) {
+ if (MD)
+ MD->invalidateCachedPredecessors();
+ InvalidBlockRPONumbers = true;
+ }
+ return Changed;
+}
+
+/// Split the critical edge connecting the given two blocks, and return
+/// the block inserted to the critical edge.
+BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
+ // GVN does not require loop-simplify, do not try to preserve it if it is not
+ // possible.
+ BasicBlock *BB = SplitCriticalEdge(
+ Pred, Succ,
+ CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify());
+ if (BB) {
+ if (MD)
+ MD->invalidateCachedPredecessors();
+ InvalidBlockRPONumbers = true;
+ }
+ return BB;
+}
+
/// BB is declared dead, which implied other blocks become dead as well. This
/// function is to add all these blocks to "DeadBlocks". For the dead blocks'
/// live successors, update their phi nodes by replacing the operands
@@ -3940,40 +3964,6 @@ void GVNPass::addDeadBlock(BasicBlock *BB) {
}
}
-// If the given branch is recognized as a foldable branch (i.e. conditional
-// branch with constant condition), it will perform following analyses and
-// transformation.
-// 1) If the dead out-coming edge is a critical-edge, split it. Let
-// R be the target of the dead out-coming edge.
-// 1) Identify the set of dead blocks implied by the branch's dead outcoming
-// edge. The result of this step will be {X| X is dominated by R}
-// 2) Identify those blocks which haves at least one dead predecessor. The
-// result of this step will be dominance-frontier(R).
-// 3) Update the PHIs in DF(R) by replacing the operands corresponding to
-// dead blocks with "UndefVal" in an hope these PHIs will optimized away.
-//
-// Return true iff *NEW* dead code are found.
-bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
- // If a branch has two identical successors, we cannot declare either dead.
- if (BI->getSuccessor(0) == BI->getSuccessor(1))
- return false;
-
- ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
- if (!Cond)
- return false;
-
- BasicBlock *DeadRoot =
- Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0);
- if (DeadBlocks.count(DeadRoot))
- return false;
-
- if (!DeadRoot->getSinglePredecessor())
- DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot);
-
- addDeadBlock(DeadRoot);
- return true;
-}
-
// performPRE() will trigger assert if it comes across an instruction without
// associated val-num. As it normally has far more live instructions than dead
// instructions, it makes more sense just to "fabricate" a val-number for the
@@ -3987,6 +3977,15 @@ void GVNPass::assignValNumForDeadCode() {
}
}
+void GVNPass::assignBlockRPONumber(Function &F) {
+ BlockRPONumber.clear();
+ uint32_t NextBlockNumber = 1;
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+ for (BasicBlock *BB : RPOT)
+ BlockRPONumber[BB] = NextBlockNumber++;
+ InvalidBlockRPONumbers = false;
+}
+
class llvm::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
More information about the llvm-commits
mailing list