[clang] [llvm] [GVN] Simple GVN-based hoisting of scalars (PR #210330)
Momchil Velikov via cfe-commits
cfe-commits at lists.llvm.org
Fri Jul 17 06:31:33 PDT 2026
https://github.com/momchil-velikov created https://github.com/llvm/llvm-project/pull/210330
None
>From e1784a875479f2bb3fd139c7fff75393a3b91dc5 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 09:48:01 +0000
Subject: [PATCH 1/7] [GVN] Remove the "private" `llvm::gvn` namespace (NFC)
Move `AvailableValue` and `AvailableValueInBlock` into GVNPass, similar
to other helper types.
Retain `llvm::gvn::GVNLegacyPass` as just `llvm::GVNLegacyPass` -
"legacy" is already a sufficent hint and it is not going to become more
"private" by stacking "gvn" prefixes to the name.
Ideally, `GVNLegacyPass` should be defined in an anonymous namespace, but
that is not possible because it is declared as a friend of GVNPass.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 19 ++++++-------------
llvm/lib/Transforms/Scalar/GVN.cpp | 10 ++++++----
2 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 9142defb34de2..8b70d61fd9f3f 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -61,15 +61,6 @@ class PHINode;
class TargetLibraryInfo;
class Value;
class IntrinsicInst;
-/// A private "module" namespace for types and utilities used by GVN. These
-/// are implementation details and should not be used by clients.
-namespace LLVM_LIBRARY_VISIBILITY_NAMESPACE gvn {
-
-struct AvailableValue;
-struct AvailableValueInBlock;
-class GVNLegacyPass;
-
-} // end namespace gvn
/// A set of parameters to control various transforms performed by GVN pass.
// Each of the optional boolean parameters can be set to:
@@ -133,6 +124,8 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
public:
struct Expression;
+ struct AvailableValue;
+ struct AvailableValueInBlock;
GVNPass(GVNOptions Options = {}) : Options(Options) {}
@@ -248,7 +241,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
};
private:
- friend class gvn::GVNLegacyPass;
+ friend class GVNLegacyPass;
friend struct DenseMapInfo<Expression>;
MemoryDependenceResults *MD = nullptr;
@@ -354,7 +347,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
bool InvalidBlockRPONumbers = true;
using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
- using AvailValInBlkVect = SmallVector<gvn::AvailableValueInBlock, 64>;
+ using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
@@ -454,7 +447,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// Given a local dependency (Def or Clobber) determine if a value is
/// available for the load.
- std::optional<gvn::AvailableValue>
+ std::optional<AvailableValue>
AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address);
@@ -462,7 +455,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
/// value is available by finding dominating values for both addresses. If
/// so, the load can be rematerialized as a select of those two values.
- std::optional<gvn::AvailableValue>
+ std::optional<AvailableValue>
AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 1b7bcb10be8f8..dc9c1fb0ba719 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -81,10 +81,12 @@
#include <utility>
using namespace llvm;
-using namespace llvm::gvn;
using namespace llvm::VNCoercion;
using namespace PatternMatch;
+using AvailableValue = GVNPass::AvailableValue;
+using AvailableValueInBlock = GVNPass::AvailableValueInBlock;
+
#define DEBUG_TYPE "gvn"
STATISTIC(NumGVNInstr, "Number of instructions deleted");
@@ -193,7 +195,7 @@ template <> struct llvm::DenseMapInfo<GVNPass::Expression> {
/// Materialization of an AvailableValue never fails. An AvailableValue is
/// implicitly associated with a rematerialization point which is the
/// location of the instruction from which it was formed.
-struct llvm::gvn::AvailableValue {
+struct llvm::GVNPass::AvailableValue {
enum class ValType {
SimpleVal, // A simple offsetted value that is accessed.
LoadVal, // A value produced by a load.
@@ -289,7 +291,7 @@ struct llvm::gvn::AvailableValue {
/// Represents an AvailableValue which can be rematerialized at the end of
/// the associated BasicBlock.
-struct llvm::gvn::AvailableValueInBlock {
+struct llvm::GVNPass::AvailableValueInBlock {
/// BB - The basic block in question.
BasicBlock *BB = nullptr;
@@ -3996,7 +3998,7 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-class llvm::gvn::GVNLegacyPass : public FunctionPass {
+class llvm::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
>From 2293acff88a03097a610c682769ad316d6b1fc0e Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 11:06:39 +0000
Subject: [PATCH 2/7] [GVN] Rename some functions to follow LLVM naming
conventions (NFC)
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 8 ++---
llvm/lib/Transforms/Scalar/GVN.cpp | 38 +++++++++++------------
2 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 8b70d61fd9f3f..ffe07fb2ec7e2 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -448,7 +448,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// Given a local dependency (Def or Clobber) determine if a value is
/// available for the load.
std::optional<AvailableValue>
- AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address);
/// Given a select-dependency for the load (the load address is a select of
@@ -456,13 +456,13 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
/// value is available by finding dominating values for both addresses. If
/// so, the load can be rematerialized as a select of those two values.
std::optional<AvailableValue>
- AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+ analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From);
/// Given a list of non-local dependencies, determine if a value is
/// available for the load in each specified block. If it is, add it to
/// ValuesPerBlock. If not, add it to UnavailableBlocks.
- void AnalyzeLoadAvailability(LoadInst *Load,
+ void analyzeLoadAvailability(LoadInst *Load,
SmallVectorImpl<ReachingMemVal> &Deps,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
@@ -472,7 +472,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
LoadInst *Load);
- bool PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
/// Try to replace a load which executes on each loop iteraiton with Phi
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index dc9c1fb0ba719..ee050d5a3861c 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -965,7 +965,7 @@ enum class AvailabilityState : char {
/// 1) we know the block *is* fully available.
/// 2) we do not know whether the block is fully available or not, but we are
/// currently speculating that it will be.
-static bool IsValueFullyAvailableInBlock(
+static bool isValueFullyAvailableInBlock(
BasicBlock *BB,
DenseMap<BasicBlock *, AvailabilityState> &FullyAvailableBlocks) {
SmallVector<BasicBlock *, 32> Worklist;
@@ -1100,7 +1100,7 @@ static void replaceValuesPerBlockEntry(
/// construct SSA form, allowing us to eliminate Load. This returns the value
/// that should be used at Load's definition site.
static Value *
-ConstructSSAForLoadSet(LoadInst *Load,
+constructSSAForLoadSet(LoadInst *Load,
SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
GVNPass &GVN) {
// Check for the fully redundant, dominating load case. In this case, we can
@@ -1330,7 +1330,7 @@ static Value *findDominatingValue(const MemoryLocation &Loc, Type *LoadTy,
}
std::optional<AvailableValue>
-GVNPass::AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+GVNPass::analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
Value *FalseAddr, Instruction *From) {
assert(TrueAddr->getType() == Load->getPointerOperandType() &&
"Invalid address type of true side of select dependency");
@@ -1352,7 +1352,7 @@ GVNPass::AnalyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
}
std::optional<AvailableValue>
-GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+GVNPass::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
Value *Address) {
assert(Load->isUnordered() && "rules below are incorrect for ordered access");
assert((Dep.Kind == DepKind::Def || Dep.Kind == DepKind::Clobber) &&
@@ -1479,7 +1479,7 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
// loads and DepInst that may clobber the loads.
if (auto *Sel = dyn_cast<SelectInst>(DepInst)) {
assert(Sel->getType() == Load->getPointerOperandType());
- if (auto AV = AnalyzeSelectAvailability(Load, Sel->getCondition(),
+ if (auto AV = analyzeSelectAvailability(Load, Sel->getCondition(),
Sel->getTrueValue(),
Sel->getFalseValue(), DepInst))
return AV;
@@ -1494,7 +1494,7 @@ GVNPass::AnalyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
return std::nullopt;
}
-void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
+void GVNPass::analyzeLoadAvailability(LoadInst *Load,
SmallVectorImpl<ReachingMemVal> &Deps,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
@@ -1521,7 +1521,7 @@ void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
// load as a select of the two reaching values (one per side). The values
// are searched for at the end of DepBB.
if (Dep.Kind == DepKind::Select) {
- if (auto AV = AnalyzeSelectAvailability(
+ if (auto AV = analyzeSelectAvailability(
Load, const_cast<Value *>(Dep.SelCond),
const_cast<Value *>(Dep.SelTrueAddr),
const_cast<Value *>(Dep.SelFalseAddr), DepBB->getTerminator())) {
@@ -1537,7 +1537,7 @@ void GVNPass::AnalyzeLoadAvailability(LoadInst *Load,
// the pointer operand of the load if PHI translation occurs. Make sure
// to consider the right address.
if (auto AV =
- AnalyzeLoadAvailability(Load, Dep, const_cast<Value *>(Dep.Addr))) {
+ analyzeLoadAvailability(Load, Dep, const_cast<Value *>(Dep.Addr))) {
// subtlety: because we know this was a non-local dependency, we know
// it's safe to materialize anywhere between the instruction within
// DepInfo and the end of it's block.
@@ -1609,7 +1609,7 @@ LoadInst *GVNPass::findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
// If an identical load doesn't depends on any local instructions, it can
// be safely moved to PredBB.
// Also check for the implicit control flow instructions. See the comments
- // in PerformLoadPRE for details.
+ // in performLoadPRE for details.
if (!HasLocalDep && !ICF->isDominatedByICFIFromSameBlock(&Inst))
return cast<LoadInst>(&Inst);
@@ -1693,8 +1693,8 @@ void GVNPass::eliminatePartiallyRedundantLoad(
}
// Perform PHI construction.
- Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // ConstructSSAForLoadSet is responsible for combining metadata.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ // constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
if (isa<PHINode>(V))
@@ -1710,7 +1710,7 @@ void GVNPass::eliminatePartiallyRedundantLoad(
salvageAndRemoveInstruction(Load);
}
-bool GVNPass::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
// Okay, we have *some* definitions of the value. This means that the value
// is available in some of our (transitive) predecessors. Lets think about
@@ -1793,7 +1793,7 @@ bool GVNPass::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
return false;
}
- if (IsValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
+ if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
continue;
}
@@ -2122,7 +2122,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
// Step 1: Analyze the availability of the load.
AvailValInBlkVect ValuesPerBlock;
UnavailBlkVect UnavailableBlocks;
- AnalyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
+ analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
// If we have no predecessors that produce a known value for this load, exit
// early.
@@ -2138,8 +2138,8 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
// Perform PHI construction.
- Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // ConstructSSAForLoadSet is responsible for combining metadata.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ // constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
@@ -2166,7 +2166,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
return Changed;
if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
- PerformLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
+ performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
return true;
return Changed;
@@ -2595,7 +2595,7 @@ void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
/// Given as input a load instruction, the function computes the set of reaching
-/// memory values, one per predecessor path, that AnalyzeLoadAvailability can
+/// memory values, one per predecessor path, that analyzeLoadAvailability can
/// later use to establish whether the load may be eliminated. A reaching value
/// may be of the following descriptor kind:
/// * Def: a precise instruction that produces the exact bits the load would
@@ -2835,7 +2835,7 @@ bool GVNPass::processLoad(LoadInst *L) {
return false;
}
- auto AV = AnalyzeLoadAvailability(L, MemVal, L->getPointerOperand());
+ auto AV = analyzeLoadAvailability(L, MemVal, L->getPointerOperand());
if (!AV)
return false;
>From 67ad2e492b9f7a5ead697bfd8ec1e21aef2973cf Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 7 Jul 2026 14:33:47 +0000
Subject: [PATCH 3/7] [GVN] Remove unused debug helper (NFC)
The `GVNPass::dump` method is not used anywhere. Moreover, there's
no `GVNPass` state that corresponds to its parameter type. Even if a
`GVNPass::dump` method could be useful, this one wasn't it.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 1 -
llvm/lib/Transforms/Scalar/GVN.cpp | 11 -----------
2 files changed, 12 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index ffe07fb2ec7e2..0b9c8dc88fe14 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -491,7 +491,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
// Other helper routines.
bool processInstruction(Instruction *I);
bool processBlock(BasicBlock *BB);
- void dump(DenseMap<uint32_t, Value *> &Map) const;
bool iterateOnFunction(Function &F);
bool performPRE(Function &F);
bool performScalarPRE(Instruction *I);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index ee050d5a3861c..155308f0e7f71 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -934,17 +934,6 @@ void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
removeInstruction(I);
}
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void GVNPass::dump(DenseMap<uint32_t, Value *> &Map) const {
- errs() << "{\n";
- for (const auto &[Num, Exp] : Map) {
- errs() << Num << "\n";
- Exp->dump();
- }
- errs() << "}\n";
-}
-#endif
-
enum class AvailabilityState : char {
/// We know the block *is not* fully available. This is a fixpoint.
Unavailable = 0,
>From 29556c748d9f533bb8bdf1a98f3bf279ebedaddb Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 8 Jul 2026 10:50:27 +0000
Subject: [PATCH 4/7] [GVN] Reorganise GVN.h/GVH.cpp to improve readability and
maintainability (NFC)
Over the years GVN.h/GVN.cpp has grown in size and complexity, and the order of
member functions and definitions has become somewhat arbitrary. This commit
reorganises the code to improve readability and maintainability.
* in `GVNPass` class, put private member variables first, followed by public
member functions, and then private member functions
* in `GVNPass` class: private type definitions are placed in front of the
logically related member variables (except `ValueTable` which need to be
public)
* definitions of `GVNPass::ValueTable` methods are grouped and reordered to
match the order of their declarations
* The following `GVNPass` member functions were made `private` and `LLVM_API`
removed: `getDominatorTree`, `getAliasAnalysis`, `getMemDep`,
`isScalarPREEnabled`, `isLoadPREEnabled`, `isLoadInLoopPREEnabled`,
`isLoadPRESplitBackedgeEnabled`, `isMemDepEnabled`, `isMemorySSAEnabled`,
and `salvageAndRemoveInstruction`
* `constructSSAForLoadSet` changed to take a `Dominator &`, in order to not
require access to the (now) private `getDominatorTree`
* member functions of `GVNPass` rearranged into a more logical order:
- starting with the main pass entry pount (`runImpl`) put utility member
functions in front of their callers, in order of calling (where it matters),
for example `runImpl` -> `iterateOnFunction` -> `perfromPRE`
- group functions of the same "theme" together, for example
`iterateOnFunction` + `processBlock` + `processInstrution`, or another
example, `performLoadPRE` + `performLoopLoadPRE`
- put miscelaneous helper member functions at the end
* rearrange definitions in `GVN.cpp` to match the order of declarations in
`GVN.h`
* place `static` helper functions close and in front of their callers
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 191 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 3045 ++++++++++-----------
2 files changed, 1629 insertions(+), 1607 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 0b9c8dc88fe14..46c54363298a2 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -120,37 +120,10 @@ struct GVNOptions {
/// FIXME: We should have a good summary of the GVN algorithm implemented by
/// this particular pass here.
class GVNPass : public OptionalPassInfoMixin<GVNPass> {
- GVNOptions Options;
-
public:
struct Expression;
struct AvailableValue;
struct AvailableValueInBlock;
-
- GVNPass(GVNOptions Options = {}) : Options(Options) {}
-
- /// Run the pass over the function.
- LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-
- LLVM_ABI void
- printPipeline(raw_ostream &OS,
- function_ref<StringRef(StringRef)> MapClassName2PassName);
-
- /// This removes the specified instruction from
- /// our various maps and marks it for deletion.
- LLVM_ABI void salvageAndRemoveInstruction(Instruction *I);
-
- DominatorTree &getDominatorTree() const { return *DT; }
- AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
- MemoryDependenceResults &getMemDep() const { return *MD; }
-
- LLVM_ABI bool isScalarPREEnabled() const;
- LLVM_ABI bool isLoadPREEnabled() const;
- LLVM_ABI bool isLoadInLoopPREEnabled() const;
- LLVM_ABI bool isLoadPRESplitBackedgeEnabled() const;
- LLVM_ABI bool isMemDepEnabled() const;
- LLVM_ABI bool isMemorySSAEnabled() const;
-
/// This class holds the mapping between values and value numbers. It is used
/// as an efficient mechanism to determine the expression-wise equivalence of
/// two values.
@@ -210,6 +183,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI ~ValueTable();
LLVM_ABI ValueTable &operator=(const ValueTable &Arg);
+ LLVM_ABI void add(Value *V, uint32_t Num);
LLVM_ABI uint32_t lookupOrAdd(MemoryAccess *MA);
LLVM_ABI uint32_t lookupOrAdd(Value *V);
LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const;
@@ -222,7 +196,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
const BasicBlock &CurrBlock);
LLVM_ABI bool exists(Value *V) const;
- LLVM_ABI void add(Value *V, uint32_t Num);
LLVM_ABI void clear();
LLVM_ABI void erase(Value *V);
void setAliasAnalysis(AAResults *A) { AA = A; }
@@ -244,6 +217,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
friend class GVNLegacyPass;
friend struct DenseMapInfo<Expression>;
+ GVNOptions Options;
MemoryDependenceResults *MD = nullptr;
DominatorTree *DT = nullptr;
const TargetLibraryInfo *TLI = nullptr;
@@ -254,7 +228,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoopInfo *LI = nullptr;
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
-
ValueTable VN;
/// A mapping from value numbers to lists of Value*'s that
@@ -346,18 +319,35 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
// of BlockRPONumber prior to accessing the contents of BlockRPONumber.
bool InvalidBlockRPONumbers = true;
+ // List of critical edges to be split between iterations.
+ SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
+
+public:
+ GVNPass(GVNOptions Options = {}) : Options(Options) {}
+
+ /// Run the pass over the function.
+ LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+ LLVM_ABI void
+ printPipeline(raw_ostream &OS,
+ function_ref<StringRef(StringRef)> MapClassName2PassName);
+
+private:
+ DominatorTree &getDominatorTree() const { return *DT; }
+ AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
+ MemoryDependenceResults &getMemDep() const { return *MD; }
+
+ bool isScalarPREEnabled() const;
+ bool isLoadPREEnabled() const;
+ bool isLoadInLoopPREEnabled() const;
+ bool isLoadPRESplitBackedgeEnabled() const;
+ bool isMemDepEnabled() const;
+ bool isMemorySSAEnabled() const;
+
using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
- bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
- const TargetLibraryInfo &RunTLI, AAResults &RunAA,
- MemoryDependenceResults *RunMD, LoopInfo &LI,
- OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
-
- // List of critical edges to be split between iterations.
- SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
-
enum class DepKind {
Other = 0, // Unknown value.
Def, // Exactly overlapping locations.
@@ -416,6 +406,41 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
+ /// Given a select-dependency for the load (the load address is a select of
+ /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
+ /// value is available by finding dominating values for both addresses. If
+ /// so, the load can be rematerialized as a select of those two values.
+ std::optional<AvailableValue>
+ analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+ Value *FalseAddr, Instruction *From);
+
+ /// Given a local dependency (Def or Clobber) determine if a value is
+ /// available for the load.
+ std::optional<AvailableValue>
+ analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ Value *Address);
+
+ /// Given a list of non-local dependencies, determine if a value is
+ /// available for the load in each specified block. If it is, add it to
+ /// ValuesPerBlock. If not, add it to UnavailableBlocks.
+ void analyzeLoadAvailability(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks);
+
+ /// Given a critical edge from Pred to LoadBB, find a load instruction
+ /// which is identical to Load from another successor of Pred.
+ LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
+ LoadInst *Load);
+
+ /// Eliminates partially redundant \p Load, replacing it with \p
+ /// AvailableLoads (connected by Phis if needed).
+ void eliminatePartiallyRedundantLoad(
+ LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ MapVector<BasicBlock *, Value *> &AvailableLoads,
+ MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
+
+ // Helper functions for d etermining load dependencies.
std::optional<GVNPass::ReachingMemVal> scanMemoryAccessesUsers(
const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
@@ -438,40 +463,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
SmallVectorImpl<ReachingMemVal> &Values,
MemorySSA &MSSA, AAResults &AA);
- // Helper functions of redundant load elimination.
- bool processLoad(LoadInst *L);
- bool processMaskedLoad(IntrinsicInst *I);
- bool processNonLocalLoad(LoadInst *L);
- bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
- bool processAssumeIntrinsic(AssumeInst *II);
-
- /// Given a local dependency (Def or Clobber) determine if a value is
- /// available for the load.
- std::optional<AvailableValue>
- analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
- Value *Address);
-
- /// Given a select-dependency for the load (the load address is a select of
- /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
- /// value is available by finding dominating values for both addresses. If
- /// so, the load can be rematerialized as a select of those two values.
- std::optional<AvailableValue>
- analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
- Value *FalseAddr, Instruction *From);
-
- /// Given a list of non-local dependencies, determine if a value is
- /// available for the load in each specified block. If it is, add it to
- /// ValuesPerBlock. If not, add it to UnavailableBlocks.
- void analyzeLoadAvailability(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks);
-
- /// Given a critical edge from Pred to LoadBB, find a load instruction
- /// which is identical to Load from another successor of Pred.
- LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
- LoadInst *Load);
-
bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
@@ -481,31 +472,63 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks);
- /// Eliminates partially redundant \p Load, replacing it with \p
- /// AvailableLoads (connected by Phis if needed).
- void eliminatePartiallyRedundantLoad(
- LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- MapVector<BasicBlock *, Value *> &AvailableLoads,
- MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
+ // Try to eliminate redundent loades with non-local dependencies.
+ bool processNonLocalLoad(LoadInst *L);
+ bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
- // Other helper routines.
+ /// Add any blocks determined to be unreachable by a conditional branch with a
+ /// constant condition to the dead blocks.
+ bool processFoldableCondBr(CondBrInst *BI);
+
+ /// Propagate equalities derived from llvm.assume intrinsics.
+ bool processAssumeIntrinsic(AssumeInst *II);
+
+ /// Try to eliminate redundant loads.
+ bool processLoad(LoadInst *L);
+
+ /// Try to eliminate masked loads which have loaded from
+ /// masked stores with the same mask.
+ bool processMaskedLoad(IntrinsicInst *I);
+
+ /// Propagate value of a condition to blocks dominated by "then" and "else"
+ /// edges, as well as certains derived equalities.
+ bool
+ propagateEquality(Value *LHS, Value *RHS,
+ const std::variant<BasicBlockEdge, Instruction *> &Root);
+
+ // Pass iteration helper functions.
bool processInstruction(Instruction *I);
bool processBlock(BasicBlock *BB);
bool iterateOnFunction(Function &F);
- bool performPRE(Function &F);
- bool performScalarPRE(Instruction *I);
+
+ // Scalar PRE helper functions
bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
BasicBlock *Curr, unsigned int ValNo);
+ bool performScalarPRE(Instruction *I);
+ bool performPRE(Function &F);
+
+ /// Main entry point for the GVN pass. Also used by the GVNLegacyPass.
+ bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+ const TargetLibraryInfo &RunTLI, AAResults &RunAA,
+ MemoryDependenceResults *RunMD, LoopInfo &LI,
+ OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
+
+ // Other helper routines.
+
Value *findLeader(const BasicBlock *BB, uint32_t Num);
void cleanupGlobalSets();
+
void removeInstruction(Instruction *I);
+
+ /// This removes the specified instruction from
+ /// our various maps and marks it for deletion.
+ void salvageAndRemoveInstruction(Instruction *I);
+
void verifyRemoved(const Instruction *I) const;
+
bool splitCriticalEdges();
BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
- bool
- propagateEquality(Value *LHS, Value *RHS,
- const std::variant<BasicBlockEdge, Instruction *> &Root);
- bool processFoldableCondBr(CondBrInst *BI);
+
void addDeadBlock(BasicBlock *BB);
void assignValNumForDeadCode();
void assignBlockRPONumber(Function &F);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 155308f0e7f71..f16d1fe9ca893 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -289,6 +289,72 @@ struct llvm::GVNPass::AvailableValue {
Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const;
};
+Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
+ Instruction *InsertPt) const {
+ Value *Res;
+ Type *LoadTy = Load->getType();
+ const DataLayout &DL = Load->getDataLayout();
+ if (isSimpleValue()) {
+ Res = getSimpleValue();
+ if (Res->getType() != LoadTy) {
+ Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
+
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
+ << " " << *getSimpleValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isCoercedLoadValue()) {
+ LoadInst *CoercedLoad = getCoercedLoadValue();
+ if (CoercedLoad->getType() == LoadTy && Offset == 0) {
+ Res = CoercedLoad;
+ combineMetadataForCSE(CoercedLoad, Load, false);
+ } else {
+ Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
+ Load->getFunction());
+ // We are adding a new user for this load, for which the original
+ // metadata may not hold. Additionally, the new load may have a different
+ // size and type, so their metadata cannot be combined in any
+ // straightforward way.
+ // Drop all metadata that is not known to cause immediate UB on violation,
+ // unless the load has !noundef, in which case all metadata violations
+ // will be promoted to UB.
+ // !noalias and !alias.scope are kept: the load is not moved and still
+ // accesses the same memory, and these are independent of the load type
+ // and offset, so they remain valid for the coerced result.
+ if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
+ CoercedLoad->dropUnknownNonDebugMetadata(
+ {LLVMContext::MD_dereferenceable,
+ LLVMContext::MD_dereferenceable_or_null,
+ LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
+ LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
+ << " " << *getCoercedLoadValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isMemIntrinValue()) {
+ Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, InsertPt,
+ DL);
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
+ << " " << *getMemIntrinValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ } else if (isSelectValue()) {
+ // Introduce a new value select for a load from an eligible pointer select.
+ Value *Cond = getSelectCondition();
+ assert(V1 && V2 && "both value operands of the select must be present");
+ Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
+ // We use the DebugLoc from the original load here, as this instruction
+ // materializes the value that would previously have been loaded.
+ cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
+ } else {
+ llvm_unreachable("Should not materialize value from dead block");
+ }
+ assert(Res && "failed to materialize?");
+ return Res;
+}
+
/// Represents an AvailableValue which can be rematerialized at the end of
/// the associated BasicBlock.
struct llvm::GVNPass::AvailableValueInBlock {
@@ -452,37 +518,6 @@ GVNPass::Expression GVNPass::ValueTable::createGEPExpr(GetElementPtrInst *GEP) {
return E;
}
-//===----------------------------------------------------------------------===//
-// ValueTable External Functions
-//===----------------------------------------------------------------------===//
-
-GVNPass::ValueTable::ValueTable() = default;
-GVNPass::ValueTable::ValueTable(const ValueTable &) = default;
-GVNPass::ValueTable::ValueTable(ValueTable &&) = default;
-GVNPass::ValueTable::~ValueTable() = default;
-GVNPass::ValueTable &
-GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default;
-
-/// add - Insert a value into the table with a specified value number.
-void GVNPass::ValueTable::add(Value *V, uint32_t Num) {
- ValueNumbering.insert(std::make_pair(V, Num));
- if (PHINode *PN = dyn_cast<PHINode>(V))
- NumberingPhi[Num] = PN;
-}
-
-/// Include the incoming memory state into the hash of the expression for the
-/// given instruction. If the incoming memory state is:
-/// * LiveOnEntry, add the value number of the entry block,
-/// * a MemoryPhi, add the value number of the basic block corresponding to that
-/// MemoryPhi,
-/// * a MemoryDef, add the value number of the memory setting instruction.
-void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
- assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
- assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
- MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
- Exp.VarArgs.push_back(lookupOrAdd(MA));
-}
-
uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) {
// FIXME: Currently the calls which may access the thread id may
// be considered as not accessing the memory. But this is
@@ -634,133 +669,326 @@ uint32_t GVNPass::ValueTable::computeLoadStoreVN(Instruction *I) {
return V;
}
-/// Returns true if a value number exists for the specified value.
-bool GVNPass::ValueTable::exists(Value *V) const {
- return ValueNumbering.contains(V);
-}
+/// Translate value number \p Num using phis, so that it has the values of
+/// the phis in BB.
+uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN) {
+ // See if we can refine the value number by looking at the PN incoming value
+ // for the given predecessor.
+ if (PHINode *PN = NumberingPhi[Num]) {
+ if (PN->getParent() != PhiBlock)
+ return Num;
+ for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) {
+ if (PN->getIncomingBlock(I) != Pred)
+ continue;
+ if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false))
+ return TransVal;
+ }
+ return Num;
+ }
-uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
- return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
- ? lookupOrAdd(MA->getBlock())
- : lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
-}
+ if (BasicBlock *BB = NumberingBB[Num]) {
+ assert(MSSA && "NumberingBB is non-empty only when using MemorySSA");
+ // Value numbers of basic blocks are used to represent memory state in
+ // load/store instructions and read-only function calls when said state is
+ // set by a MemoryPhi.
+ if (BB != PhiBlock)
+ return Num;
+ MemoryPhi *MPhi = MSSA->getMemoryAccess(BB);
+ for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) {
+ if (MPhi->getIncomingBlock(i) != Pred)
+ continue;
+ MemoryAccess *MA = MPhi->getIncomingValue(i);
+ if (auto *PredPhi = dyn_cast<MemoryPhi>(MA))
+ return lookupOrAdd(PredPhi->getBlock());
+ if (MSSA->isLiveOnEntryDef(MA))
+ return lookupOrAdd(&BB->getParent()->getEntryBlock());
+ return lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
+ }
+ llvm_unreachable(
+ "CFG/MemorySSA mismatch: predecessor not found among incoming blocks");
+ }
-/// lookupOrAdd - Returns the value number for the specified value, assigning
-/// it a new number if it did not have one before.
-uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
- auto VI = ValueNumbering.find(V);
- if (VI != ValueNumbering.end())
- return VI->second;
+ // If there is any value related with Num is defined in a BB other than
+ // PhiBlock, it cannot depend on a phi in PhiBlock without going through
+ // a backedge. We can do an early exit in that case to save compile time.
+ if (!areAllValsInBB(Num, PhiBlock, GVN))
+ return Num;
- auto *I = dyn_cast<Instruction>(V);
- if (!I) {
- ValueNumbering[V] = NextValueNumber;
- if (isa<BasicBlock>(V))
- NumberingBB[NextValueNumber] = cast<BasicBlock>(V);
- return NextValueNumber++;
+ if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
+ return Num;
+ Expression Exp = Expressions[ExprIdx[Num]];
+
+ for (unsigned I = 0; I < Exp.VarArgs.size(); I++) {
+ // For InsertValue and ExtractValue, some varargs are index numbers
+ // instead of value numbers. Those index numbers should not be
+ // translated.
+ if ((I > 1 && Exp.Opcode == Instruction::InsertValue) ||
+ (I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
+ (I > 1 && Exp.Opcode == Instruction::ShuffleVector))
+ continue;
+ Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN);
}
- Expression Exp;
- switch (I->getOpcode()) {
- case Instruction::Call:
- return lookupOrAddCall(cast<CallInst>(I));
- case Instruction::FNeg:
- case Instruction::Add:
- case Instruction::FAdd:
- case Instruction::Sub:
- case Instruction::FSub:
- case Instruction::Mul:
- case Instruction::FMul:
- case Instruction::UDiv:
- case Instruction::SDiv:
- case Instruction::FDiv:
- case Instruction::URem:
- case Instruction::SRem:
- case Instruction::FRem:
- case Instruction::Shl:
- case Instruction::LShr:
- case Instruction::AShr:
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor:
- case Instruction::ICmp:
- case Instruction::FCmp:
- case Instruction::Trunc:
- case Instruction::ZExt:
- case Instruction::SExt:
- case Instruction::FPToUI:
- case Instruction::FPToSI:
- case Instruction::UIToFP:
- case Instruction::SIToFP:
- case Instruction::FPTrunc:
- case Instruction::FPExt:
- case Instruction::PtrToInt:
- case Instruction::PtrToAddr:
- case Instruction::IntToPtr:
- case Instruction::AddrSpaceCast:
- case Instruction::BitCast:
- case Instruction::Select:
- case Instruction::Freeze:
- case Instruction::ExtractElement:
- case Instruction::InsertElement:
- case Instruction::ShuffleVector:
- case Instruction::InsertValue:
- Exp = createExpr(I);
- break;
- case Instruction::GetElementPtr:
- Exp = createGEPExpr(cast<GetElementPtrInst>(I));
- break;
- case Instruction::ExtractValue:
- Exp = createExtractvalueExpr(cast<ExtractValueInst>(I));
- break;
- case Instruction::PHI:
- ValueNumbering[V] = NextValueNumber;
- NumberingPhi[NextValueNumber] = cast<PHINode>(V);
- return NextValueNumber++;
- case Instruction::Load:
- case Instruction::Store:
- return computeLoadStoreVN(I);
- default:
- ValueNumbering[V] = NextValueNumber;
- return NextValueNumber++;
+ if (Exp.Commutative) {
+ assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!");
+ if (Exp.VarArgs[0] > Exp.VarArgs[1]) {
+ std::swap(Exp.VarArgs[0], Exp.VarArgs[1]);
+ uint32_t Opcode = Exp.Opcode >> 8;
+ if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
+ Exp.Opcode = (Opcode << 8) |
+ CmpInst::getSwappedPredicate(
+ static_cast<CmpInst::Predicate>(Exp.Opcode & 255));
+ }
}
- uint32_t E = assignExpNewValueNum(Exp).first;
- ValueNumbering[V] = E;
- return E;
+ if (uint32_t NewNum = ExpressionNumbering[Exp]) {
+ if (Exp.Opcode == Instruction::Call && NewNum != Num)
+ return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num;
+ return NewNum;
+ }
+ return Num;
}
-/// Returns the value number of the specified value. Fails if
-/// the value has not yet been numbered.
-uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
- auto VI = ValueNumbering.find(V);
- if (Verify) {
- assert(VI != ValueNumbering.end() && "Value not numbered?");
- return VI->second;
+// Return true if the value number \p Num and NewNum have equal value.
+// Return false if the result is unknown.
+bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
+ const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ GVNPass &GVN) {
+ CallInst *Call = nullptr;
+ auto Leaders = GVN.LeaderTable.getLeaders(Num);
+ for (const auto &Entry : Leaders) {
+ Call = dyn_cast<CallInst>(&*Entry.Val);
+ if (Call && Call->getParent() == PhiBlock)
+ break;
}
- return (VI != ValueNumbering.end()) ? VI->second : 0;
-}
-/// Returns the value number of the given comparison,
-/// assigning it a new number if it did not have one before. Useful when
-/// we deduced the result of a comparison, but don't immediately have an
-/// instruction realizing that comparison to hand.
-uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode,
- CmpInst::Predicate Predicate,
- Value *LHS, Value *RHS) {
- Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
- return assignExpNewValueNum(Exp).first;
-}
+ if (AA->doesNotAccessMemory(Call))
+ return true;
-/// Returns the value number of ptrtoint \p Ptr to \Ty.
-uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
- Expression Exp(Instruction::PtrToInt);
- Exp.Ty = Ty;
- Exp.VarArgs.push_back(lookupOrAdd(Ptr));
- return ExpressionNumbering.lookup(Exp);
+ if (!MD || !AA->onlyReadsMemory(Call))
+ return false;
+
+ MemDepResult LocalDep = MD->getDependency(Call);
+ if (!LocalDep.isNonLocal())
+ return false;
+
+ const MemoryDependenceResults::NonLocalDepInfo &Deps =
+ MD->getNonLocalCallDependency(Call);
+
+ // Check to see if the Call has no function local clobber.
+ for (const NonLocalDepEntry &D : Deps) {
+ if (D.getResult().isNonFuncLocal())
+ return true;
+ }
+ return false;
}
-/// Remove all entries from the ValueTable.
+/// Return a pair the first field showing the value number of \p Exp and the
+/// second field showing whether it is a value number newly created.
+std::pair<uint32_t, bool>
+GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
+ uint32_t &E = ExpressionNumbering[Exp];
+ bool CreateNewValNum = !E;
+ if (CreateNewValNum) {
+ Expressions.push_back(Exp);
+ if (ExprIdx.size() < NextValueNumber + 1)
+ ExprIdx.resize(NextValueNumber * 2);
+ E = NextValueNumber;
+ ExprIdx[NextValueNumber++] = NextExprNumber++;
+ }
+ return {E, CreateNewValNum};
+}
+
+/// Return whether all the values related with the same \p num are
+/// defined in \p BB.
+bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
+ GVNPass &GVN) {
+ return all_of(
+ GVN.LeaderTable.getLeaders(Num),
+ [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
+}
+
+/// Include the incoming memory state into the hash of the expression for the
+/// given instruction. If the incoming memory state is:
+/// * LiveOnEntry, add the value number of the entry block,
+/// * a MemoryPhi, add the value number of the basic block corresponding to that
+/// MemoryPhi,
+/// * a MemoryDef, add the value number of the memory setting instruction.
+void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
+ assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
+ assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
+ MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
+ Exp.VarArgs.push_back(lookupOrAdd(MA));
+}
+
+//===----------------------------------------------------------------------===//
+// ValueTable External Functions
+//===----------------------------------------------------------------------===//
+
+GVNPass::ValueTable::ValueTable() = default;
+GVNPass::ValueTable::ValueTable(const ValueTable &) = default;
+GVNPass::ValueTable::ValueTable(ValueTable &&) = default;
+GVNPass::ValueTable::~ValueTable() = default;
+GVNPass::ValueTable &
+GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default;
+
+/// add - Insert a value into the table with a specified value number.
+void GVNPass::ValueTable::add(Value *V, uint32_t Num) {
+ ValueNumbering.insert(std::make_pair(V, Num));
+ if (PHINode *PN = dyn_cast<PHINode>(V))
+ NumberingPhi[Num] = PN;
+}
+
+uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
+ return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
+ ? lookupOrAdd(MA->getBlock())
+ : lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
+}
+
+/// lookupOrAdd - Returns the value number for the specified value, assigning
+/// it a new number if it did not have one before.
+uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
+ auto VI = ValueNumbering.find(V);
+ if (VI != ValueNumbering.end())
+ return VI->second;
+
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I) {
+ ValueNumbering[V] = NextValueNumber;
+ if (isa<BasicBlock>(V))
+ NumberingBB[NextValueNumber] = cast<BasicBlock>(V);
+ return NextValueNumber++;
+ }
+
+ Expression Exp;
+ switch (I->getOpcode()) {
+ case Instruction::Call:
+ return lookupOrAddCall(cast<CallInst>(I));
+ case Instruction::FNeg:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ case Instruction::Trunc:
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::UIToFP:
+ case Instruction::SIToFP:
+ case Instruction::FPTrunc:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::PtrToAddr:
+ case Instruction::IntToPtr:
+ case Instruction::AddrSpaceCast:
+ case Instruction::BitCast:
+ case Instruction::Select:
+ case Instruction::Freeze:
+ case Instruction::ExtractElement:
+ case Instruction::InsertElement:
+ case Instruction::ShuffleVector:
+ case Instruction::InsertValue:
+ Exp = createExpr(I);
+ break;
+ case Instruction::GetElementPtr:
+ Exp = createGEPExpr(cast<GetElementPtrInst>(I));
+ break;
+ case Instruction::ExtractValue:
+ Exp = createExtractvalueExpr(cast<ExtractValueInst>(I));
+ break;
+ case Instruction::PHI:
+ ValueNumbering[V] = NextValueNumber;
+ NumberingPhi[NextValueNumber] = cast<PHINode>(V);
+ return NextValueNumber++;
+ case Instruction::Load:
+ case Instruction::Store:
+ return computeLoadStoreVN(I);
+ default:
+ ValueNumbering[V] = NextValueNumber;
+ return NextValueNumber++;
+ }
+
+ uint32_t E = assignExpNewValueNum(Exp).first;
+ ValueNumbering[V] = E;
+ return E;
+}
+
+/// Returns the value number of the specified value. Fails if
+/// the value has not yet been numbered.
+uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
+ auto VI = ValueNumbering.find(V);
+ if (Verify) {
+ assert(VI != ValueNumbering.end() && "Value not numbered?");
+ return VI->second;
+ }
+ return (VI != ValueNumbering.end()) ? VI->second : 0;
+}
+
+/// Returns the value number of the given comparison,
+/// assigning it a new number if it did not have one before. Useful when
+/// we deduced the result of a comparison, but don't immediately have an
+/// instruction realizing that comparison to hand.
+uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode,
+ CmpInst::Predicate Predicate,
+ Value *LHS, Value *RHS) {
+ Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
+ return assignExpNewValueNum(Exp).first;
+}
+
+/// Returns the value number of ptrtoint \p Ptr to \Ty.
+uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
+ Expression Exp(Instruction::PtrToInt);
+ Exp.Ty = Ty;
+ Exp.VarArgs.push_back(lookupOrAdd(Ptr));
+ return ExpressionNumbering.lookup(Exp);
+}
+
+/// Wrap phiTranslateImpl to provide caching functionality.
+uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN) {
+ auto FindRes = PhiTranslateTable.find({Num, Pred});
+ if (FindRes != PhiTranslateTable.end())
+ return FindRes->second;
+ uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN);
+ PhiTranslateTable.insert({{Num, Pred}, NewNum});
+ return NewNum;
+}
+
+/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
+/// again.
+void GVNPass::ValueTable::eraseTranslateCacheEntry(
+ uint32_t Num, const BasicBlock &CurrBlock) {
+ for (const BasicBlock *Pred : predecessors(&CurrBlock))
+ PhiTranslateTable.erase({Num, Pred});
+}
+
+/// Returns true if a value number exists for the specified value.
+bool GVNPass::ValueTable::exists(Value *V) const {
+ return ValueNumbering.contains(V);
+}
+
+/// Remove all entries from the ValueTable.
void GVNPass::ValueTable::clear() {
ValueNumbering.clear();
ExpressionNumbering.clear();
@@ -851,31 +1079,6 @@ void GVNPass::LeaderMap::erase(uint32_t N, Instruction *I,
// GVN Pass
//===----------------------------------------------------------------------===//
-bool GVNPass::isScalarPREEnabled() const {
- return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE);
-}
-
-bool GVNPass::isLoadPREEnabled() const {
- return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE);
-}
-
-bool GVNPass::isLoadInLoopPREEnabled() const {
- return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE);
-}
-
-bool GVNPass::isLoadPRESplitBackedgeEnabled() const {
- return Options.AllowLoadPRESplitBackedge.value_or(
- GVNEnableSplitBackedgeInLoadPRE);
-}
-
-bool GVNPass::isMemDepEnabled() const {
- return Options.AllowMemDep.value_or(GVNEnableMemDep);
-}
-
-bool GVNPass::isMemorySSAEnabled() const {
- return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA);
-}
-
PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
// FIXME: The order of evaluation of these 'getResult' calls is very
// significant! Re-ordering these variables will cause GVN when run alone to
@@ -928,10 +1131,29 @@ void GVNPass::printPipeline(
OS << '>';
}
-void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
- salvageKnowledge(I, AC);
- salvageDebugInfo(*I);
- removeInstruction(I);
+bool GVNPass::isScalarPREEnabled() const {
+ return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE);
+}
+
+bool GVNPass::isLoadPREEnabled() const {
+ return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE);
+}
+
+bool GVNPass::isLoadInLoopPREEnabled() const {
+ return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE);
+}
+
+bool GVNPass::isLoadPRESplitBackedgeEnabled() const {
+ return Options.AllowLoadPRESplitBackedge.value_or(
+ GVNEnableSplitBackedgeInLoadPRE);
+}
+
+bool GVNPass::isMemDepEnabled() const {
+ return Options.AllowMemDep.value_or(GVNEnableMemDep);
+}
+
+bool GVNPass::isMemorySSAEnabled() const {
+ return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA);
}
enum class AvailabilityState : char {
@@ -1091,12 +1313,11 @@ static void replaceValuesPerBlockEntry(
static Value *
constructSSAForLoadSet(LoadInst *Load,
SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
- GVNPass &GVN) {
+ DominatorTree &DT) {
// Check for the fully redundant, dominating load case. In this case, we can
// just use the dominating value directly.
if (ValuesPerBlock.size() == 1 &&
- GVN.getDominatorTree().properlyDominates(ValuesPerBlock[0].BB,
- Load->getParent())) {
+ DT.properlyDominates(ValuesPerBlock[0].BB, Load->getParent())) {
assert(!ValuesPerBlock[0].AV.isUndefValue() &&
"Dead BB dominate this block");
return ValuesPerBlock[0].MaterializeAdjustedValue(Load);
@@ -1132,110 +1353,44 @@ constructSSAForLoadSet(LoadInst *Load,
return SSAUpdate.GetValueInMiddleOfBlock(Load->getParent());
}
-Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
- Instruction *InsertPt) const {
- Value *Res;
- Type *LoadTy = Load->getType();
- const DataLayout &DL = Load->getDataLayout();
- if (isSimpleValue()) {
- Res = getSimpleValue();
- if (Res->getType() != LoadTy) {
- Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
+static bool isLifetimeStart(const Instruction *Inst) {
+ if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))
+ return II->getIntrinsicID() == Intrinsic::lifetime_start;
+ return false;
+}
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
- << " " << *getSimpleValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isCoercedLoadValue()) {
- LoadInst *CoercedLoad = getCoercedLoadValue();
- if (CoercedLoad->getType() == LoadTy && Offset == 0) {
- Res = CoercedLoad;
- combineMetadataForCSE(CoercedLoad, Load, false);
- } else {
- Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
- Load->getFunction());
- // We are adding a new user for this load, for which the original
- // metadata may not hold. Additionally, the new load may have a different
- // size and type, so their metadata cannot be combined in any
- // straightforward way.
- // Drop all metadata that is not known to cause immediate UB on violation,
- // unless the load has !noundef, in which case all metadata violations
- // will be promoted to UB.
- // !noalias and !alias.scope are kept: the load is not moved and still
- // accesses the same memory, and these are independent of the load type
- // and offset, so they remain valid for the coerced result.
- if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
- CoercedLoad->dropUnknownNonDebugMetadata(
- {LLVMContext::MD_dereferenceable,
- LLVMContext::MD_dereferenceable_or_null,
- LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
- LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
- << " " << *getCoercedLoadValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isMemIntrinValue()) {
- Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy,
- InsertPt, DL);
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
- << " " << *getMemIntrinValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- } else if (isSelectValue()) {
- // Introduce a new value select for a load from an eligible pointer select.
- Value *Cond = getSelectCondition();
- assert(V1 && V2 && "both value operands of the select must be present");
- Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
- // We use the DebugLoc from the original load here, as this instruction
- // materializes the value that would previously have been loaded.
- cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
- } else {
- llvm_unreachable("Should not materialize value from dead block");
- }
- assert(Res && "failed to materialize?");
- return Res;
-}
-
-static bool isLifetimeStart(const Instruction *Inst) {
- if (const IntrinsicInst* II = dyn_cast<IntrinsicInst>(Inst))
- return II->getIntrinsicID() == Intrinsic::lifetime_start;
- return false;
-}
-
-/// Assuming To can be reached from both From and Between, does Between lie on
-/// every path from From to To?
-static bool liesBetween(const Instruction *From, Instruction *Between,
- const Instruction *To, const DominatorTree *DT) {
- if (From->getParent() == Between->getParent())
- return DT->dominates(From, Between);
- SmallPtrSet<BasicBlock *, 1> Exclusion;
- Exclusion.insert(Between->getParent());
- return !isPotentiallyReachable(From, To, &Exclusion, DT);
-}
-
-static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
- const DominatorTree *DT) {
- Value *PtrOp = Load->getPointerOperand();
- if (!PtrOp->hasUseList())
- return nullptr;
-
- Instruction *OtherAccess = nullptr;
-
- for (auto *U : PtrOp->users()) {
- if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
- auto *I = cast<Instruction>(U);
- if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
- // Use the most immediately dominating value.
- if (OtherAccess) {
- if (DT->dominates(OtherAccess, I))
- OtherAccess = I;
- else
- assert(U == OtherAccess || DT->dominates(I, OtherAccess));
- } else
- OtherAccess = I;
- }
+/// Assuming To can be reached from both From and Between, does Between lie on
+/// every path from From to To?
+static bool liesBetween(const Instruction *From, Instruction *Between,
+ const Instruction *To, const DominatorTree *DT) {
+ if (From->getParent() == Between->getParent())
+ return DT->dominates(From, Between);
+ SmallPtrSet<BasicBlock *, 1> Exclusion;
+ Exclusion.insert(Between->getParent());
+ return !isPotentiallyReachable(From, To, &Exclusion, DT);
+}
+
+static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
+ const DominatorTree *DT) {
+ Value *PtrOp = Load->getPointerOperand();
+ if (!PtrOp->hasUseList())
+ return nullptr;
+
+ Instruction *OtherAccess = nullptr;
+
+ for (auto *U : PtrOp->users()) {
+ if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
+ auto *I = cast<Instruction>(U);
+ if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
+ // Use the most immediately dominating value.
+ if (OtherAccess) {
+ if (DT->dominates(OtherAccess, I))
+ OtherAccess = I;
+ else
+ assert(U == OtherAccess || DT->dominates(I, OtherAccess));
+ } else
+ OtherAccess = I;
+ }
}
}
@@ -1682,7 +1837,7 @@ void GVNPass::eliminatePartiallyRedundantLoad(
}
// Perform PHI construction.
- Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
// constructSSAForLoadSet is responsible for combining metadata.
ICF->removeUsersOf(Load);
Load->replaceAllUsesWith(V);
@@ -1699,1076 +1854,1010 @@ void GVNPass::eliminatePartiallyRedundantLoad(
salvageAndRemoveInstruction(Load);
}
-bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
- // Okay, we have *some* definitions of the value. This means that the value
- // is available in some of our (transitive) predecessors. Lets think about
- // doing PRE of this load. This will involve inserting a new load into the
- // predecessor when it's not available. We could do this in general, but
- // prefer to not increase code size. As such, we only do this when we know
- // that we only have to insert *one* load (which means we're basically moving
- // the load, not inserting a new one).
-
- SmallPtrSet<BasicBlock *, 4> Blockers(llvm::from_range, UnavailableBlocks);
+static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
+ OptimizationRemarkEmitter *ORE) {
+ using namespace ore;
- // Let's find the first basic block with more than one predecessor. Walk
- // backwards through predecessors if needed.
- BasicBlock *LoadBB = Load->getParent();
- BasicBlock *TmpBB = LoadBB;
+ ORE->emit([&]() {
+ return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
+ << "load of type " << NV("Type", Load->getType()) << " eliminated"
+ << setExtraArgs() << " in favor of "
+ << NV("InfavorOfValue", AvailableValue);
+ });
+}
- // Check that there is no implicit control flow instructions above our load in
- // its block. If there is an instruction that doesn't always pass the
- // execution to the following instruction, then moving through it may become
- // invalid. For example:
- //
- // int arr[LEN];
- // int index = ???;
- // ...
- // guard(0 <= index && index < LEN);
- // use(arr[index]);
- //
- // It is illegal to move the array access to any point above the guard,
- // because if the index is out of bounds we should deoptimize rather than
- // access the array.
- // Check that there is no guard in this block above our instruction.
- bool MustEnsureSafetyOfSpeculativeExecution =
- ICF->isDominatedByICFIFromSameBlock(Load);
+/// If a load has !invariant.group, try to find the most-dominating instruction
+/// with the same metadata and equivalent pointer (modulo bitcasts and zero
+/// GEPs). If one is found that dominates the load, its value can be reused.
+static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) {
+ Value *PointerOperand = L->getPointerOperand()->stripPointerCasts();
- while (TmpBB->getSinglePredecessor()) {
- TmpBB = TmpBB->getSinglePredecessor();
- if (TmpBB == LoadBB) // Infinite (unreachable) loop.
- return false;
- if (Blockers.count(TmpBB))
- return false;
+ // It's not safe to walk the use list of a global value because function
+ // passes aren't allowed to look outside their functions.
+ // FIXME: this could be fixed by filtering instructions from outside of
+ // current function.
+ if (isa<Constant>(PointerOperand))
+ return nullptr;
- // If any of these blocks has more than one successor (i.e. if the edge we
- // just traversed was critical), then there are other paths through this
- // block along which the load may not be anticipated. Hoisting the load
- // above this block would be adding the load to execution paths along
- // which it was not previously executed.
- if (TmpBB->getTerminator()->getNumSuccessors() != 1)
- return false;
+ // Queue to process all pointers that are equivalent to load operand.
+ SmallVector<Value *, 8> PointerUsesQueue;
+ PointerUsesQueue.push_back(PointerOperand);
- // Check that there is no implicit control flow in a block above.
- MustEnsureSafetyOfSpeculativeExecution =
- MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB);
- }
+ Instruction *MostDominatingInstruction = L;
- assert(TmpBB);
- LoadBB = TmpBB;
+ // FIXME: This loop is potentially O(n^2) due to repeated dominates checks.
+ while (!PointerUsesQueue.empty()) {
+ Value *Ptr = PointerUsesQueue.pop_back_val();
+ assert(Ptr && !isa<GlobalValue>(Ptr) &&
+ "Null or GlobalValue should not be inserted");
- // Check to see how many predecessors have the loaded value fully
- // available.
- MapVector<BasicBlock *, Value *> PredLoads;
- DenseMap<BasicBlock *, AvailabilityState> FullyAvailableBlocks;
- for (const AvailableValueInBlock &AV : ValuesPerBlock)
- FullyAvailableBlocks[AV.BB] = AvailabilityState::Available;
- for (BasicBlock *UnavailableBB : UnavailableBlocks)
- FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable;
+ for (User *U : Ptr->users()) {
+ auto *I = dyn_cast<Instruction>(U);
+ if (!I || I == L || !DT.dominates(I, MostDominatingInstruction))
+ continue;
- // The edge from Pred to LoadBB is a critical edge will be splitted.
- SmallVector<BasicBlock *, 4> CriticalEdgePredSplit;
- // The edge from Pred to LoadBB is a critical edge, another successor of Pred
- // contains a load can be moved to Pred. This data structure maps the Pred to
- // the movable load.
- MapVector<BasicBlock *, LoadInst *> CriticalEdgePredAndLoad;
- for (BasicBlock *Pred : predecessors(LoadBB)) {
- // If any predecessor block is an EH pad that does not allow non-PHI
- // instructions before the terminator, we can't PRE the load.
- if (Pred->getTerminator()->isEHPad()) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+ // Add bitcasts and zero GEPs to queue.
+ // TODO: Should drop bitcast?
+ if (isa<BitCastInst>(I) ||
+ (isa<GetElementPtrInst>(I) &&
+ cast<GetElementPtrInst>(I)->hasAllZeroIndices())) {
+ PointerUsesQueue.push_back(I);
+ continue;
+ }
- if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
- continue;
+ // If we hit a load/store with an invariant.group metadata and the same
+ // pointer operand, we can assume that value pointed to by the pointer
+ // operand didn't change.
+ if (I->hasMetadata(LLVMContext::MD_invariant_group) &&
+ Ptr == getLoadStorePointerOperand(I) && !I->isVolatile())
+ MostDominatingInstruction = I;
}
+ }
- if (Pred->getTerminator()->getNumSuccessors() != 1) {
- if (isa<IndirectBrInst>(Pred->getTerminator())) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
-
- if (LoadBB->isEHPad()) {
- LLVM_DEBUG(
- dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+ return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr;
+}
- // Do not split backedge as it will break the canonical loop form.
- if (!isLoadPRESplitBackedgeEnabled())
- if (DT->dominates(LoadBB, Pred)) {
- LLVM_DEBUG(
- dbgs()
- << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
- << Pred->getName() << "': " << *Load << '\n');
- return false;
- }
+/// Return the memory location accessed by the (masked) load/store instruction
+/// `I`, if the instruction could potentially provide a useful value for
+/// eliminating the load.
+static std::optional<MemoryLocation>
+maybeLoadStoreLocation(Instruction *I, bool AllowStores,
+ const TargetLibraryInfo *TLI) {
+ if (auto *LI = dyn_cast<LoadInst>(I))
+ return MemoryLocation::get(LI);
- if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load))
- CriticalEdgePredAndLoad[Pred] = LI;
- else
- CriticalEdgePredSplit.push_back(Pred);
- } else {
- // Only add the predecessors that will not be split for now.
- PredLoads[Pred] = nullptr;
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::masked_load:
+ return MemoryLocation::getForArgument(II, 0, TLI);
+ case Intrinsic::masked_store:
+ if (AllowStores)
+ return MemoryLocation::getForArgument(II, 1, TLI);
+ return std::nullopt;
+ default:
+ break;
}
}
- // Decide whether PRE is profitable for this load.
- unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size();
- unsigned NumUnavailablePreds = NumInsertPreds +
- CriticalEdgePredAndLoad.size();
- assert(NumUnavailablePreds != 0 &&
- "Fully available value should already be eliminated!");
- (void)NumUnavailablePreds;
-
- // If we need to insert new load in multiple predecessors, reject it.
- // FIXME: If we could restructure the CFG, we could make a common pred with
- // all the preds that don't have an available Load and insert a new load into
- // that one block.
- if (NumInsertPreds > 1)
- return false;
-
- // Now we know where we will insert load. We must ensure that it is safe
- // to speculatively execute the load at that points.
- if (MustEnsureSafetyOfSpeculativeExecution) {
- if (CriticalEdgePredSplit.size())
- if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC,
- DT))
- return false;
- for (auto &PL : PredLoads)
- if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC,
- DT))
- return false;
- for (auto &CEP : CriticalEdgePredAndLoad)
- if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC,
- DT))
- return false;
- }
-
- // Split critical edges, and update the unavailable predecessors accordingly.
- for (BasicBlock *OrigPred : CriticalEdgePredSplit) {
- BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB);
- assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!");
- PredLoads[NewPred] = nullptr;
- LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->"
- << LoadBB->getName() << '\n');
- }
-
- for (auto &CEP : CriticalEdgePredAndLoad)
- PredLoads[CEP.first] = nullptr;
+ if (!AllowStores)
+ return std::nullopt;
- // Check if the load can safely be moved to all the unavailable predecessors.
- bool CanDoPRE = true;
- const DataLayout &DL = Load->getDataLayout();
- SmallVector<Instruction*, 8> NewInsts;
- for (auto &PredLoad : PredLoads) {
- BasicBlock *UnavailablePred = PredLoad.first;
+ if (auto *SI = dyn_cast<StoreInst>(I))
+ return MemoryLocation::get(SI);
+ return std::nullopt;
+}
- // Do PHI translation to get its value in the predecessor if necessary. The
- // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
- // We do the translation for each edge we skipped by going from Load's block
- // to LoadBB, otherwise we might miss pieces needing translation.
+/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
+/// looking for memory reads whose location aliases `Loc` and dominates our
+/// load.
+std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
+ const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
+ const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
+ BatchAAResults &AA, LoadInst *L) {
- // If all preds have a single successor, then we know it is safe to insert
- // the load on the pred (?!?), so we can insert code to materialize the
- // pointer if it is not available.
- Value *LoadPtr = Load->getPointerOperand();
- BasicBlock *Cur = Load->getParent();
- while (Cur != LoadBB) {
- PHITransAddr Address(LoadPtr, DL, AC);
- LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(),
- *DT, NewInsts);
- if (!LoadPtr) {
- CanDoPRE = false;
- break;
- }
- Cur = Cur->getSinglePredecessor();
+ // Prefer a candidate that is closer to the load within the same block.
+ auto UpdateChoice = [&](std::optional<ReachingMemVal> &Choice,
+ AliasResult &AR, Instruction *Candidate) {
+ if (!Choice) {
+ if (AR == AliasResult::PartialAlias)
+ Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset());
+ else
+ Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate);
+ return;
}
+ if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst),
+ MSSA.getMemoryAccess(Candidate)))
+ return;
- if (LoadPtr) {
- PHITransAddr Address(LoadPtr, DL, AC);
- LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT,
- NewInsts);
- }
- // If we couldn't find or insert a computation of this phi translated value,
- // we fail PRE.
- if (!LoadPtr) {
- LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
- << *Load->getPointerOperand() << "\n");
- CanDoPRE = false;
- break;
+ if (AR == AliasResult::PartialAlias) {
+ Choice->Kind = DepKind::Clobber;
+ Choice->Offset = AR.getOffset();
+ } else {
+ Choice->Kind = DepKind::Def;
+ Choice->Offset = -1;
}
- PredLoad.second = LoadPtr;
- }
+ Choice->Inst = Candidate;
+ Choice->Block = Candidate->getParent();
+ };
- if (!CanDoPRE) {
- while (!NewInsts.empty()) {
- // Erase instructions generated by the failed PHI translation before
- // trying to number them. PHI translation might insert instructions
- // in basic blocks other than the current one, and we delete them
- // directly, as salvageAndRemoveInstruction only allows removing from the
- // current basic block.
- NewInsts.pop_back_val()->eraseFromParent();
- }
- // HINT: Don't revert the edge-splitting as following transformation may
- // also need to split these critical edges.
- return !CriticalEdgePredSplit.empty();
- }
+ std::optional<ReachingMemVal> ReachingVal;
+ for (MemoryAccess *MA : ClobbersList) {
+ unsigned Scanned = 0;
+ for (User *U : MA->users()) {
+ if (++Scanned >= ScanUsersLimit)
+ return ReachingMemVal::getUnknown(BB, Loc.Ptr);
- // Okay, we can eliminate this load by inserting a reload in the predecessor
- // and using PHI construction to get the value in the other predecessors, do
- // it.
- LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
- LLVM_DEBUG(if (!NewInsts.empty()) dbgs() << "INSERTED " << NewInsts.size()
- << " INSTS: " << *NewInsts.back()
- << '\n');
+ auto *UseOrDef = dyn_cast<MemoryUseOrDef>(U);
+ if (!UseOrDef || UseOrDef->getBlock() != BB)
+ continue;
- // Assign value numbers to the new instructions.
- for (Instruction *I : NewInsts) {
- // Instructions that have been inserted in predecessor(s) to materialize
- // the load address do not retain their original debug locations. Doing
- // so could lead to confusing (but correct) source attributions.
- I->updateLocationAfterHoist();
+ Instruction *MemI = UseOrDef->getMemoryInst();
+ if (MemI == L ||
+ (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L))))
+ continue;
- // FIXME: We really _ought_ to insert these value numbers into their
- // parent's availability map. However, in doing so, we risk getting into
- // ordering issues. If a block hasn't been processed yet, we would be
- // marking a value as AVAIL-IN, which isn't what we intend.
- VN.lookupOrAdd(I);
+ if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) {
+ AliasResult AR = AA.alias(*MaybeLoc, Loc);
+ // If the locations do not certainly alias, we cannot possibly infer the
+ // following load loads the same value.
+ if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias)
+ continue;
+
+ // Locations partially overlap, but neither is a subset of the other, or
+ // the second location is before the first.
+ if (AR == AliasResult::PartialAlias &&
+ (!AR.hasOffset() || AR.getOffset() < 0))
+ continue;
+
+ // Found candidate, the new load memory location and the given location
+ // must alias: precise overlap, or subset with non-negative offset.
+ UpdateChoice(ReachingVal, AR, MemI);
+ }
+ }
+ if (ReachingVal)
+ break;
}
- eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads,
- &CriticalEdgePredAndLoad);
- ++NumPRELoad;
- return true;
+ return ReachingVal;
}
-bool GVNPass::performLoopLoadPRE(LoadInst *Load,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
- const Loop *L = LI->getLoopFor(Load->getParent());
- // TODO: Generalize to other loop blocks that dominate the latch.
- if (!L || L->getHeader() != Load->getParent())
- return false;
-
- BasicBlock *Preheader = L->getLoopPreheader();
- BasicBlock *Latch = L->getLoopLatch();
- if (!Preheader || !Latch)
- return false;
+/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
+/// given location. Returns a ReachingMemVal describing the dependency.
+std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
+ MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad,
+ BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
+ assert(ClobberMA->getBlock() == BB);
- Value *LoadPtr = Load->getPointerOperand();
- // Must be available in preheader.
- if (!L->isLoopInvariant(LoadPtr))
- return false;
+ // If the clobbering access is the entry memory state, we cannot say anything
+ // about the content of the memory, except when we are accessing a local
+ // object, which can be turned later into producing `undef`.
+ if (MSSA.isLiveOnEntryDef(ClobberMA)) {
+ if (auto *Alloc = dyn_cast<AllocaInst>(getUnderlyingObject(Loc.Ptr)))
+ if (Alloc->getParent() == BB)
+ return ReachingMemVal::getDef(Loc.Ptr, const_cast<AllocaInst *>(Alloc));
+ return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ }
- // We plan to hoist the load to preheader without introducing a new fault.
- // In order to do it, we need to prove that we cannot side-exit the loop
- // once loop header is first entered before execution of the load.
- if (ICF->isDominatedByICFIFromSameBlock(Load))
- return false;
+ // Loads from "constant" memory can't be clobbered.
+ if (IsInvariantLoad || AA.pointsToConstantMemory(Loc))
+ return std::nullopt;
- BasicBlock *LoopBlock = nullptr;
- for (auto *Blocker : UnavailableBlocks) {
- // Blockers from outside the loop are handled in preheader.
- if (!L->contains(Blocker))
- continue;
+ auto GetOrdering = [](const Instruction *I) {
+ if (auto *L = dyn_cast<LoadInst>(I))
+ return L->getOrdering();
+ return cast<StoreInst>(I)->getOrdering();
+ };
+ Instruction *ClobberI = cast<MemoryDef>(ClobberMA)->getMemoryInst();
- // Only allow one loop block. Loop header is not less frequently executed
- // than each loop block, and likely it is much more frequently executed. But
- // in case of multiple loop blocks, we need extra information (such as block
- // frequency info) to understand whether it is profitable to PRE into
- // multiple loop blocks.
- if (LoopBlock)
- return false;
+ // Check if the clobbering access is a load or a store that we can reuse.
+ if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) {
+ AliasResult AR = AA.alias(*MaybeLoc, Loc);
+ if (AR == AliasResult::MustAlias)
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
- // Do not sink into inner loops. This may be non-profitable.
- if (L != LI->getLoopFor(Blocker))
- return false;
+ if (AR == AliasResult::NoAlias) {
+ // If the locations do not alias we may still be able to skip over the
+ // clobbering instruction, even if it is atomic.
+ // The original load is either non-atomic or unordered. We can reorder
+ // these across non-atomic, unordered or monotonic loads or across any
+ // store.
+ if (!ClobberI->isAtomic() ||
+ !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) ||
+ isa<StoreInst>(ClobberI))
+ return std::nullopt;
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ }
- // Blocks that dominate the latch execute on every single iteration, maybe
- // except the last one. So PREing into these blocks doesn't make much sense
- // in most cases. But the blocks that do not necessarily execute on each
- // iteration are sometimes much colder than the header, and this is when
- // PRE is potentially profitable.
- if (DT->dominates(Blocker, Latch))
- return false;
+ // Skip over volatile loads (the original load is non-volatile, non-atomic).
+ if (!ClobberI->isAtomic() && isa<LoadInst>(ClobberI))
+ return std::nullopt;
- // Make sure that the terminator itself doesn't clobber.
- if (Blocker->getTerminator()->mayWriteToMemory())
- return false;
+ if (AR == AliasResult::MayAlias ||
+ (AR == AliasResult::PartialAlias &&
+ (!AR.hasOffset() || AR.getOffset() < 0)))
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
- LoopBlock = Blocker;
+ // The only option left is a store of the superset of the required bits.
+ assert(AR == AliasResult::PartialAlias && AR.hasOffset() &&
+ AR.getOffset() > 0 &&
+ "Must be the superset/partial overlap case with positive offset");
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset());
}
- if (!LoopBlock)
- return false;
-
- // Make sure the memory at this pointer cannot be freed, therefore we can
- // safely reload from it after clobber.
- if (LoadPtr->canBeFreed())
- return false;
+ if (auto *II = dyn_cast<IntrinsicInst>(ClobberI)) {
+ if (isa<DbgInfoIntrinsic>(II))
+ return std::nullopt;
+ if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI);
+ if (AA.isMustAlias(IIObjLoc, Loc))
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ return std::nullopt;
+ }
+ }
- // TODO: Support critical edge splitting if blocker has more than 1 successor.
- MapVector<BasicBlock *, Value *> AvailableLoads;
- AvailableLoads[LoopBlock] = LoadPtr;
- AvailableLoads[Preheader] = LoadPtr;
+ // If we are at a malloc-like function call, we can turn the load into `undef`
+ // or zero.
+ if (isNoAliasCall(ClobberI)) {
+ const Value *Obj = getUnderlyingObject(Loc.Ptr);
+ if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr))
+ return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ }
- LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
- eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads,
- /*CriticalEdgePredAndLoad*/ nullptr);
- ++NumPRELoopLoad;
- return true;
-}
+ // Can reorder loads across a release fence.
+ if (auto *FI = dyn_cast<FenceInst>(ClobberI))
+ if (FI->getOrdering() == AtomicOrdering::Release)
+ return std::nullopt;
-static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
- OptimizationRemarkEmitter *ORE) {
- using namespace ore;
+ // See if the clobber instruction (e.g., a generic call) may modify the
+ // location.
+ ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc);
+ // If may modify the location, analyze deeper, to exclude accesses to
+ // non-escaping local allocations.
+ if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref)
+ return std::nullopt;
- ORE->emit([&]() {
- return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
- << "load of type " << NV("Type", Load->getType()) << " eliminated"
- << setExtraArgs() << " in favor of "
- << NV("InfavorOfValue", AvailableValue);
- });
+ // Conservatively assume the clobbering memory access may overwrite the
+ // location.
+ return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
}
-/// Attempt to eliminate a load whose dependencies are
-/// non-local by performing PHI construction.
-bool GVNPass::processNonLocalLoad(LoadInst *Load) {
- // Non-local speculations are not allowed under asan.
- if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
- Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+/// Collect the predecessors of block, while doing phi-translation of the memory
+/// address and the memory clobber. Return false if the block should be marked
+/// as clobbering the memory location in an unknown way.
+bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
+ MemoryAccess *ClobberMA,
+ DependencyBlockSet &Blocks,
+ SmallVectorImpl<BasicBlock *> &Worklist) {
+ if (Addr.needsPHITranslationFromBlock(BB) &&
+ !Addr.isPotentiallyPHITranslatable())
return false;
- // Find the non-local dependencies of the load.
- LoadDepVect Deps;
- MD->getNonLocalPointerDependency(Load, Deps);
+ auto *MPhi =
+ ClobberMA->getBlock() == BB ? dyn_cast<MemoryPhi>(ClobberMA) : nullptr;
+ SmallVector<std::pair<BasicBlock *, DependencyBlockInfo>, 8> Preds;
+ for (BasicBlock *Pred : predecessors(BB)) {
+ // Skip unreachable predecessors.
+ if (!DT->isReachableFromEntry(Pred))
+ continue;
- // If we had to process more than one hundred blocks to find the
- // dependencies, this load isn't worth worrying about. Optimizing
- // it will be too expensive.
- unsigned NumDeps = Deps.size();
- if (NumDeps > MaxNumDeps)
- return false;
+ // Skip already visited predecessors.
+ if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; }))
+ continue;
- SmallVector<ReachingMemVal, 64> MemVals;
- MemVals.reserve(Deps.size());
+ PHITransAddr TransAddr = Addr;
+ if (TransAddr.needsPHITranslationFromBlock(BB))
+ TransAddr.translateValue(BB, Pred, DT, false);
- for (const NonLocalDepResult &Dep : Deps) {
- const auto &R = Dep.getResult();
- SelectAddr SelAddr = Dep.getAddress();
- BasicBlock *BB = Dep.getBB();
- Instruction *Inst = R.getInst();
- if (R.isSelect()) {
- auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs();
- MemVals.emplace_back(
- ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second));
+ auto It = Blocks.find(Pred);
+ if (It != Blocks.end()) {
+ // If we reach a visited block with a different address, set the
+ // current block as clobbering the memory location in an unknown way
+ // (by returning false).
+ if (It->second.Addr.getAddr() != TransAddr.getAddr())
+ return false;
+ // Otherwise, just stop the traversal.
continue;
}
- Value *Address = SelAddr.getAddr();
- if (R.isClobber())
- MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst));
- else if (R.isDef())
- MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst));
- else
- MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst));
- }
- return processNonLocalLoad(Load, MemVals);
-}
-
-bool GVNPass::processNonLocalLoad(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps) {
- // If we had a phi translation failure, we'll have a single entry which is a
- // clobber in the current block. Reject this early.
- if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
- LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
- dbgs() << " has unknown dependencies\n";);
- return false;
+ Preds.emplace_back(
+ Pred, DependencyBlockInfo(TransAddr,
+ MPhi ? MPhi->getIncomingValueForBlock(Pred)
+ : ClobberMA));
}
- bool Changed = false;
- // This is a limited form of scalar PRE for load indices. If this load follows
- // a GEP, see if we can PRE the indices before analyzing.
- if (isScalarPREEnabled()) {
- if (GetElementPtrInst *GEP =
- dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
- for (Use &U : GEP->indices())
- if (Instruction *I = dyn_cast<Instruction>(U.get()))
- Changed |= performScalarPRE(I);
- }
+ // We collected the predecessors and stored them in Preds. Now, populate the
+ // worklist with the predecessors found, and cache the eventual translated
+ // address for each block.
+ for (auto &P : Preds) {
+ [[maybe_unused]] auto It =
+ Blocks.try_emplace(P.first, std::move(P.second)).first;
+ Worklist.push_back(P.first);
}
- // Step 1: Analyze the availability of the load.
- AvailValInBlkVect ValuesPerBlock;
- UnavailBlkVect UnavailableBlocks;
- analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
-
- // If we have no predecessors that produce a known value for this load, exit
- // early.
- if (ValuesPerBlock.empty())
- return Changed;
+ return true;
+}
- // Step 2: Eliminate fully redundancy.
- //
- // If all of the instructions we depend on produce a known value for this
- // load, then it is fully redundant and we can use PHI insertion to compute
- // its value. Insert PHIs and remove the fully redundant value now.
- if (UnavailableBlocks.empty()) {
- LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
+/// Build a list of MemoryAccesses whose users could potentially alias the
+/// memory location being queried. Starts from StartInfo's initial clobber,
+/// walk the use-def chain to the final clobber. If the chain extends beyond
+/// `BB`, continue into that block but only if it is in the previously collected
+/// set.
+void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
+ BasicBlock *BB,
+ const DependencyBlockInfo &StartInfo,
+ const DependencyBlockSet &Blocks,
+ MemorySSA &MSSA) {
+ MemoryAccess *MA = StartInfo.InitialClobberMA;
+ MemoryAccess *LastMA = StartInfo.ClobberMA;
- // Perform PHI construction.
- Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this);
- // constructSSAForLoadSet is responsible for combining metadata.
- ICF->removeUsersOf(Load);
- Load->replaceAllUsesWith(V);
+ for (;;) {
+ while (MA != LastMA) {
+ Clobbers.push_back(MA);
+ MA = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
+ }
+ Clobbers.push_back(MA);
- if (isa<PHINode>(V))
- V->takeName(Load);
- if (Instruction *I = dyn_cast<Instruction>(V))
- // If instruction I has debug info, then we should not update it.
- // Also, if I has a null DebugLoc, then it is still potentially incorrect
- // to propagate Load's DebugLoc because Load may not post-dominate I.
- if (Load->getDebugLoc() && Load->getParent() == I->getParent())
- I->setDebugLoc(Load->getDebugLoc());
- if (MD && V->getType()->isPtrOrPtrVectorTy())
- MD->invalidateCachedPointerInfo(V);
- ++NumGVNLoad;
- reportLoadElim(Load, V, ORE);
- salvageAndRemoveInstruction(Load);
- return true;
- }
+ if (MSSA.isLiveOnEntryDef(MA) ||
+ (MA->getBlock() == BB && !isa<MemoryPhi>(MA)))
+ break;
- // Step 3: Eliminate partial redundancy.
- if (!isLoadPREEnabled())
- return Changed;
- if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
- return Changed;
+ // If the final clobber in the current block is a MemoryPhi, go to the
+ // immediate dominator; otherwise, just get to the block containing the
+ // final clobber.
+ if (MA->getBlock() == BB)
+ BB = DT->getNode(BB)->getIDom()->getBlock();
+ else
+ BB = MA->getBlock();
- if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
- performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
- return true;
+ auto It = Blocks.find(BB);
+ if (It == Blocks.end())
+ break;
- return Changed;
+ MA = It->second.InitialClobberMA;
+ LastMA = It->second.ClobberMA;
+ if (MA == Clobbers.back())
+ Clobbers.pop_back();
+ }
}
-bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
- Value *V = IntrinsicI->getArgOperand(0);
-
- if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
- if (Cond->isZero()) {
- Type *Int8Ty = Type::getInt8Ty(V->getContext());
- Type *PtrTy = PointerType::get(V->getContext(), 0);
- // Insert a new store to null instruction before the load to indicate that
- // this code is not reachable. FIXME: We could insert unreachable
- // instruction directly because we can modify the CFG.
- auto *NewS =
- new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy),
- IntrinsicI->getIterator());
- if (MSSAU) {
- const MemoryUseOrDef *FirstNonDom = nullptr;
- const auto *AL =
- MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent());
-
- // If there are accesses in the current basic block, find the first one
- // that does not come before NewS. The new memory access is inserted
- // after the found access or before the terminator if no such access is
- // found.
- if (AL) {
- for (const auto &Acc : *AL) {
- if (auto *Current = dyn_cast<MemoryUseOrDef>(&Acc))
- if (!Current->getMemoryInst()->comesBefore(NewS)) {
- FirstNonDom = Current;
- break;
- }
- }
- }
-
- auto *NewDef =
- FirstNonDom ? MSSAU->createMemoryAccessBefore(
- NewS, nullptr,
- const_cast<MemoryUseOrDef *>(FirstNonDom))
- : MSSAU->createMemoryAccessInBB(
- NewS, nullptr,
- NewS->getParent(), MemorySSA::BeforeTerminator);
+/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
+/// Given as input a load instruction, the function computes the set of reaching
+/// memory values, one per predecessor path, that analyzeLoadAvailability can
+/// later use to establish whether the load may be eliminated. A reaching value
+/// may be of the following descriptor kind:
+/// * Def: a precise instruction that produces the exact bits the load would
+/// read (e.g., an equivalent load or a MustAlias store);
+/// * Clobber: a write that clobbers a superset of the bits the load would read
+/// (e.g., a memset over a larger region);
+/// * Other: we know which block defines the memory location in some way, but
+/// could not identify a precise instruction (e.g., memory already live at
+/// function entry).
+bool GVNPass::findReachingValuesForLoad(LoadInst *L,
+ SmallVectorImpl<ReachingMemVal> &Values,
+ MemorySSA &MSSA, AAResults &AAR) {
+ EarliestEscapeAnalysis EA(*DT, LI);
+ BatchAAResults AA(AAR, &EA);
+ BasicBlock *StartBlock = L->getParent();
+ bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load);
+ // TODO: Simplify later work by just getClobberingMemoryAccess().
+ MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess();
+ const MemoryLocation Loc = MemoryLocation::get(L);
- MSSAU->insertDef(cast<MemoryDef>(NewDef), /*RenameUses=*/false);
- }
- }
- if (isAssumeWithEmptyBundle(*IntrinsicI)) {
- salvageAndRemoveInstruction(IntrinsicI);
+ // Fast path for load tagged with !invariant.group.
+ if (L->hasMetadata(LLVMContext::MD_invariant_group)) {
+ if (Instruction *G = findInvariantGroupValue(L, *DT)) {
+ Values.emplace_back(
+ ReachingMemVal::getDef(getLoadStorePointerOperand(G), G));
return true;
}
- return false;
- }
-
- if (isa<Constant>(V)) {
- // If it's not false, and constant, it must evaluate to true. This means our
- // assume is assume(true), and thus, pointless, and we don't want to do
- // anything more here.
- return false;
}
- Constant *True = ConstantInt::getTrue(V->getContext());
- return propagateEquality(V, True, IntrinsicI);
-}
+ // Phase 1. First off, look for a local dependency to avoid having to
+ // disambiguate between before the load and after the load of the starting
+ // block (as the load may be visited from a backedge).
+ do {
+ // Scan users of the clobbering memory access.
+ if (auto RMV = scanMemoryAccessesUsers(
+ Loc, IsInvariantLoad, StartBlock,
+ SmallVector<MemoryAccess *, 1>{ClobberMA}, MSSA, AA, L)) {
+ Values.emplace_back(*RMV);
+ return true;
+ }
-static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
- patchReplacementInstruction(I, Repl);
- I->replaceAllUsesWith(Repl);
-}
+ // Exit from here, and proceed visiting predecessors if the clobbering
+ // access is non-local or is a MemoryPhi.
+ if (ClobberMA->getBlock() != StartBlock || isa<MemoryPhi>(ClobberMA))
+ break;
-/// If a load has !invariant.group, try to find the most-dominating instruction
-/// with the same metadata and equivalent pointer (modulo bitcasts and zero
-/// GEPs). If one is found that dominates the load, its value can be reused.
-static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) {
- Value *PointerOperand = L->getPointerOperand()->stripPointerCasts();
+ // Check if the clobber actually aliases the load location.
+ if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad,
+ StartBlock, MSSA, AA)) {
+ Values.emplace_back(*RMV);
+ return true;
+ }
- // It's not safe to walk the use list of a global value because function
- // passes aren't allowed to look outside their functions.
- // FIXME: this could be fixed by filtering instructions from outside of
- // current function.
- if (isa<Constant>(PointerOperand))
- return nullptr;
+ // It may happen that the clobbering memory access does not actually
+ // clobber our load location, transition to its defining memory access.
+ ClobberMA = cast<MemoryUseOrDef>(ClobberMA)->getDefiningAccess();
+ } while (ClobberMA->getBlock() == StartBlock);
- // Queue to process all pointers that are equivalent to load operand.
- SmallVector<Value *, 8> PointerUsesQueue;
- PointerUsesQueue.push_back(PointerOperand);
+ // Non-local speculations are not allowed under ASan.
+ if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
+ L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ return false;
- Instruction *MostDominatingInstruction = L;
+ // Phase 2. Walk backwards through the CFG, collecting all the blocks that
+ // contain an instruction that modifies the load memory location, or that lie
+ // on a path between a clobbering block and our load. Start off by collecting
+ // the predecessors of `StartBlock`. All the visited blocks are stored in a
+ // the set `Blocks`. If possible, the memory address maintained for the block
+ // visited does get phi-translated.
+ DependencyBlockSet Blocks;
+ SmallVector<BasicBlock *, 16> InitialWorklist;
+ const DataLayout &DL = L->getModule()->getDataLayout();
+ if (!collectPredecessors(StartBlock,
+ PHITransAddr(L->getPointerOperand(), DL, AC),
+ ClobberMA, Blocks, InitialWorklist))
+ return false;
- // FIXME: This loop is potentially O(n^2) due to repeated dominates checks.
- while (!PointerUsesQueue.empty()) {
- Value *Ptr = PointerUsesQueue.pop_back_val();
- assert(Ptr && !isa<GlobalValue>(Ptr) &&
- "Null or GlobalValue should not be inserted");
+ // Do a bottom-up DFS.
+ auto Worklist = InitialWorklist;
+ while (!Worklist.empty()) {
+ auto *BB = Worklist.pop_back_val();
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
- for (User *U : Ptr->users()) {
- auto *I = dyn_cast<Instruction>(U);
- if (!I || I == L || !DT.dominates(I, MostDominatingInstruction))
- continue;
+ // Phi-translation may have failed.
+ if (!Info.Addr.getAddr())
+ continue;
- // Add bitcasts and zero GEPs to queue.
- // TODO: Should drop bitcast?
- if (isa<BitCastInst>(I) ||
- (isa<GetElementPtrInst>(I) &&
- cast<GetElementPtrInst>(I)->hasAllZeroIndices())) {
- PointerUsesQueue.push_back(I);
+ // If the clobbering memory access is in the current block and it indeed
+ // clobbers our load location, record the dependency and do not visit the
+ // predecessors of this block further, continue with the blocks in the
+ // worklist.
+ if (Info.ClobberMA->getBlock() == BB && !isa<MemoryPhi>(Info.ClobberMA)) {
+ if (auto RMV = accessMayModifyLocation(
+ Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()),
+ IsInvariantLoad, BB, MSSA, AA)) {
+ Info.MemVal = RMV;
continue;
}
+ assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) &&
+ "LiveOnEntry aliases everything");
- // If we hit a load/store with an invariant.group metadata and the same
- // pointer operand, we can assume that value pointed to by the pointer
- // operand didn't change.
- if (I->hasMetadata(LLVMContext::MD_invariant_group) &&
- Ptr == getLoadStorePointerOperand(I) && !I->isVolatile())
- MostDominatingInstruction = I;
+ // If, however, the clobbering memory access does not actually clobber
+ // our load location, transition to its defining memory access, but
+ // keep examining the same basic block.
+ Info.ClobberMA =
+ cast<MemoryUseOrDef>(Info.ClobberMA)->getDefiningAccess();
+ Worklist.emplace_back(BB);
+ continue;
}
- }
-
- return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr;
-}
-
-/// Return the memory location accessed by the (masked) load/store instruction
-/// `I`, if the instruction could potentially provide a useful value for
-/// eliminating the load.
-static std::optional<MemoryLocation>
-maybeLoadStoreLocation(Instruction *I, bool AllowStores,
- const TargetLibraryInfo *TLI) {
- if (auto *LI = dyn_cast<LoadInst>(I))
- return MemoryLocation::get(LI);
- if (auto *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::masked_load:
- return MemoryLocation::getForArgument(II, 0, TLI);
- case Intrinsic::masked_store:
- if (AllowStores)
- return MemoryLocation::getForArgument(II, 1, TLI);
- return std::nullopt;
- default:
- break;
+ // At this point we know the current block is "transparent", i.e. the memory
+ // location is not modified when execution goes through this block.
+ // Continue to its predecessors, unless a predecessor has already been
+ // visited with a different address. We currently cannot represent such a
+ // dependency.
+ if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) {
+ Info.ForceUnknown = true;
+ continue;
}
+ if (BB != StartBlock &&
+ !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist))
+ Info.ForceUnknown = true;
}
- if (!AllowStores)
- return std::nullopt;
-
- if (auto *SI = dyn_cast<StoreInst>(I))
- return MemoryLocation::get(SI);
- return std::nullopt;
-}
+ // Phase 3. We have collected all the blocks that either write a value to the
+ // memory location of the load, or there exists a path to the load, along
+ // which the memory location is not modified. Perform a second DFS to find
+ // load-to-load dependencies; namely, look at the dominating memory reads,
+ // that alias our load. These are the MemoryUses that are users of the
+ // MemoryDefs we previously identified. If no memory read is encountered,
+ // either confirm the clobbering write found before or set to unknown.
+ Worklist = InitialWorklist;
+ for (BasicBlock *BB : Worklist) {
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
+ Info.Visited = true;
+ }
-/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
-/// looking for memory reads whose location aliases `Loc` and dominates our
-/// load.
-std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
- const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
- const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
- BatchAAResults &AA, LoadInst *L) {
+ SmallVector<MemoryAccess *> Clobbers;
+ while (!Worklist.empty()) {
+ auto *BB = Worklist.pop_back_val();
+ DependencyBlockInfo &Info = Blocks.find(BB)->second;
- // Prefer a candidate that is closer to the load within the same block.
- auto UpdateChoice = [&](std::optional<ReachingMemVal> &Choice,
- AliasResult &AR, Instruction *Candidate) {
- if (!Choice) {
- if (AR == AliasResult::PartialAlias)
- Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset());
- else
- Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate);
- return;
+ // If phi-translation failed, assume the memory location is modified in
+ // unknown way.
+ if (!Info.Addr.getAddr()) {
+ Values.push_back(ReachingMemVal::getUnknown(BB, nullptr));
+ continue;
}
- if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst),
- MSSA.getMemoryAccess(Candidate)))
- return;
- if (AR == AliasResult::PartialAlias) {
- Choice->Kind = DepKind::Clobber;
- Choice->Offset = AR.getOffset();
- } else {
- Choice->Kind = DepKind::Def;
- Choice->Offset = -1;
+ Clobbers.clear();
+ collectClobberList(Clobbers, BB, Info, Blocks, MSSA);
+ if (auto RMV =
+ scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()),
+ IsInvariantLoad, BB, Clobbers, MSSA, AA)) {
+ Values.push_back(*RMV);
+ continue;
}
- Choice->Inst = Candidate;
- Choice->Block = Candidate->getParent();
- };
+ // If no reusable memory use was found, and the current block is not
+ // transparent, use the already established memory def.
+ if (Info.MemVal) {
+ Values.push_back(*Info.MemVal);
+ continue;
+ }
- std::optional<ReachingMemVal> ReachingVal;
- for (MemoryAccess *MA : ClobbersList) {
- unsigned Scanned = 0;
- for (User *U : MA->users()) {
- if (++Scanned >= ScanUsersLimit)
- return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ if (Info.ForceUnknown) {
+ Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr()));
+ continue;
+ }
- auto *UseOrDef = dyn_cast<MemoryUseOrDef>(U);
- if (!UseOrDef || UseOrDef->getBlock() != BB)
+ // If the current block is transparent, continue to its predecessors.
+ for (BasicBlock *Pred : predecessors(BB)) {
+ auto It = Blocks.find(Pred);
+ if (It == Blocks.end())
continue;
-
- Instruction *MemI = UseOrDef->getMemoryInst();
- if (MemI == L ||
- (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L))))
+ DependencyBlockInfo &PredInfo = It->second;
+ if (PredInfo.Visited)
continue;
+ PredInfo.Visited = true;
+ Worklist.push_back(Pred);
+ }
+ }
- if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) {
- AliasResult AR = AA.alias(*MaybeLoc, Loc);
- // If the locations do not certainly alias, we cannot possibly infer the
- // following load loads the same value.
- if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias)
- continue;
+ return true;
+}
- // Locations partially overlap, but neither is a subset of the other, or
- // the second location is before the first.
- if (AR == AliasResult::PartialAlias &&
- (!AR.hasOffset() || AR.getOffset() < 0))
- continue;
+bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
+ // Okay, we have *some* definitions of the value. This means that the value
+ // is available in some of our (transitive) predecessors. Lets think about
+ // doing PRE of this load. This will involve inserting a new load into the
+ // predecessor when it's not available. We could do this in general, but
+ // prefer to not increase code size. As such, we only do this when we know
+ // that we only have to insert *one* load (which means we're basically moving
+ // the load, not inserting a new one).
- // Found candidate, the new load memory location and the given location
- // must alias: precise overlap, or subset with non-negative offset.
- UpdateChoice(ReachingVal, AR, MemI);
- }
- }
- if (ReachingVal)
- break;
- }
+ SmallPtrSet<BasicBlock *, 4> Blockers(llvm::from_range, UnavailableBlocks);
- return ReachingVal;
-}
+ // Let's find the first basic block with more than one predecessor. Walk
+ // backwards through predecessors if needed.
+ BasicBlock *LoadBB = Load->getParent();
+ BasicBlock *TmpBB = LoadBB;
-/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
-/// given location. Returns a ReachingMemVal describing the dependency.
-std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
- MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad,
- BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
- assert(ClobberMA->getBlock() == BB);
+ // Check that there is no implicit control flow instructions above our load in
+ // its block. If there is an instruction that doesn't always pass the
+ // execution to the following instruction, then moving through it may become
+ // invalid. For example:
+ //
+ // int arr[LEN];
+ // int index = ???;
+ // ...
+ // guard(0 <= index && index < LEN);
+ // use(arr[index]);
+ //
+ // It is illegal to move the array access to any point above the guard,
+ // because if the index is out of bounds we should deoptimize rather than
+ // access the array.
+ // Check that there is no guard in this block above our instruction.
+ bool MustEnsureSafetyOfSpeculativeExecution =
+ ICF->isDominatedByICFIFromSameBlock(Load);
- // If the clobbering access is the entry memory state, we cannot say anything
- // about the content of the memory, except when we are accessing a local
- // object, which can be turned later into producing `undef`.
- if (MSSA.isLiveOnEntryDef(ClobberMA)) {
- if (auto *Alloc = dyn_cast<AllocaInst>(getUnderlyingObject(Loc.Ptr)))
- if (Alloc->getParent() == BB)
- return ReachingMemVal::getDef(Loc.Ptr, const_cast<AllocaInst *>(Alloc));
- return ReachingMemVal::getUnknown(BB, Loc.Ptr);
+ while (TmpBB->getSinglePredecessor()) {
+ TmpBB = TmpBB->getSinglePredecessor();
+ if (TmpBB == LoadBB) // Infinite (unreachable) loop.
+ return false;
+ if (Blockers.count(TmpBB))
+ return false;
+
+ // If any of these blocks has more than one successor (i.e. if the edge we
+ // just traversed was critical), then there are other paths through this
+ // block along which the load may not be anticipated. Hoisting the load
+ // above this block would be adding the load to execution paths along
+ // which it was not previously executed.
+ if (TmpBB->getTerminator()->getNumSuccessors() != 1)
+ return false;
+
+ // Check that there is no implicit control flow in a block above.
+ MustEnsureSafetyOfSpeculativeExecution =
+ MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB);
}
- // Loads from "constant" memory can't be clobbered.
- if (IsInvariantLoad || AA.pointsToConstantMemory(Loc))
- return std::nullopt;
+ assert(TmpBB);
+ LoadBB = TmpBB;
- auto GetOrdering = [](const Instruction *I) {
- if (auto *L = dyn_cast<LoadInst>(I))
- return L->getOrdering();
- return cast<StoreInst>(I)->getOrdering();
- };
- Instruction *ClobberI = cast<MemoryDef>(ClobberMA)->getMemoryInst();
+ // Check to see how many predecessors have the loaded value fully
+ // available.
+ MapVector<BasicBlock *, Value *> PredLoads;
+ DenseMap<BasicBlock *, AvailabilityState> FullyAvailableBlocks;
+ for (const AvailableValueInBlock &AV : ValuesPerBlock)
+ FullyAvailableBlocks[AV.BB] = AvailabilityState::Available;
+ for (BasicBlock *UnavailableBB : UnavailableBlocks)
+ FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable;
- // Check if the clobbering access is a load or a store that we can reuse.
- if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) {
- AliasResult AR = AA.alias(*MaybeLoc, Loc);
- if (AR == AliasResult::MustAlias)
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ // The edge from Pred to LoadBB is a critical edge will be splitted.
+ SmallVector<BasicBlock *, 4> CriticalEdgePredSplit;
+ // The edge from Pred to LoadBB is a critical edge, another successor of Pred
+ // contains a load can be moved to Pred. This data structure maps the Pred to
+ // the movable load.
+ MapVector<BasicBlock *, LoadInst *> CriticalEdgePredAndLoad;
+ for (BasicBlock *Pred : predecessors(LoadBB)) {
+ // If any predecessor block is an EH pad that does not allow non-PHI
+ // instructions before the terminator, we can't PRE the load.
+ if (Pred->getTerminator()->isEHPad()) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (AR == AliasResult::NoAlias) {
- // If the locations do not alias we may still be able to skip over the
- // clobbering instruction, even if it is atomic.
- // The original load is either non-atomic or unordered. We can reorder
- // these across non-atomic, unordered or monotonic loads or across any
- // store.
- if (!ClobberI->isAtomic() ||
- !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) ||
- isa<StoreInst>(ClobberI))
- return std::nullopt;
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
+ continue;
}
- // Skip over volatile loads (the original load is non-volatile, non-atomic).
- if (!ClobberI->isAtomic() && isa<LoadInst>(ClobberI))
- return std::nullopt;
+ if (Pred->getTerminator()->getNumSuccessors() != 1) {
+ if (isa<IndirectBrInst>(Pred->getTerminator())) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (AR == AliasResult::MayAlias ||
- (AR == AliasResult::PartialAlias &&
- (!AR.hasOffset() || AR.getOffset() < 0)))
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
+ if (LoadBB->isEHPad()) {
+ LLVM_DEBUG(
+ dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- // The only option left is a store of the superset of the required bits.
- assert(AR == AliasResult::PartialAlias && AR.hasOffset() &&
- AR.getOffset() > 0 &&
- "Must be the superset/partial overlap case with positive offset");
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset());
- }
+ // Do not split backedge as it will break the canonical loop form.
+ if (!isLoadPRESplitBackedgeEnabled())
+ if (DT->dominates(LoadBB, Pred)) {
+ LLVM_DEBUG(
+ dbgs()
+ << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
+ << Pred->getName() << "': " << *Load << '\n');
+ return false;
+ }
- if (auto *II = dyn_cast<IntrinsicInst>(ClobberI)) {
- if (isa<DbgInfoIntrinsic>(II))
- return std::nullopt;
- if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
- MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI);
- if (AA.isMustAlias(IIObjLoc, Loc))
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
- return std::nullopt;
+ if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load))
+ CriticalEdgePredAndLoad[Pred] = LI;
+ else
+ CriticalEdgePredSplit.push_back(Pred);
+ } else {
+ // Only add the predecessors that will not be split for now.
+ PredLoads[Pred] = nullptr;
}
}
- // If we are at a malloc-like function call, we can turn the load into `undef`
- // or zero.
- if (isNoAliasCall(ClobberI)) {
- const Value *Obj = getUnderlyingObject(Loc.Ptr);
- if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr))
- return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
+ // Decide whether PRE is profitable for this load.
+ unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size();
+ unsigned NumUnavailablePreds =
+ NumInsertPreds + CriticalEdgePredAndLoad.size();
+ assert(NumUnavailablePreds != 0 &&
+ "Fully available value should already be eliminated!");
+ (void)NumUnavailablePreds;
+
+ // If we need to insert new load in multiple predecessors, reject it.
+ // FIXME: If we could restructure the CFG, we could make a common pred with
+ // all the preds that don't have an available Load and insert a new load into
+ // that one block.
+ if (NumInsertPreds > 1)
+ return false;
+
+ // Now we know where we will insert load. We must ensure that it is safe
+ // to speculatively execute the load at that points.
+ if (MustEnsureSafetyOfSpeculativeExecution) {
+ if (CriticalEdgePredSplit.size())
+ if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC,
+ DT))
+ return false;
+ for (auto &PL : PredLoads)
+ if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC,
+ DT))
+ return false;
+ for (auto &CEP : CriticalEdgePredAndLoad)
+ if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC,
+ DT))
+ return false;
}
- // Can reorder loads across a release fence.
- if (auto *FI = dyn_cast<FenceInst>(ClobberI))
- if (FI->getOrdering() == AtomicOrdering::Release)
- return std::nullopt;
+ // Split critical edges, and update the unavailable predecessors accordingly.
+ for (BasicBlock *OrigPred : CriticalEdgePredSplit) {
+ BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB);
+ assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!");
+ PredLoads[NewPred] = nullptr;
+ LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->"
+ << LoadBB->getName() << '\n');
+ }
- // See if the clobber instruction (e.g., a generic call) may modify the
- // location.
- ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc);
- // If may modify the location, analyze deeper, to exclude accesses to
- // non-escaping local allocations.
- if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref)
- return std::nullopt;
+ for (auto &CEP : CriticalEdgePredAndLoad)
+ PredLoads[CEP.first] = nullptr;
- // Conservatively assume the clobbering memory access may overwrite the
- // location.
- return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
-}
+ // Check if the load can safely be moved to all the unavailable predecessors.
+ bool CanDoPRE = true;
+ const DataLayout &DL = Load->getDataLayout();
+ SmallVector<Instruction *, 8> NewInsts;
+ for (auto &PredLoad : PredLoads) {
+ BasicBlock *UnavailablePred = PredLoad.first;
-/// Collect the predecessors of block, while doing phi-translation of the memory
-/// address and the memory clobber. Return false if the block should be marked
-/// as clobbering the memory location in an unknown way.
-bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
- MemoryAccess *ClobberMA,
- DependencyBlockSet &Blocks,
- SmallVectorImpl<BasicBlock *> &Worklist) {
- if (Addr.needsPHITranslationFromBlock(BB) &&
- !Addr.isPotentiallyPHITranslatable())
- return false;
+ // Do PHI translation to get its value in the predecessor if necessary. The
+ // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
+ // We do the translation for each edge we skipped by going from Load's block
+ // to LoadBB, otherwise we might miss pieces needing translation.
- auto *MPhi =
- ClobberMA->getBlock() == BB ? dyn_cast<MemoryPhi>(ClobberMA) : nullptr;
- SmallVector<std::pair<BasicBlock *, DependencyBlockInfo>, 8> Preds;
- for (BasicBlock *Pred : predecessors(BB)) {
- // Skip unreachable predecessors.
- if (!DT->isReachableFromEntry(Pred))
- continue;
+ // If all preds have a single successor, then we know it is safe to insert
+ // the load on the pred (?!?), so we can insert code to materialize the
+ // pointer if it is not available.
+ Value *LoadPtr = Load->getPointerOperand();
+ BasicBlock *Cur = Load->getParent();
+ while (Cur != LoadBB) {
+ PHITransAddr Address(LoadPtr, DL, AC);
+ LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(),
+ *DT, NewInsts);
+ if (!LoadPtr) {
+ CanDoPRE = false;
+ break;
+ }
+ Cur = Cur->getSinglePredecessor();
+ }
- // Skip already visited predecessors.
- if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; }))
- continue;
+ if (LoadPtr) {
+ PHITransAddr Address(LoadPtr, DL, AC);
+ LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT,
+ NewInsts);
+ }
+ // If we couldn't find or insert a computation of this phi translated value,
+ // we fail PRE.
+ if (!LoadPtr) {
+ LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
+ << *Load->getPointerOperand() << "\n");
+ CanDoPRE = false;
+ break;
+ }
- PHITransAddr TransAddr = Addr;
- if (TransAddr.needsPHITranslationFromBlock(BB))
- TransAddr.translateValue(BB, Pred, DT, false);
+ PredLoad.second = LoadPtr;
+ }
- auto It = Blocks.find(Pred);
- if (It != Blocks.end()) {
- // If we reach a visited block with a different address, set the
- // current block as clobbering the memory location in an unknown way
- // (by returning false).
- if (It->second.Addr.getAddr() != TransAddr.getAddr())
- return false;
- // Otherwise, just stop the traversal.
- continue;
+ if (!CanDoPRE) {
+ while (!NewInsts.empty()) {
+ // Erase instructions generated by the failed PHI translation before
+ // trying to number them. PHI translation might insert instructions
+ // in basic blocks other than the current one, and we delete them
+ // directly, as salvageAndRemoveInstruction only allows removing from the
+ // current basic block.
+ NewInsts.pop_back_val()->eraseFromParent();
}
-
- Preds.emplace_back(
- Pred, DependencyBlockInfo(TransAddr,
- MPhi ? MPhi->getIncomingValueForBlock(Pred)
- : ClobberMA));
+ // HINT: Don't revert the edge-splitting as following transformation may
+ // also need to split these critical edges.
+ return !CriticalEdgePredSplit.empty();
}
- // We collected the predecessors and stored them in Preds. Now, populate the
- // worklist with the predecessors found, and cache the eventual translated
- // address for each block.
- for (auto &P : Preds) {
- [[maybe_unused]] auto It =
- Blocks.try_emplace(P.first, std::move(P.second)).first;
- Worklist.push_back(P.first);
+ // Okay, we can eliminate this load by inserting a reload in the predecessor
+ // and using PHI construction to get the value in the other predecessors, do
+ // it.
+ LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
+ LLVM_DEBUG(if (!NewInsts.empty()) dbgs()
+ << "INSERTED " << NewInsts.size() << " INSTS: " << *NewInsts.back()
+ << '\n');
+
+ // Assign value numbers to the new instructions.
+ for (Instruction *I : NewInsts) {
+ // Instructions that have been inserted in predecessor(s) to materialize
+ // the load address do not retain their original debug locations. Doing
+ // so could lead to confusing (but correct) source attributions.
+ I->updateLocationAfterHoist();
+
+ // FIXME: We really _ought_ to insert these value numbers into their
+ // parent's availability map. However, in doing so, we risk getting into
+ // ordering issues. If a block hasn't been processed yet, we would be
+ // marking a value as AVAIL-IN, which isn't what we intend.
+ VN.lookupOrAdd(I);
}
+ eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads,
+ &CriticalEdgePredAndLoad);
+ ++NumPRELoad;
return true;
}
-/// Build a list of MemoryAccesses whose users could potentially alias the
-/// memory location being queried. Starts from StartInfo's initial clobber,
-/// walk the use-def chain to the final clobber. If the chain extends beyond
-/// `BB`, continue into that block but only if it is in the previously collected
-/// set.
-void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
- BasicBlock *BB,
- const DependencyBlockInfo &StartInfo,
- const DependencyBlockSet &Blocks,
- MemorySSA &MSSA) {
- MemoryAccess *MA = StartInfo.InitialClobberMA;
- MemoryAccess *LastMA = StartInfo.ClobberMA;
-
- for (;;) {
- while (MA != LastMA) {
- Clobbers.push_back(MA);
- MA = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
- }
- Clobbers.push_back(MA);
-
- if (MSSA.isLiveOnEntryDef(MA) ||
- (MA->getBlock() == BB && !isa<MemoryPhi>(MA)))
- break;
+bool GVNPass::performLoopLoadPRE(LoadInst *Load,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
+ const Loop *L = LI->getLoopFor(Load->getParent());
+ // TODO: Generalize to other loop blocks that dominate the latch.
+ if (!L || L->getHeader() != Load->getParent())
+ return false;
- // If the final clobber in the current block is a MemoryPhi, go to the
- // immediate dominator; otherwise, just get to the block containing the
- // final clobber.
- if (MA->getBlock() == BB)
- BB = DT->getNode(BB)->getIDom()->getBlock();
- else
- BB = MA->getBlock();
+ BasicBlock *Preheader = L->getLoopPreheader();
+ BasicBlock *Latch = L->getLoopLatch();
+ if (!Preheader || !Latch)
+ return false;
- auto It = Blocks.find(BB);
- if (It == Blocks.end())
- break;
+ Value *LoadPtr = Load->getPointerOperand();
+ // Must be available in preheader.
+ if (!L->isLoopInvariant(LoadPtr))
+ return false;
- MA = It->second.InitialClobberMA;
- LastMA = It->second.ClobberMA;
- if (MA == Clobbers.back())
- Clobbers.pop_back();
- }
-}
+ // We plan to hoist the load to preheader without introducing a new fault.
+ // In order to do it, we need to prove that we cannot side-exit the loop
+ // once loop header is first entered before execution of the load.
+ if (ICF->isDominatedByICFIFromSameBlock(Load))
+ return false;
-/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
-/// Given as input a load instruction, the function computes the set of reaching
-/// memory values, one per predecessor path, that analyzeLoadAvailability can
-/// later use to establish whether the load may be eliminated. A reaching value
-/// may be of the following descriptor kind:
-/// * Def: a precise instruction that produces the exact bits the load would
-/// read (e.g., an equivalent load or a MustAlias store);
-/// * Clobber: a write that clobbers a superset of the bits the load would read
-/// (e.g., a memset over a larger region);
-/// * Other: we know which block defines the memory location in some way, but
-/// could not identify a precise instruction (e.g., memory already live at
-/// function entry).
-bool GVNPass::findReachingValuesForLoad(LoadInst *L,
- SmallVectorImpl<ReachingMemVal> &Values,
- MemorySSA &MSSA, AAResults &AAR) {
- EarliestEscapeAnalysis EA(*DT, LI);
- BatchAAResults AA(AAR, &EA);
- BasicBlock *StartBlock = L->getParent();
- bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load);
- // TODO: Simplify later work by just getClobberingMemoryAccess().
- MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess();
- const MemoryLocation Loc = MemoryLocation::get(L);
+ BasicBlock *LoopBlock = nullptr;
+ for (auto *Blocker : UnavailableBlocks) {
+ // Blockers from outside the loop are handled in preheader.
+ if (!L->contains(Blocker))
+ continue;
- // Fast path for load tagged with !invariant.group.
- if (L->hasMetadata(LLVMContext::MD_invariant_group)) {
- if (Instruction *G = findInvariantGroupValue(L, *DT)) {
- Values.emplace_back(
- ReachingMemVal::getDef(getLoadStorePointerOperand(G), G));
- return true;
- }
- }
+ // Only allow one loop block. Loop header is not less frequently executed
+ // than each loop block, and likely it is much more frequently executed. But
+ // in case of multiple loop blocks, we need extra information (such as block
+ // frequency info) to understand whether it is profitable to PRE into
+ // multiple loop blocks.
+ if (LoopBlock)
+ return false;
- // Phase 1. First off, look for a local dependency to avoid having to
- // disambiguate between before the load and after the load of the starting
- // block (as the load may be visited from a backedge).
- do {
- // Scan users of the clobbering memory access.
- if (auto RMV = scanMemoryAccessesUsers(
- Loc, IsInvariantLoad, StartBlock,
- SmallVector<MemoryAccess *, 1>{ClobberMA}, MSSA, AA, L)) {
- Values.emplace_back(*RMV);
- return true;
- }
+ // Do not sink into inner loops. This may be non-profitable.
+ if (L != LI->getLoopFor(Blocker))
+ return false;
- // Exit from here, and proceed visiting predecessors if the clobbering
- // access is non-local or is a MemoryPhi.
- if (ClobberMA->getBlock() != StartBlock || isa<MemoryPhi>(ClobberMA))
- break;
+ // Blocks that dominate the latch execute on every single iteration, maybe
+ // except the last one. So PREing into these blocks doesn't make much sense
+ // in most cases. But the blocks that do not necessarily execute on each
+ // iteration are sometimes much colder than the header, and this is when
+ // PRE is potentially profitable.
+ if (DT->dominates(Blocker, Latch))
+ return false;
- // Check if the clobber actually aliases the load location.
- if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad,
- StartBlock, MSSA, AA)) {
- Values.emplace_back(*RMV);
- return true;
- }
+ // Make sure that the terminator itself doesn't clobber.
+ if (Blocker->getTerminator()->mayWriteToMemory())
+ return false;
- // It may happen that the clobbering memory access does not actually
- // clobber our load location, transition to its defining memory access.
- ClobberMA = cast<MemoryUseOrDef>(ClobberMA)->getDefiningAccess();
- } while (ClobberMA->getBlock() == StartBlock);
+ LoopBlock = Blocker;
+ }
- // Non-local speculations are not allowed under ASan.
- if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
- L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ if (!LoopBlock)
return false;
- // Phase 2. Walk backwards through the CFG, collecting all the blocks that
- // contain an instruction that modifies the load memory location, or that lie
- // on a path between a clobbering block and our load. Start off by collecting
- // the predecessors of `StartBlock`. All the visited blocks are stored in a
- // the set `Blocks`. If possible, the memory address maintained for the block
- // visited does get phi-translated.
- DependencyBlockSet Blocks;
- SmallVector<BasicBlock *, 16> InitialWorklist;
- const DataLayout &DL = L->getModule()->getDataLayout();
- if (!collectPredecessors(StartBlock,
- PHITransAddr(L->getPointerOperand(), DL, AC),
- ClobberMA, Blocks, InitialWorklist))
+ // Make sure the memory at this pointer cannot be freed, therefore we can
+ // safely reload from it after clobber.
+ if (LoadPtr->canBeFreed())
return false;
- // Do a bottom-up DFS.
- auto Worklist = InitialWorklist;
- while (!Worklist.empty()) {
- auto *BB = Worklist.pop_back_val();
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
+ // TODO: Support critical edge splitting if blocker has more than 1 successor.
+ MapVector<BasicBlock *, Value *> AvailableLoads;
+ AvailableLoads[LoopBlock] = LoadPtr;
+ AvailableLoads[Preheader] = LoadPtr;
+
+ LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
+ eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads,
+ /*CriticalEdgePredAndLoad*/ nullptr);
+ ++NumPRELoopLoad;
+ return true;
+}
+
+/// Attempt to eliminate a load whose dependencies are
+/// non-local by performing PHI construction.
+bool GVNPass::processNonLocalLoad(LoadInst *Load) {
+ // Non-local speculations are not allowed under asan.
+ if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
+ Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
+ return false;
- // Phi-translation may have failed.
- if (!Info.Addr.getAddr())
- continue;
+ // Find the non-local dependencies of the load.
+ LoadDepVect Deps;
+ MD->getNonLocalPointerDependency(Load, Deps);
- // If the clobbering memory access is in the current block and it indeed
- // clobbers our load location, record the dependency and do not visit the
- // predecessors of this block further, continue with the blocks in the
- // worklist.
- if (Info.ClobberMA->getBlock() == BB && !isa<MemoryPhi>(Info.ClobberMA)) {
- if (auto RMV = accessMayModifyLocation(
- Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()),
- IsInvariantLoad, BB, MSSA, AA)) {
- Info.MemVal = RMV;
- continue;
- }
- assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) &&
- "LiveOnEntry aliases everything");
+ // If we had to process more than one hundred blocks to find the
+ // dependencies, this load isn't worth worrying about. Optimizing
+ // it will be too expensive.
+ unsigned NumDeps = Deps.size();
+ if (NumDeps > MaxNumDeps)
+ return false;
- // If, however, the clobbering memory access does not actually clobber
- // our load location, transition to its defining memory access, but
- // keep examining the same basic block.
- Info.ClobberMA =
- cast<MemoryUseOrDef>(Info.ClobberMA)->getDefiningAccess();
- Worklist.emplace_back(BB);
- continue;
- }
+ SmallVector<ReachingMemVal, 64> MemVals;
+ MemVals.reserve(Deps.size());
- // At this point we know the current block is "transparent", i.e. the memory
- // location is not modified when execution goes through this block.
- // Continue to its predecessors, unless a predecessor has already been
- // visited with a different address. We currently cannot represent such a
- // dependency.
- if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) {
- Info.ForceUnknown = true;
+ for (const NonLocalDepResult &Dep : Deps) {
+ const auto &R = Dep.getResult();
+ SelectAddr SelAddr = Dep.getAddress();
+ BasicBlock *BB = Dep.getBB();
+ Instruction *Inst = R.getInst();
+ if (R.isSelect()) {
+ auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs();
+ MemVals.emplace_back(
+ ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second));
continue;
}
- if (BB != StartBlock &&
- !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist))
- Info.ForceUnknown = true;
+ Value *Address = SelAddr.getAddr();
+ if (R.isClobber())
+ MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst));
+ else if (R.isDef())
+ MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst));
+ else
+ MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst));
}
- // Phase 3. We have collected all the blocks that either write a value to the
- // memory location of the load, or there exists a path to the load, along
- // which the memory location is not modified. Perform a second DFS to find
- // load-to-load dependencies; namely, look at the dominating memory reads,
- // that alias our load. These are the MemoryUses that are users of the
- // MemoryDefs we previously identified. If no memory read is encountered,
- // either confirm the clobbering write found before or set to unknown.
- Worklist = InitialWorklist;
- for (BasicBlock *BB : Worklist) {
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
- Info.Visited = true;
- }
+ return processNonLocalLoad(Load, MemVals);
+}
- SmallVector<MemoryAccess *> Clobbers;
- while (!Worklist.empty()) {
- auto *BB = Worklist.pop_back_val();
- DependencyBlockInfo &Info = Blocks.find(BB)->second;
+bool GVNPass::processNonLocalLoad(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps) {
+ // If we had a phi translation failure, we'll have a single entry which is a
+ // clobber in the current block. Reject this early.
+ if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
+ LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
+ dbgs() << " has unknown dependencies\n";);
+ return false;
+ }
- // If phi-translation failed, assume the memory location is modified in
- // unknown way.
- if (!Info.Addr.getAddr()) {
- Values.push_back(ReachingMemVal::getUnknown(BB, nullptr));
- continue;
+ bool Changed = false;
+ // This is a limited form of scalar PRE for load indices. If this load follows
+ // a GEP, see if we can PRE the indices before analyzing.
+ if (isScalarPREEnabled()) {
+ if (GetElementPtrInst *GEP =
+ dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
+ for (Use &U : GEP->indices())
+ if (Instruction *I = dyn_cast<Instruction>(U.get()))
+ Changed |= performScalarPRE(I);
}
+ }
- Clobbers.clear();
- collectClobberList(Clobbers, BB, Info, Blocks, MSSA);
- if (auto RMV =
- scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()),
- IsInvariantLoad, BB, Clobbers, MSSA, AA)) {
- Values.push_back(*RMV);
- continue;
- }
+ // Step 1: Analyze the availability of the load.
+ AvailValInBlkVect ValuesPerBlock;
+ UnavailBlkVect UnavailableBlocks;
+ analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
- // If no reusable memory use was found, and the current block is not
- // transparent, use the already established memory def.
- if (Info.MemVal) {
- Values.push_back(*Info.MemVal);
- continue;
- }
+ // If we have no predecessors that produce a known value for this load, exit
+ // early.
+ if (ValuesPerBlock.empty())
+ return Changed;
- if (Info.ForceUnknown) {
- Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr()));
- continue;
- }
+ // Step 2: Eliminate fully redundancy.
+ //
+ // If all of the instructions we depend on produce a known value for this
+ // load, then it is fully redundant and we can use PHI insertion to compute
+ // its value. Insert PHIs and remove the fully redundant value now.
+ if (UnavailableBlocks.empty()) {
+ LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
- // If the current block is transparent, continue to its predecessors.
- for (BasicBlock *Pred : predecessors(BB)) {
- auto It = Blocks.find(Pred);
- if (It == Blocks.end())
- continue;
- DependencyBlockInfo &PredInfo = It->second;
- if (PredInfo.Visited)
- continue;
- PredInfo.Visited = true;
- Worklist.push_back(Pred);
- }
+ // Perform PHI construction.
+ Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
+ // constructSSAForLoadSet is responsible for combining metadata.
+ ICF->removeUsersOf(Load);
+ Load->replaceAllUsesWith(V);
+
+ if (isa<PHINode>(V))
+ V->takeName(Load);
+ if (Instruction *I = dyn_cast<Instruction>(V))
+ // If instruction I has debug info, then we should not update it.
+ // Also, if I has a null DebugLoc, then it is still potentially incorrect
+ // to propagate Load's DebugLoc because Load may not post-dominate I.
+ if (Load->getDebugLoc() && Load->getParent() == I->getParent())
+ I->setDebugLoc(Load->getDebugLoc());
+ if (MD && V->getType()->isPtrOrPtrVectorTy())
+ MD->invalidateCachedPointerInfo(V);
+ ++NumGVNLoad;
+ reportLoadElim(Load, V, ORE);
+ salvageAndRemoveInstruction(Load);
+ return true;
}
- return true;
+ // Step 3: Eliminate partial redundancy.
+ if (!isLoadPREEnabled())
+ return Changed;
+ if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
+ return Changed;
+
+ if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
+ performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
+ return true;
+
+ return Changed;
}
/// Attempt to eliminate a load, first by eliminating it
@@ -2874,188 +2963,99 @@ bool GVNPass::processMaskedLoad(IntrinsicInst *I) {
return true;
}
-/// Return a pair the first field showing the value number of \p Exp and the
-/// second field showing whether it is a value number newly created.
-std::pair<uint32_t, bool>
-GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
- uint32_t &E = ExpressionNumbering[Exp];
- bool CreateNewValNum = !E;
- if (CreateNewValNum) {
- Expressions.push_back(Exp);
- if (ExprIdx.size() < NextValueNumber + 1)
- ExprIdx.resize(NextValueNumber * 2);
- E = NextValueNumber;
- ExprIdx[NextValueNumber++] = NextExprNumber++;
- }
- return {E, CreateNewValNum};
-}
-
-/// Return whether all the values related with the same \p num are
-/// defined in \p BB.
-bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
- GVNPass &GVN) {
- return all_of(
- GVN.LeaderTable.getLeaders(Num),
- [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
-}
-
-/// Wrap phiTranslateImpl to provide caching functionality.
-uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
- auto FindRes = PhiTranslateTable.find({Num, Pred});
- if (FindRes != PhiTranslateTable.end())
- return FindRes->second;
- uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN);
- PhiTranslateTable.insert({{Num, Pred}, NewNum});
- return NewNum;
-}
-
-// Return true if the value number \p Num and NewNum have equal value.
-// Return false if the result is unknown.
-bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
- const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- GVNPass &GVN) {
- CallInst *Call = nullptr;
- auto Leaders = GVN.LeaderTable.getLeaders(Num);
- for (const auto &Entry : Leaders) {
- Call = dyn_cast<CallInst>(&*Entry.Val);
- if (Call && Call->getParent() == PhiBlock)
- break;
- }
-
- if (AA->doesNotAccessMemory(Call))
- return true;
+// If the given branch is recognized as a foldable branch (i.e. conditional
+// branch with constant condition), it will perform following analyses and
+// transformation.
+// 1) If the dead out-coming edge is a critical-edge, split it. Let
+// R be the target of the dead out-coming edge.
+// 1) Identify the set of dead blocks implied by the branch's dead outcoming
+// edge. The result of this step will be {X| X is dominated by R}
+// 2) Identify those blocks which haves at least one dead predecessor. The
+// result of this step will be dominance-frontier(R).
+// 3) Update the PHIs in DF(R) by replacing the operands corresponding to
+// dead blocks with "UndefVal" in an hope these PHIs will optimized away.
+//
+// Return true iff *NEW* dead code are found.
+bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
+ // If a branch has two identical successors, we cannot declare either dead.
+ if (BI->getSuccessor(0) == BI->getSuccessor(1))
+ return false;
- if (!MD || !AA->onlyReadsMemory(Call))
+ ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
+ if (!Cond)
return false;
- MemDepResult LocalDep = MD->getDependency(Call);
- if (!LocalDep.isNonLocal())
+ BasicBlock *DeadRoot =
+ Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0);
+ if (DeadBlocks.count(DeadRoot))
return false;
- const MemoryDependenceResults::NonLocalDepInfo &Deps =
- MD->getNonLocalCallDependency(Call);
+ if (!DeadRoot->getSinglePredecessor())
+ DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot);
- // Check to see if the Call has no function local clobber.
- for (const NonLocalDepEntry &D : Deps) {
- if (D.getResult().isNonFuncLocal())
- return true;
- }
- return false;
+ addDeadBlock(DeadRoot);
+ return true;
}
-/// Translate value number \p Num using phis, so that it has the values of
-/// the phis in BB.
-uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
- // See if we can refine the value number by looking at the PN incoming value
- // for the given predecessor.
- if (PHINode *PN = NumberingPhi[Num]) {
- if (PN->getParent() != PhiBlock)
- return Num;
- for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) {
- if (PN->getIncomingBlock(I) != Pred)
- continue;
- if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false))
- return TransVal;
- }
- return Num;
- }
-
- if (BasicBlock *BB = NumberingBB[Num]) {
- assert(MSSA && "NumberingBB is non-empty only when using MemorySSA");
- // Value numbers of basic blocks are used to represent memory state in
- // load/store instructions and read-only function calls when said state is
- // set by a MemoryPhi.
- if (BB != PhiBlock)
- return Num;
- MemoryPhi *MPhi = MSSA->getMemoryAccess(BB);
- for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) {
- if (MPhi->getIncomingBlock(i) != Pred)
- continue;
- MemoryAccess *MA = MPhi->getIncomingValue(i);
- if (auto *PredPhi = dyn_cast<MemoryPhi>(MA))
- return lookupOrAdd(PredPhi->getBlock());
- if (MSSA->isLiveOnEntryDef(MA))
- return lookupOrAdd(&BB->getParent()->getEntryBlock());
- return lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
- }
- llvm_unreachable(
- "CFG/MemorySSA mismatch: predecessor not found among incoming blocks");
- }
+bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
+ Value *V = IntrinsicI->getArgOperand(0);
- // If there is any value related with Num is defined in a BB other than
- // PhiBlock, it cannot depend on a phi in PhiBlock without going through
- // a backedge. We can do an early exit in that case to save compile time.
- if (!areAllValsInBB(Num, PhiBlock, GVN))
- return Num;
+ if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
+ if (Cond->isZero()) {
+ Type *Int8Ty = Type::getInt8Ty(V->getContext());
+ Type *PtrTy = PointerType::get(V->getContext(), 0);
+ // Insert a new store to null instruction before the load to indicate that
+ // this code is not reachable. FIXME: We could insert unreachable
+ // instruction directly because we can modify the CFG.
+ auto *NewS =
+ new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy),
+ IntrinsicI->getIterator());
+ if (MSSAU) {
+ const MemoryUseOrDef *FirstNonDom = nullptr;
+ const auto *AL =
+ MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent());
- if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
- return Num;
- Expression Exp = Expressions[ExprIdx[Num]];
+ // If there are accesses in the current basic block, find the first one
+ // that does not come before NewS. The new memory access is inserted
+ // after the found access or before the terminator if no such access is
+ // found.
+ if (AL) {
+ for (const auto &Acc : *AL) {
+ if (auto *Current = dyn_cast<MemoryUseOrDef>(&Acc))
+ if (!Current->getMemoryInst()->comesBefore(NewS)) {
+ FirstNonDom = Current;
+ break;
+ }
+ }
+ }
- for (unsigned I = 0; I < Exp.VarArgs.size(); I++) {
- // For InsertValue and ExtractValue, some varargs are index numbers
- // instead of value numbers. Those index numbers should not be
- // translated.
- if ((I > 1 && Exp.Opcode == Instruction::InsertValue) ||
- (I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
- (I > 1 && Exp.Opcode == Instruction::ShuffleVector))
- continue;
- Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN);
- }
+ auto *NewDef =
+ FirstNonDom
+ ? MSSAU->createMemoryAccessBefore(
+ NewS, nullptr, const_cast<MemoryUseOrDef *>(FirstNonDom))
+ : MSSAU->createMemoryAccessInBB(NewS, nullptr,
+ NewS->getParent(),
+ MemorySSA::BeforeTerminator);
- if (Exp.Commutative) {
- assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!");
- if (Exp.VarArgs[0] > Exp.VarArgs[1]) {
- std::swap(Exp.VarArgs[0], Exp.VarArgs[1]);
- uint32_t Opcode = Exp.Opcode >> 8;
- if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
- Exp.Opcode = (Opcode << 8) |
- CmpInst::getSwappedPredicate(
- static_cast<CmpInst::Predicate>(Exp.Opcode & 255));
+ MSSAU->insertDef(cast<MemoryDef>(NewDef), /*RenameUses=*/false);
+ }
}
+ if (isAssumeWithEmptyBundle(*IntrinsicI)) {
+ salvageAndRemoveInstruction(IntrinsicI);
+ return true;
+ }
+ return false;
}
- if (uint32_t NewNum = ExpressionNumbering[Exp]) {
- if (Exp.Opcode == Instruction::Call && NewNum != Num)
- return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num;
- return NewNum;
- }
- return Num;
-}
-
-/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
-/// again.
-void GVNPass::ValueTable::eraseTranslateCacheEntry(
- uint32_t Num, const BasicBlock &CurrBlock) {
- for (const BasicBlock *Pred : predecessors(&CurrBlock))
- PhiTranslateTable.erase({Num, Pred});
-}
-
-// In order to find a leader for a given value number at a
-// specific basic block, we first obtain the list of all Values for that number,
-// and then scan the list to find one whose block dominates the block in
-// question. This is fast because dominator tree queries consist of only
-// a few comparisons of DFS numbers.
-Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
- auto Leaders = LeaderTable.getLeaders(Num);
- if (Leaders.empty())
- return nullptr;
-
- Value *Val = nullptr;
- for (const auto &Entry : Leaders) {
- if (DT->dominates(Entry.BB, BB)) {
- Val = Entry.Val;
- if (isa<Constant>(Val))
- return Val;
- }
+ if (isa<Constant>(V)) {
+ // If it's not false, and constant, it must evaluate to true. This means our
+ // assume is assume(true), and thus, pointless, and we don't want to do
+ // anything more here.
+ return false;
}
- return Val;
+ Constant *True = ConstantInt::getTrue(V->getContext());
+ return propagateEquality(V, True, IntrinsicI);
}
/// There is an edge from 'Src' to 'Dst'. Return
@@ -3074,15 +3074,6 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E,
return Pred != nullptr;
}
-void GVNPass::assignBlockRPONumber(Function &F) {
- BlockRPONumber.clear();
- uint32_t NextBlockNumber = 1;
- ReversePostOrderTraversal<Function *> RPOT(&F);
- for (BasicBlock *BB : RPOT)
- BlockRPONumber[BB] = NextBlockNumber++;
- InvalidBlockRPONumbers = false;
-}
-
/// The given values are known to be equal in every use
/// dominated by 'Root'. Exploit this, for example by replacing 'LHS' with
/// 'RHS' everywhere in the scope. Returns whether a change was made.
@@ -3290,6 +3281,11 @@ bool GVNPass::propagateEquality(
return Changed;
}
+static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
+ patchReplacementInstruction(I, Repl);
+ I->replaceAllUsesWith(Repl);
+}
+
/// When calculating availability, handle an instruction
/// by inserting it into the appropriate sets.
bool GVNPass::processInstruction(Instruction *I) {
@@ -3433,94 +3429,18 @@ bool GVNPass::processInstruction(Instruction *I) {
return false;
}
- if (Repl == I) {
- // If I was the result of a shortcut PRE, it might already be in the table
- // and the best replacement for itself. Nothing to do.
- return false;
- }
-
- // Remove it!
- patchAndReplaceAllUsesWith(I, Repl);
- if (MD && Repl->getType()->isPtrOrPtrVectorTy())
- MD->invalidateCachedPointerInfo(Repl);
- salvageAndRemoveInstruction(I);
- return true;
-}
-
-/// runOnFunction - This is the main transformation entry point for a function.
-bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
- const TargetLibraryInfo &RunTLI, AAResults &RunAA,
- MemoryDependenceResults *RunMD, LoopInfo &LI,
- OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
- AC = &RunAC;
- DT = &RunDT;
- VN.setDomTree(DT);
- TLI = &RunTLI;
- AA = &RunAA;
- VN.setAliasAnalysis(&RunAA);
- MD = RunMD;
- ImplicitControlFlowTracking ImplicitCFT;
- ICF = &ImplicitCFT;
- this->LI = &LI;
- VN.setMemDep(MD);
- // Propagate the MSSA-enabled flag so the value-numbering paths in
- // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether
- // IsMSSAEnabled is turned on.
- VN.setMemorySSA(MSSA, isMemorySSAEnabled());
- ORE = RunORE;
- InvalidBlockRPONumbers = true;
- MemorySSAUpdater Updater(MSSA);
- MSSAU = MSSA ? &Updater : nullptr;
-
- bool Changed = false;
- bool ShouldContinue = true;
-
- DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
- // Merge unconditional branches, allowing PRE to catch more
- // optimization opportunities.
- for (BasicBlock &BB : make_early_inc_range(F)) {
- bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
- if (RemovedBlock)
- ++NumGVNBlocks;
-
- Changed |= RemovedBlock;
- }
- DTU.flush();
-
- unsigned Iteration = 0;
- while (ShouldContinue) {
- LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n");
- (void) Iteration;
- ShouldContinue = iterateOnFunction(F);
- Changed |= ShouldContinue;
- ++Iteration;
- }
-
- if (isScalarPREEnabled()) {
- // Fabricate val-num for dead-code in order to suppress assertion in
- // performPRE().
- assignValNumForDeadCode();
- bool PREChanged = true;
- while (PREChanged) {
- PREChanged = performPRE(F);
- Changed |= PREChanged;
- }
- }
-
- // FIXME: Should perform GVN again after PRE does something. PRE can move
- // computations into blocks where they become fully redundant. Note that
- // we can't do this until PRE's critical edge splitting updates memdep.
- // Actually, when this happens, we should just fully integrate PRE into GVN.
-
- cleanupGlobalSets();
- // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each
- // iteration.
- DeadBlocks.clear();
-
- if (MSSA && VerifyMemorySSA)
- MSSA->verifyMemorySSA();
+ if (Repl == I) {
+ // If I was the result of a shortcut PRE, it might already be in the table
+ // and the best replacement for itself. Nothing to do.
+ return false;
+ }
- return Changed;
+ // Remove it!
+ patchAndReplaceAllUsesWith(I, Repl);
+ if (MD && Repl->getType()->isPtrOrPtrVectorTy())
+ MD->invalidateCachedPointerInfo(Repl);
+ salvageAndRemoveInstruction(I);
+ return true;
}
bool GVNPass::processBlock(BasicBlock *BB) {
@@ -3543,6 +3463,23 @@ bool GVNPass::processBlock(BasicBlock *BB) {
return ChangedFunction;
}
+/// Executes one iteration of GVN.
+bool GVNPass::iterateOnFunction(Function &F) {
+ cleanupGlobalSets();
+
+ // Top-down walk of the dominator tree.
+ bool Changed = false;
+ // Needed for value numbering with phi construction to work.
+ // RPOT walks the graph in its constructor and will not be invalidated during
+ // processBlock.
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+
+ for (BasicBlock *BB : RPOT)
+ Changed |= processBlock(BB);
+
+ return Changed;
+}
+
// Instantiate an expression in a predecessor that lacked it.
bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
BasicBlock *Curr, unsigned int ValNo) {
@@ -3780,60 +3717,104 @@ bool GVNPass::performPRE(Function &F) {
return Changed;
}
-/// Split the critical edge connecting the given two blocks, and return
-/// the block inserted to the critical edge.
-BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
- // GVN does not require loop-simplify, do not try to preserve it if it is not
- // possible.
- BasicBlock *BB = SplitCriticalEdge(
- Pred, Succ,
- CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify());
- if (BB) {
- if (MD)
- MD->invalidateCachedPredecessors();
- InvalidBlockRPONumbers = true;
+/// runOnFunction - This is the main transformation entry point for a function.
+bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+ const TargetLibraryInfo &RunTLI, AAResults &RunAA,
+ MemoryDependenceResults *RunMD, LoopInfo &LI,
+ OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
+ AC = &RunAC;
+ DT = &RunDT;
+ VN.setDomTree(DT);
+ TLI = &RunTLI;
+ AA = &RunAA;
+ VN.setAliasAnalysis(&RunAA);
+ MD = RunMD;
+ ImplicitControlFlowTracking ImplicitCFT;
+ ICF = &ImplicitCFT;
+ this->LI = &LI;
+ VN.setMemDep(MD);
+ // Propagate the MSSA-enabled flag so the value-numbering paths in
+ // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether
+ // IsMSSAEnabled is turned on.
+ VN.setMemorySSA(MSSA, isMemorySSAEnabled());
+ ORE = RunORE;
+ InvalidBlockRPONumbers = true;
+ MemorySSAUpdater Updater(MSSA);
+ MSSAU = MSSA ? &Updater : nullptr;
+
+ bool Changed = false;
+ bool ShouldContinue = true;
+
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+ // Merge unconditional branches, allowing PRE to catch more
+ // optimization opportunities.
+ for (BasicBlock &BB : make_early_inc_range(F)) {
+ bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
+ if (RemovedBlock)
+ ++NumGVNBlocks;
+
+ Changed |= RemovedBlock;
}
- return BB;
-}
+ DTU.flush();
-/// Split critical edges found during the previous
-/// iteration that may enable further optimization.
-bool GVNPass::splitCriticalEdges() {
- if (ToSplit.empty())
- return false;
+ unsigned Iteration = 0;
+ while (ShouldContinue) {
+ LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n");
+ (void)Iteration;
+ ShouldContinue = iterateOnFunction(F);
+ Changed |= ShouldContinue;
+ ++Iteration;
+ }
- bool Changed = false;
- do {
- std::pair<Instruction *, unsigned> Edge = ToSplit.pop_back_val();
- Changed |= SplitCriticalEdge(Edge.first, Edge.second,
- CriticalEdgeSplittingOptions(DT, LI, MSSAU)) !=
- nullptr;
- } while (!ToSplit.empty());
- if (Changed) {
- if (MD)
- MD->invalidateCachedPredecessors();
- InvalidBlockRPONumbers = true;
+ if (isScalarPREEnabled()) {
+ // Fabricate val-num for dead-code in order to suppress assertion in
+ // performPRE().
+ assignValNumForDeadCode();
+ bool PREChanged = true;
+ while (PREChanged) {
+ PREChanged = performPRE(F);
+ Changed |= PREChanged;
+ }
}
- return Changed;
-}
-/// Executes one iteration of GVN.
-bool GVNPass::iterateOnFunction(Function &F) {
- cleanupGlobalSets();
+ // FIXME: Should perform GVN again after PRE does something. PRE can move
+ // computations into blocks where they become fully redundant. Note that
+ // we can't do this until PRE's critical edge splitting updates memdep.
+ // Actually, when this happens, we should just fully integrate PRE into GVN.
- // Top-down walk of the dominator tree.
- bool Changed = false;
- // Needed for value numbering with phi construction to work.
- // RPOT walks the graph in its constructor and will not be invalidated during
- // processBlock.
- ReversePostOrderTraversal<Function *> RPOT(&F);
+ cleanupGlobalSets();
+ // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each
+ // iteration.
+ DeadBlocks.clear();
- for (BasicBlock *BB : RPOT)
- Changed |= processBlock(BB);
+ if (MSSA && VerifyMemorySSA)
+ MSSA->verifyMemorySSA();
return Changed;
}
+// In order to find a leader for a given value number at a
+// specific basic block, we first obtain the list of all Values for that number,
+// and then scan the list to find one whose block dominates the block in
+// question. This is fast because dominator tree queries consist of only
+// a few comparisons of DFS numbers.
+Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
+ auto Leaders = LeaderTable.getLeaders(Num);
+ if (Leaders.empty())
+ return nullptr;
+
+ Value *Val = nullptr;
+ for (const auto &Entry : Leaders) {
+ if (DT->dominates(Entry.BB, BB)) {
+ Val = Entry.Val;
+ if (isa<Constant>(Val))
+ return Val;
+ }
+ }
+
+ return Val;
+}
+
void GVNPass::cleanupGlobalSets() {
VN.clear();
LeaderTable.clear();
@@ -3855,12 +3836,55 @@ void GVNPass::removeInstruction(Instruction *I) {
++NumGVNInstr;
}
+void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
+ salvageKnowledge(I, AC);
+ salvageDebugInfo(*I);
+ removeInstruction(I);
+}
+
/// Verify that the specified instruction does not occur in our
/// internal data structures.
void GVNPass::verifyRemoved(const Instruction *Inst) const {
VN.verifyRemoved(Inst);
}
+/// Split critical edges found during the previous
+/// iteration that may enable further optimization.
+bool GVNPass::splitCriticalEdges() {
+ if (ToSplit.empty())
+ return false;
+
+ bool Changed = false;
+ do {
+ std::pair<Instruction *, unsigned> Edge = ToSplit.pop_back_val();
+ Changed |= SplitCriticalEdge(Edge.first, Edge.second,
+ CriticalEdgeSplittingOptions(DT, LI, MSSAU)) !=
+ nullptr;
+ } while (!ToSplit.empty());
+ if (Changed) {
+ if (MD)
+ MD->invalidateCachedPredecessors();
+ InvalidBlockRPONumbers = true;
+ }
+ return Changed;
+}
+
+/// Split the critical edge connecting the given two blocks, and return
+/// the block inserted to the critical edge.
+BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
+ // GVN does not require loop-simplify, do not try to preserve it if it is not
+ // possible.
+ BasicBlock *BB = SplitCriticalEdge(
+ Pred, Succ,
+ CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify());
+ if (BB) {
+ if (MD)
+ MD->invalidateCachedPredecessors();
+ InvalidBlockRPONumbers = true;
+ }
+ return BB;
+}
+
/// BB is declared dead, which implied other blocks become dead as well. This
/// function is to add all these blocks to "DeadBlocks". For the dead blocks'
/// live successors, update their phi nodes by replacing the operands
@@ -3940,40 +3964,6 @@ void GVNPass::addDeadBlock(BasicBlock *BB) {
}
}
-// If the given branch is recognized as a foldable branch (i.e. conditional
-// branch with constant condition), it will perform following analyses and
-// transformation.
-// 1) If the dead out-coming edge is a critical-edge, split it. Let
-// R be the target of the dead out-coming edge.
-// 1) Identify the set of dead blocks implied by the branch's dead outcoming
-// edge. The result of this step will be {X| X is dominated by R}
-// 2) Identify those blocks which haves at least one dead predecessor. The
-// result of this step will be dominance-frontier(R).
-// 3) Update the PHIs in DF(R) by replacing the operands corresponding to
-// dead blocks with "UndefVal" in an hope these PHIs will optimized away.
-//
-// Return true iff *NEW* dead code are found.
-bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
- // If a branch has two identical successors, we cannot declare either dead.
- if (BI->getSuccessor(0) == BI->getSuccessor(1))
- return false;
-
- ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
- if (!Cond)
- return false;
-
- BasicBlock *DeadRoot =
- Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0);
- if (DeadBlocks.count(DeadRoot))
- return false;
-
- if (!DeadRoot->getSinglePredecessor())
- DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot);
-
- addDeadBlock(DeadRoot);
- return true;
-}
-
// performPRE() will trigger assert if it comes across an instruction without
// associated val-num. As it normally has far more live instructions than dead
// instructions, it makes more sense just to "fabricate" a val-number for the
@@ -3987,6 +3977,15 @@ void GVNPass::assignValNumForDeadCode() {
}
}
+void GVNPass::assignBlockRPONumber(Function &F) {
+ BlockRPONumber.clear();
+ uint32_t NextBlockNumber = 1;
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+ for (BasicBlock *BB : RPOT)
+ BlockRPONumber[BB] = NextBlockNumber++;
+ InvalidBlockRPONumbers = false;
+}
+
class llvm::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
>From ee5c81489e276d8c6f5c8d8b2ae281b8939c94c0 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 15 Jul 2026 15:41:09 +0100
Subject: [PATCH 5/7] [GVN] Assign unique VNs to calls with operand bundles
Call instructions with operand bundles may be assigned the same value number,
even if operand bundles differ. The GVN may eliminate one of the calls in favour
of another and drop one of the operand bundles.
Work around this by assigning unique value numbers to calls with operand
bundles.
---
llvm/lib/Transforms/Scalar/GVN.cpp | 6 ++++++
.../Transforms/GVN/operand-bundle-unique-vn.ll | 18 ++++++++++++++++++
2 files changed, 24 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index f16d1fe9ca893..10b9dd77a3acb 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -539,6 +539,12 @@ uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) {
return NextValueNumber++;
}
+ // Conservatively assign unique value numbers to calls with operand bundles.
+ if (C->hasOperandBundles()) {
+ ValueNumbering[C] = NextValueNumber;
+ return NextValueNumber++;
+ }
+
if (AA->doesNotAccessMemory(C)) {
Expression Exp = createExpr(C);
uint32_t E = assignExpNewValueNum(Exp).first;
diff --git a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
new file mode 100644
index 0000000000000..d027a875310f9
--- /dev/null
+++ b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -p gvn < %s | FileCheck %s
+
+; Check GVN does not eliminate the second call bacause of operand bundle presence.
+
+define i32 @f(ptr %p) {
+; CHECK-LABEL: define i32 @f(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1:[0-9]+]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: ret i32 [[V]]
+;
+ %u = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ %v = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ ret i32 %v
+}
+
+declare void @g(i1) nounwind willreturn
>From 12654f87243c8ec685d68442622bfd3e87e51873 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Fri, 10 Jul 2026 13:54:09 +0100
Subject: [PATCH 6/7] [GVN] Simple GVN-based hoisting of scalars: precommit
tests
---
.../Transforms/GVN/simple-gvn-hoist-limits.ll | 155 +++++++
.../GVN/simple-gvn-hoist-scalars.ll | 382 ++++++++++++++++++
2 files changed, 537 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
create mode 100644 llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
new file mode 100644
index 0000000000000..0c0789e6e4144
--- /dev/null
+++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S --passes=gvn --gvn-max-num-insns=3 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH3
+; RUN: opt -S --passes=gvn --gvn-max-num-insns=4 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH4
+; RUN: opt -S --passes=gvn --gvn-max-num-insns=5 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH5
+
+; Check effect of limiting the how deep in a block we look for
+; instructions to hoist; At max depth 3 we shouldn't hoist
+; anything to the `entry` block, at max depth 4 we should hoist
+; the first `add` and at max depth 5 the second `add`.
+
+; In any case we shouldn't hoist the `and` and the `icmp`, in order
+; to not separate them from the `br`
+
+define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
+; MAX-DEPTH3-LABEL: @f(
+; MAX-DEPTH3-NEXT: entry:
+; MAX-DEPTH3-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH3: if.then:
+; MAX-DEPTH3-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH3-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH3-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH3-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH3-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH3-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH3: if.then1:
+; MAX-DEPTH3-NEXT: [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH3-NEXT: br label [[EXIT:%.*]]
+; MAX-DEPTH3: if.else1:
+; MAX-DEPTH3-NEXT: [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH3-NEXT: br label [[EXIT]]
+; MAX-DEPTH3: if.else:
+; MAX-DEPTH3-NEXT: [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH3-NEXT: [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH3-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH3-NEXT: [[D1:%.*]] = add i32 [[D]], 1
+; MAX-DEPTH3-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]]
+; MAX-DEPTH3-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH3: if.then2:
+; MAX-DEPTH3-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH3-NEXT: br label [[EXIT]]
+; MAX-DEPTH3: if.else2:
+; MAX-DEPTH3-NEXT: [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH3-NEXT: br label [[EXIT]]
+; MAX-DEPTH3: exit:
+; MAX-DEPTH3-NEXT: [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH3-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH3-NEXT: [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH3-NEXT: ret i32 [[R]]
+;
+; MAX-DEPTH4-LABEL: @f(
+; MAX-DEPTH4-NEXT: entry:
+; MAX-DEPTH4-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH4: if.then:
+; MAX-DEPTH4-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH4-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH4-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH4-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH4-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH4: if.then1:
+; MAX-DEPTH4-NEXT: [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH4-NEXT: br label [[EXIT:%.*]]
+; MAX-DEPTH4: if.else1:
+; MAX-DEPTH4-NEXT: [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH4-NEXT: br label [[EXIT]]
+; MAX-DEPTH4: if.else:
+; MAX-DEPTH4-NEXT: [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH4-NEXT: [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH4-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH4-NEXT: [[D1:%.*]] = add i32 [[D]], 1
+; MAX-DEPTH4-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]]
+; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH4: if.then2:
+; MAX-DEPTH4-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH4-NEXT: br label [[EXIT]]
+; MAX-DEPTH4: if.else2:
+; MAX-DEPTH4-NEXT: [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH4-NEXT: br label [[EXIT]]
+; MAX-DEPTH4: exit:
+; MAX-DEPTH4-NEXT: [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH4-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH4-NEXT: [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH4-NEXT: ret i32 [[R]]
+;
+; MAX-DEPTH5-LABEL: @f(
+; MAX-DEPTH5-NEXT: entry:
+; MAX-DEPTH5-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH5: if.then:
+; MAX-DEPTH5-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH5-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH5-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH5-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH5-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH5: if.then1:
+; MAX-DEPTH5-NEXT: [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH5-NEXT: br label [[EXIT:%.*]]
+; MAX-DEPTH5: if.else1:
+; MAX-DEPTH5-NEXT: [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH5-NEXT: br label [[EXIT]]
+; MAX-DEPTH5: if.else:
+; MAX-DEPTH5-NEXT: [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH5-NEXT: [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH5-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH5-NEXT: [[D1:%.*]] = add i32 [[D]], 1
+; MAX-DEPTH5-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]]
+; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH5: if.then2:
+; MAX-DEPTH5-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH5-NEXT: br label [[EXIT]]
+; MAX-DEPTH5: if.else2:
+; MAX-DEPTH5-NEXT: [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH5-NEXT: br label [[EXIT]]
+; MAX-DEPTH5: exit:
+; MAX-DEPTH5-NEXT: [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH5-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH5-NEXT: [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH5-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %c, label %if.then, label %if.else
+
+if.then:
+ %r0 = add i32 %b, 1
+ %and0 = and i32 %a, 1
+ %tobool.and0 = icmp eq i32 %and0, 0
+ %d0 = add i32 %d, 1
+ %s0 = add i32 %d0, %b
+ br i1 %tobool.and0, label %if.then1, label %if.else1
+if.then1:
+ %or0 = or i32 %r0, 1
+ br label %exit
+if.else1:
+ %or1 = or i32 %r0, 2
+ br label %exit
+
+if.else:
+ %r1 = add i32 %b, 2
+ %and1 = and i32 %a, 1
+ %tobool.and1 = icmp eq i32 %and1, 0
+ %d1 = add i32 %d, 1
+ %s1 = add i32 %d1, %b
+ br i1 %tobool.and1, label %if.then2, label %if.else2
+if.then2:
+ %or2 = or i32 %r1, 4
+ br label %exit
+if.else2:
+ %or3 = or i32 %r1, 8
+ br label %exit
+exit:
+ %or = phi i32 [%or0, %if.then1], [%or1, %if.else1], [%or2, %if.then2], [%or3, %if.else2]
+ %s = phi i32 [%s0, %if.then1], [%s0, %if.else1], [%s1, %if.then2], [%s1, %if.else2]
+ %r = add i32 %or, %s
+ ret i32 %r
+}
diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
new file mode 100644
index 0000000000000..e774f64d4a4e5
--- /dev/null
+++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
@@ -0,0 +1,382 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt --passes=gvn -S %s | FileCheck %s
+; RUN: opt --passes="gvn<memoryssa>" -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux"
+
+define dso_local i32 @everything_hoisted(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @everything_hoisted(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @barrier(i32 [[A:%.*]])
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP1]], [[C:%.*]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @barrier(i32 [[A]])
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[B]]
+; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP4]], [[C]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = call i32 @barrier(i32 %a)
+ %1 = add i32 %0, %b
+ %2 = sdiv i32 %1, %c
+ br label %if.end
+
+if.else:
+ %3 = call i32 @barrier(i32 %a)
+ %4 = add i32 %3, %b
+ %5 = sdiv i32 %4, %c
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%2, %if.then], [%5, %if.else]
+ ret i32 %r
+}
+
+; speculation barrier on the short(collect) side
+define dso_local i32 @spec_barrier_short_side(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @spec_barrier_short_side(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A:%.*]])
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = call i32 @barrier(i32 %a)
+ %1 = add nsw i32 %a, %b
+ %2 = sdiv i32 %c, %1
+ br label %if.end
+
+if.else:
+ %3 = add nsw i32 %a, %b
+ %4 = sdiv i32 %c, %3
+ %5 = add i32 %4, 1
+ %6 = add i32 %5, 2
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%2, %if.then], [%6, %if.else]
+ ret i32 %r
+}
+
+; speculation barrier on the long(match) side
+define dso_local i32 @spec_barrier_long_side(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @spec_barrier_long_side(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A]])
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = add nsw i32 %a, %b
+ %1 = sdiv i32 %c, %0
+ br label %if.end
+
+if.else:
+ %2 = call i32 @barrier(i32 %a)
+ %3 = add nsw i32 %a, %b
+ %4 = sdiv i32 %c, %3
+ %5 = add i32 %4, 1
+ %6 = add i32 %5, 2
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%1, %if.then], [%6, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @no_reorder_across_volatile(i1 %cc, i32 %a, i32 %b, i32 %c, ptr %p) {
+; CHECK-LABEL: @no_reorder_across_volatile(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: store volatile i32 0, ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[C]], [[TMP3]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = add nsw i32 %a, %b
+ %1 = sdiv i32 %c, %0
+ br label %if.end
+
+if.else:
+ store volatile i32 0, ptr %p
+ %2 = add nsw i32 %a, %b
+ %3 = sdiv i32 %c, %2
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%1, %if.then], [%3, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @no_barrier_call(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @no_barrier_call(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: call void @will_return()
+; CHECK-NEXT: [[TMP0:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[C:%.*]], [[TMP0]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: call void @will_return()
+; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[C]], [[TMP2]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP3]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ call void @will_return()
+ %0 = sdiv i32 %a, %b
+ %1 = add nsw i32 %c, %0
+ br label %if.end
+
+if.else:
+ call void @will_return()
+ %2 = sdiv i32 %a, %b
+ %3 = add nsw i32 %c, %2
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%1, %if.then], [%3, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @no_reorder_atomic(i1 %cc, i32 %a, i32 %b, i32 %c, ptr %p) {
+; CHECK-LABEL: @no_reorder_atomic(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = load atomic volatile i32, ptr [[P:%.*]] acquire, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[C:%.*]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul nsw i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP4:%.*]] = load atomic volatile i32, ptr [[P]] acquire, align 4
+; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[C]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul nsw i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP3]], [[IF_THEN]] ], [ [[TMP7]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = load atomic volatile i32, ptr %p acquire, align 4
+ %1 = sdiv i32 %a, %b
+ %2 = add nsw i32 %c, %1
+ %3 = mul nsw i32 %0, %2
+ br label %if.end
+
+if.else:
+ %4 = load atomic volatile i32, ptr %p acquire, align 4
+ %5 = sdiv i32 %a, %b
+ %6 = add nsw i32 %c, %5
+ %7 = mul nsw i32 %4, %6
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%3, %if.then], [%7, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @multiple_use(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @multiple_use(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i32 [[TMP0]], [[C:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i32 [[TMP3]], [[C]]
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = add nsw i32 %a, %b
+ %1 = mul nsw i32 %0, %c
+ %2 = add nsw i32 %0, %1
+ br label %if.end
+
+if.else:
+ %3 = add nsw i32 %a, %b
+ %4 = mul nsw i32 %3, %c
+ %5 = add nsw i32 %3, %4
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%2, %if.then], [%5, %if.else]
+ ret i32 %r
+}
+
+; Different operand order in commutative operations
+define dso_local i32 @commutative_ops(i1 %cc, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @commutative_ops(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP0]], [[C:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[C]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = add nsw i32 %a, %b
+ %1 = add nsw i32 %0, %c
+ %2 = sdiv i32 %0, %1
+ br label %if.end
+
+if.else:
+ %3 = add nsw i32 %a, %b
+ %4 = add nsw i32 %c, %3
+ %5 = sdiv i32 %3, %4
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%2, %if.then], [%5, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @no_hoist_mem(i1 %cc, ptr %p) {
+; CHECK-LABEL: @no_hoist_mem(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP0]], 1
+; CHECK-NEXT: br label [[IF_END:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP3]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = load i32, ptr %p
+ %1 = add nsw i32 %0, 1
+ br label %if.end
+
+if.else:
+ %2 = load i32, ptr %p
+ %3 = add nsw i32 %2, 1
+ br label %if.end
+
+if.end:
+ %r = phi i32 [%1, %if.then], [%3, %if.else]
+ ret i32 %r
+}
+
+define dso_local i32 @no_hoist_musttail(i1 %cc, i32 %x, ptr %p) {
+; CHECK-LABEL: @no_hoist_musttail(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[W:%.*]] = add i32 [[V]], [[X:%.*]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = musttail call i32 @will_return(i1 true, i32 [[W]], ptr [[P]])
+; CHECK-NEXT: ret i32 [[TMP0]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP1:%.*]] = musttail call i32 @will_return(i1 false, i32 [[W]], ptr [[P]])
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %v = load i32, ptr %p
+ %w = add i32 %v, %x
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = musttail call i32 @will_return(i1 %cc, i32 %w, ptr %p)
+ ret i32 %0
+
+if.else:
+ %1 = musttail call i32 @will_return(i1 %cc, i32 %w, ptr %p)
+ ret i32 %1
+}
+
+declare i32 @barrier(i32) memory(none)
+declare void @will_return() nounwind willreturn
>From 9d7161d7eb6d82fa4564428d0a0bbccd356d2a2d Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 8 Jul 2026 12:48:20 +0000
Subject: [PATCH 7/7] [GVN] Simple GVN-based hoisring of scalars
RFC/discussion: https://lists.llvm.org/pipermail/llvm-dev/2021-September/152665.html
This patch is a update of https://reviews.llvm.org/D110817
This patch implements simple hoisting of instructions from two
single-predecessor blocks to their common predecessor, as a subroutine
in the GVN pass.
The patch pairs two instructions (A and B) with the same value number,
moves A to the predecessor block, replaces all uses of B with A, and
deletes B.
Outline of the algorithm follows:
Scan the then-block to collect hoist candidates ("then-" and "else-"
prefixes are purely naming and have no connection to the condition in
the predecessor block)
Scan the else-block for hoist candidates, that match some already
selected instruction from the then-block.
During both scans, instructions which are not guaranteed to transfer
control to the following instruction act as "hoist barriers" - after we
encounter such an instruction, we select for potential hoisting/merge
only instructions, which are safe to execute speculatively. Also
instructions which read/write memory are not considered for hoisting,
subject for a follow-up patch. The hoist barriers can itself be hoisted,
opening opportunities for other instructions. For each hoist candidate
pair, the immediately preceding hoist barriers from then- and
else-blocks are recorded as prerequisites for hoisting the pair.
Next we try hoist to hoist each candidate pair. We begin by trying to
hoist dependencies of the then-instruction, which would be its
immediately preceding hoist barrier and its operands. Each of these
dependencies must already be in a dominating block or is itself paired
with an instruction from the else-block. If we cannot hoist an
dependency for whatever reason, the we stop trying to hoist the pair.
Now that all the operands of the then-instruction are in a dominating
block, we check the barriers/operands of the else-instruction. They all
must already be in a dominating block, either initially or as a result
of hoisting barriers/operands of the then-instruction. If any dependency
is still in the else-block, we stop trying to hoist the pair.
As a last step, we move the then-instruction to the predecessor block
and delete the else-instruction.
---
.../CodeGen/attr-counted-by-with-sanitizers.c | 54 ++--
llvm/include/llvm/Transforms/Scalar/GVN.h | 25 ++
llvm/lib/Transforms/Scalar/GVN.cpp | 232 +++++++++++++++++-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 147 +++++------
.../NVPTX/gvn-scalar-pre-reg-pressure.ll | 4 +-
.../Transforms/GVN/2012-05-22-PreCrash.ll | 2 +-
.../GVN/PRE/load-pre-across-backedge.ll | 4 +-
llvm/test/Transforms/GVN/PRE/local-pre.ll | 4 +-
llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll | 8 +-
llvm/test/Transforms/GVN/PRE/phi-translate.ll | 4 +-
llvm/test/Transforms/GVN/PRE/pre-basic-add.ll | 6 +-
.../GVN/PRE/pre-load-through-select.ll | 4 +-
.../Transforms/GVN/PRE/pre-no-cost-phi.ll | 4 +-
.../test/Transforms/GVN/PRE/pre-poison-add.ll | 4 +-
llvm/test/Transforms/GVN/freeze.ll | 2 +-
llvm/test/Transforms/GVN/gc_relocate.ll | 2 +-
.../Transforms/GVN/simple-gvn-hoist-limits.ll | 15 +-
.../GVN/simple-gvn-hoist-scalars.ll | 107 +++++---
18 files changed, 449 insertions(+), 179 deletions(-)
diff --git a/clang/test/CodeGen/attr-counted-by-with-sanitizers.c b/clang/test/CodeGen/attr-counted-by-with-sanitizers.c
index e840db632957e..81be6bdad9936 100644
--- a/clang/test/CodeGen/attr-counted-by-with-sanitizers.c
+++ b/clang/test/CodeGen/attr-counted-by-with-sanitizers.c
@@ -234,16 +234,16 @@ size_t test_return_bdos_cast_of_whole_struct(struct annotated *p) {
// SANITIZE-WITH-ATTR: [[CONT1]]:
// SANITIZE-WITH-ATTR-NEXT: [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[DOTCOUNTED_BY_LOAD]], 2
// SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = icmp ult i32 [[INDEX]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
-// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext i32 [[INDEX]] to i64
+// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = zext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP1]], label %[[CONT12:.*]], label %[[HANDLER_OUT_OF_BOUNDS8:.*]], !prof [[PROF7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS8]]:
-// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[TMP2]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT12]]:
// SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 244
-// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = and i32 [[RESULT]], 252
-// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP2]], ptr [[ARRAYIDX10]], align 4, !tbaa [[INT_TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = and i32 [[RESULT]], 252
+// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP2]]
+// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP3]], ptr [[ARRAYIDX10]], align 4, !tbaa [[INT_TBAA8]]
// SANITIZE-WITH-ATTR-NEXT: [[DOTNOT79:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 3
// SANITIZE-WITH-ATTR-NEXT: br i1 [[DOTNOT79]], label %[[HANDLER_OUT_OF_BOUNDS18:.*]], label %[[CONT19:.*]], !prof [[PROF9:![0-9]+]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS18]]:
@@ -251,37 +251,37 @@ size_t test_return_bdos_cast_of_whole_struct(struct annotated *p) {
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT19]]:
// SANITIZE-WITH-ATTR-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[INDEX]], 1
-// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = icmp samesign ult i32 [[ADD]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
-// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM31:%.*]] = zext nneg i32 [[ADD]] to i64
-// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP3]], label %[[CONT38:.*]], label %[[HANDLER_OUT_OF_BOUNDS34:.*]], !prof [[PROF7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = icmp samesign ult i32 [[ADD]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = zext nneg i32 [[ADD]] to i64
+// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP4]], label %[[CONT38:.*]], label %[[HANDLER_OUT_OF_BOUNDS34:.*]], !prof [[PROF7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS34]]:
-// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT38]]:
// SANITIZE-WITH-ATTR-NEXT: [[RESULT25:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 240
-// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = and i32 [[RESULT25]], 252
-// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX36:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM31]]
-// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP4]], ptr [[ARRAYIDX36]], align 4, !tbaa [[INT_TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = and i32 [[RESULT25]], 252
+// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX36:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP5]]
+// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP6]], ptr [[ARRAYIDX36]], align 4, !tbaa [[INT_TBAA8]]
// SANITIZE-WITH-ATTR-NEXT: [[DOTNOT:%.*]] = icmp ugt i32 [[FAM_IDX]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: br i1 [[DOTNOT]], label %[[HANDLER_OUT_OF_BOUNDS45:.*]], label %[[CONT46:.*]], !prof [[PROF9]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS45]]:
-// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = zext i32 [[FAM_IDX]] to i64, !nosanitize [[META6]]
-// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP7:%.*]] = zext i32 [[FAM_IDX]] to i64, !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[TMP7]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT46]]:
// SANITIZE-WITH-ATTR-NEXT: [[ADD59:%.*]] = add nuw nsw i32 [[INDEX]], 2
-// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = icmp samesign ult i32 [[ADD59]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
-// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM60:%.*]] = zext nneg i32 [[ADD59]] to i64
-// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP6]], label %[[CONT67:.*]], label %[[HANDLER_OUT_OF_BOUNDS63:.*]], !prof [[PROF7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP8:%.*]] = icmp samesign ult i32 [[ADD59]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP9:%.*]] = zext nneg i32 [[ADD59]] to i64
+// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP8]], label %[[CONT67:.*]], label %[[HANDLER_OUT_OF_BOUNDS63:.*]], !prof [[PROF7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS63]]:
-// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR7]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[TMP9]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT67]]:
-// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM60]]
+// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP9]]
// SANITIZE-WITH-ATTR-NEXT: [[DOTTR:%.*]] = sub nsw i32 [[DOTCOUNTED_BY_LOAD]], [[FAM_IDX]]
-// SANITIZE-WITH-ATTR-NEXT: [[TMP7:%.*]] = shl i32 [[DOTTR]], 2
-// SANITIZE-WITH-ATTR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 252
-// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP8]], ptr [[ARRAYIDX65]], align 4, !tbaa [[INT_TBAA8]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP10:%.*]] = shl i32 [[DOTTR]], 2
+// SANITIZE-WITH-ATTR-NEXT: [[TMP11:%.*]] = and i32 [[TMP10]], 252
+// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP11]], ptr [[ARRAYIDX65]], align 4, !tbaa [[INT_TBAA8]]
// SANITIZE-WITH-ATTR-NEXT: ret void
//
// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test_assign_size_of_pointer_into_fam(
@@ -483,15 +483,14 @@ size_t test_return_bdos_of_fam_in_anon_struct(struct anon_struct *p) {
// SANITIZE-WITH-ATTR-NEXT: [[DOTCOUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
// SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = zext i8 [[DOTCOUNTED_BY_LOAD]] to i32, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = icmp ult i32 [[INDEX]], [[TMP1]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP2]], label %[[CONT7:.*]], label %[[HANDLER_OUT_OF_BOUNDS:.*]], !prof [[PROF7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS]]:
-// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP3]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT7]]:
// SANITIZE-WITH-ATTR-NEXT: [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
-// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext nneg i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[TMP3]]
// SANITIZE-WITH-ATTR-NEXT: store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA10:![0-9]+]]
// SANITIZE-WITH-ATTR-NEXT: ret void
//
@@ -529,15 +528,14 @@ size_t test_return_bdos_of_anon_struct(struct union_of_fams *p) {
// SANITIZE-WITH-ATTR-NEXT: [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
// SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i32, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = icmp ult i32 [[INDEX]], [[TMP1]], !nosanitize [[META6]]
+// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64
// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP2]], label %[[CONT14:.*]], label %[[HANDLER_OUT_OF_BOUNDS:.*]], !prof [[PROF7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS]]:
-// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[TMP3]]) #[[ATTR7]], !nosanitize [[META6]]
// SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]]
// SANITIZE-WITH-ATTR: [[CONT14]]:
// SANITIZE-WITH-ATTR-NEXT: [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
-// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext nneg i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[TMP3]]
// SANITIZE-WITH-ATTR-NEXT: store i8 [[COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA10]]
// SANITIZE-WITH-ATTR-NEXT: ret void
//
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 46c54363298a2..dd0fbcee1e253 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -322,6 +322,24 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
// List of critical edges to be split between iterations.
SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
+ // A pair of instructions with the same value number to be hoisted and merged,
+ // together with their respective hoist barriers. A pair of insructions can be
+ // hoisted iff both their barriers (if not null) are hoisted as well. The
+ // `WeakVH` is used to track when the barrier instruction itself is hoisted.
+ struct HoistPair {
+ Instruction *ThenI = nullptr;
+ Instruction *ThenB = nullptr;
+ Instruction *ElseI = nullptr;
+ WeakVH ElseB = nullptr;
+ };
+
+ /// A mapping from value numbers to a pair of instructions. This map
+ /// stores pairs of instructions with the same value number, from two blocks
+ /// having a single common predecessor, for the duration of a single top level
+ /// iteration in `performHoist`.
+ using HoistMap = DenseMap<uint32_t, HoistPair>;
+ HoistMap HoistPairs;
+
public:
GVNPass(GVNOptions Options = {}) : Options(Options) {}
@@ -507,6 +525,13 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
bool performScalarPRE(Instruction *I);
bool performPRE(Function &F);
+ void collectHoistCandidates(BasicBlock *ThenBB);
+ void matchHoistCandidates(BasicBlock *ElseBB);
+ void replaceInstruction(Instruction *I, Instruction *Repl);
+ std::pair<bool, bool> hoistPair(BasicBlock *DestBB, BasicBlock *ThenBB,
+ BasicBlock *ElseBB, Instruction *ThenI);
+ bool performHoist(Function &F);
+
/// Main entry point for the GVN pass. Also used by the GVNLegacyPass.
bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
const TargetLibraryInfo &RunTLI, AAResults &RunAA,
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 10b9dd77a3acb..b8bec704d9411 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -118,7 +118,8 @@ GVNEnableSplitBackedgeInLoadPRE("enable-split-backedge-in-load-pre",
static cl::opt<bool> GVNEnableMemDep("enable-gvn-memdep", cl::init(true));
static cl::opt<bool> GVNEnableMemorySSA("enable-gvn-memoryssa",
cl::init(false));
-
+static cl::opt<bool> GVNEnableSimpleGVNHoist("enable-simple-gvn-hoist",
+ cl::init(true));
static cl::opt<unsigned> ScanUsersLimit(
"gvn-scan-users-limit", cl::Hidden, cl::init(100),
cl::desc("The number of memory accesses to scan in a block in reaching "
@@ -3723,6 +3724,230 @@ bool GVNPass::performPRE(Function &F) {
return Changed;
}
+// Won't reorder above these instructions.
+static bool isHoistBarrier(const Instruction &I) {
+ return I.mayWriteToMemory() || I.mayHaveSideEffects() || !isGuaranteedToTransferExecutionToSuccessor(&I);
+}
+
+static bool isHoistCandidate(const Instruction &I) {
+ if (I.mayReadOrWriteMemory())
+ return false;
+ if (!isa<CallBase>(I))
+ return true;
+ const auto &CB = cast<CallBase>(I);
+ if (CB.isMustTailCall() || CB.cannotMerge())
+ return false;
+ return true;
+}
+
+void GVNPass::collectHoistCandidates(BasicBlock *BB) {
+ uint32_t Depth = 0;
+ Instruction *Barrier = nullptr;
+ for (Instruction &I : *BB) {
+ if (++Depth > MaxNumInsnsPerBlock)
+ break;
+ if (I.isTerminator())
+ break;
+ if (isa<PHINode>(I))
+ continue;
+ if (isHoistCandidate(I)) {
+ HoistPair &HP = HoistPairs[VN.lookupOrAdd(&I)];
+ HP.ThenI = &I;
+ HP.ThenB = isSafeToSpeculativelyExecute(&I) ? nullptr : Barrier;
+ }
+ Barrier = isHoistBarrier(I) ? &I : Barrier;
+ }
+}
+
+void GVNPass::matchHoistCandidates(BasicBlock *BB) {
+ uint32_t Depth = 0;
+ Instruction *Barrier = nullptr;
+ for (Instruction &I : *BB) {
+ if (++Depth > MaxNumInsnsPerBlock)
+ break;
+ if (I.isTerminator())
+ break;
+ if (isa<PHINode>(I))
+ continue;
+ if (isHoistCandidate(I)) {
+ uint32_t N = VN.lookupOrAdd(&I);
+ if (auto It = HoistPairs.find(N);
+ It != HoistPairs.end() && It->second.ElseI == nullptr) {
+ It->second.ElseI = &I;
+ It->second.ElseB = isSafeToSpeculativelyExecute(&I) ? nullptr : Barrier;
+ }
+ }
+ Barrier = isHoistBarrier(I) ? &I : Barrier;
+ }
+}
+
+void GVNPass::replaceInstruction(Instruction *I, Instruction *Repl) {
+ LLVM_DEBUG(dbgs() << "Simple GVNHoist: replacing" << *I << " by" << *Repl
+ << '\n';);
+ patchReplacementInstruction(I, Repl);
+ ICF->removeUsersOf(I);
+ I->replaceAllUsesWith(Repl);
+ salvageKnowledge(I, AC);
+ salvageDebugInfo(*I);
+ if (MD)
+ MD->removeInstruction(I);
+ if (MSSAU)
+ MSSAU->removeMemoryAccess(I);
+ VN.erase(I);
+ ICF->removeInstruction(I);
+ LLVM_DEBUG(verifyRemoved(I));
+ I->eraseFromParent();
+ ++NumGVNInstr;
+}
+
+// Only hoist instructions from the "then" block.
+// Each hoisted instruction must be paired with an instruction from the "else"
+// block.
+std::pair<bool, bool> GVNPass::hoistPair(BasicBlock *DestBB, BasicBlock *ThenBB,
+ BasicBlock *ElseBB, Instruction *ThenI) {
+ // If the instruction is moved out of the "then" block there's nothing to do.
+ if (ThenI->getParent() != ThenBB)
+ return {false, false};
+
+ // Instruction must have already been selected for hoisting and matched with
+ // another instruction.
+ auto It = HoistPairs.find(VN.lookupOrAdd(ThenI));
+ if (It == HoistPairs.end())
+ return {false, true};
+
+ // Do not attempt to hoist a pair twice. If `ElseI` is nullptr, it means
+ // either there was no match for `ThenI` or there was already an attempt
+ // (successful or not) to hoist the pair.
+ Instruction *ElseI = It->second.ElseI;
+ if (ElseI == nullptr)
+ return {false, true};
+ It->second.ElseI = nullptr;
+
+ assert(ElseI->getParent() == ElseBB && "Instruction already removed");
+ assert(!ThenI->mayReadOrWriteMemory() && !ElseI->mayReadOrWriteMemory() &&
+ "Memory read/write instructions must not be hoisted.");
+
+ bool Change = false;
+
+ // Hoist the `Then` barrier, if any.
+ Instruction *ThenB = It->second.ThenB;
+ if (ThenB != nullptr && ThenB->getParent() == ThenBB) {
+ auto [LocalChange, StopHoisting] = hoistPair(DestBB, ThenBB, ElseBB, ThenB);
+ Change |= LocalChange;
+ if (StopHoisting)
+ return {Change, true};
+ }
+
+ // Check the `Else` barrier instruction, if any, was deleted from the `Else`
+ // block as a result of a previous hoisting.
+ if (dyn_cast_or_null<Instruction>(It->second.ElseB) != nullptr)
+ return {Change, true};
+
+ // Hoist operands. Begin by hoisting all of the operands of the "then"
+ // instruction, then check that all of the operands of the "else" instruction
+ // strictly dominate its block.
+ for (unsigned I = 0, N = ThenI->getNumOperands(); I < N; ++I) {
+ auto *Op = dyn_cast<Instruction>(ThenI->getOperand(I));
+ if (Op == nullptr)
+ continue;
+ auto [LocalChange, StopHoisting] = hoistPair(DestBB, ThenBB, ElseBB, Op);
+ Change |= LocalChange;
+ if (StopHoisting)
+ return {Change, true};
+ }
+
+ for (unsigned I = 0, N = ElseI->getNumOperands(); I < N; ++I) {
+ auto *Op = dyn_cast<Instruction>(ElseI->getOperand(I));
+ if (Op == nullptr)
+ continue;
+ if (Op->getParent() == ElseBB)
+ return {Change, true};
+ }
+
+ // Hoist one of the instructions and replace all uses of the other with it.
+ ICF->removeInstruction(ThenI);
+ ICF->insertInstructionTo(ThenI, DestBB);
+ ThenI->moveBefore(DestBB->getTerminator()->getIterator());
+ replaceInstruction(ElseI, ThenI);
+
+ return {true, false};
+}
+
+// Determine if an instruction should be used to initiate hoisting a
+// dependency chain. The aim is to avoid separating instructions, for which it's
+// (heuristically) considered better to keep them together, as it's common that
+// they can be fused in some way. An instruction, which is denied hoisting by
+// this function can still be hoisted if it appears as a dependency (e.g
+// operand) of another hoisted instruction.
+static bool shouldNotInitiateHoisting(const Instruction *I) {
+ // Don't separate GEP's from their loads/stores.
+ if (isa<GetElementPtrInst>(I))
+ return true;
+ const bool IsBinop = isa<BinaryOperator>(I);
+ for (const User *U : I->users()) {
+ // Don't separate conditions from `br` or `select`.
+ if ((isa<CondBrInst>(U) || isa<SelectInst>(U)) && U->getOperand(0) == I)
+ return true;
+ // Don't separate a value from converting that value to a boolean by
+ // comparing it to zero.
+ if (!IsBinop)
+ continue;
+ const auto *ICmp = dyn_cast<ICmpInst>(U);
+ if (ICmp == nullptr || (ICmp->getPredicate() != CmpInst::ICMP_EQ &&
+ ICmp->getPredicate() != CmpInst::ICMP_NE))
+ continue;
+ const auto *Zero = dyn_cast<ConstantInt>(ICmp->getOperand(1));
+ if (Zero != nullptr && Zero->isZero())
+ return true;
+ }
+ return false;
+}
+
+// Perform trivial hoisting of values from two blocks to their common
+// predecessor.
+bool GVNPass::performHoist(Function &F) {
+ LLVM_DEBUG(dbgs() << "Simple GVNHoist: running on function " << F.getName()
+ << '\n';);
+ bool Change = false;
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+ for (BasicBlock *BB : RPOT) {
+ // Check we have a block of the desired shape.
+ auto *BI = dyn_cast<CondBrInst>(BB->getTerminator());
+ if (!BI)
+ continue;
+
+ BasicBlock *Then = BI->getSuccessor(0);
+ BasicBlock *Else = BI->getSuccessor(1);
+
+ if (!Then->getSinglePredecessor() || !Else->getSinglePredecessor())
+ continue;
+
+ LLVM_DEBUG(dbgs() << "Simple GVNHoist: looking at block " << BB->getName()
+ << '\n');
+
+ // Collect all hoistable instructions from the smaller block, then match
+ // them by value number with the instructions from the other block.
+ if (Then->size() > Else->size())
+ std::swap(Then, Else);
+
+ HoistPairs.clear();
+ collectHoistCandidates(Then);
+ matchHoistCandidates(Else);
+
+ // Hoist matched pairs.
+ for (const auto &P : HoistPairs) {
+ const HoistPair &HP = P.second;
+ if (shouldNotInitiateHoisting(HP.ThenI))
+ continue;
+ auto [LocalChange, _] = hoistPair(BB, Then, Else, HP.ThenI);
+ Change |= LocalChange;
+ }
+ }
+
+ return Change;
+}
+
+
/// runOnFunction - This is the main transformation entry point for a function.
bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
const TargetLibraryInfo &RunTLI, AAResults &RunAA,
@@ -3783,6 +4008,11 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
}
}
+ if (GVNEnableSimpleGVNHoist) {
+ LeaderTable.clear();
+ Changed |= performHoist(F);
+ }
+
// FIXME: Should perform GVN again after PRE does something. PRE can move
// computations into blocks where they become fully redundant. Note that
// we can't do this until PRE's critical edge splitting updates memdep.
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index d90c9771c3e4a..531b4ca00f2ec 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -8,144 +8,127 @@ define void @issue63986(i64 %0, i64 %idxprom, ptr inreg %ptr) {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_lshlrev_b64 v[4:5], 6, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v6, s17
-; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s16, v4
-; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v6, v5, vcc
+; CHECK-NEXT: v_mov_b32_e32 v2, s17
+; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s16, v4
+; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v2, v5, vcc
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: .LBB0_1: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v7, s5
-; CHECK-NEXT: v_mov_b32_e32 v6, s4
-; CHECK-NEXT: flat_load_dwordx4 v[10:13], v[6:7]
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s4, v8
+; CHECK-NEXT: v_mov_b32_e32 v2, s4
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT: v_add_co_u32_e32 v2, vcc, s4, v6
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_cmp_lt_u64_e64 s[6:7], s[4:5], 32
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
+; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v7, v3, vcc
; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13]
+; CHECK-NEXT: flat_store_dwordx4 v[2:3], v[8:11]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %dynamic-memcpy-expansion-residual-cond
-; CHECK-NEXT: s_branch .LBB0_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: s_mov_b64 s[4:5], -1
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; CHECK-NEXT: s_cselect_b32 s4, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s4, 1
-; CHECK-NEXT: s_cbranch_scc0 .LBB0_5
-; CHECK-NEXT: s_branch .LBB0_8
-; CHECK-NEXT: .LBB0_4: ; %dynamic-memcpy-expansion-residual-cond.dynamic-memcpy-post-expansion_crit_edge
-; CHECK-NEXT: v_lshlrev_b64 v[6:7], 6, v[2:3]
-; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; CHECK-NEXT: s_cselect_b32 s4, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s4, 1
-; CHECK-NEXT: s_cbranch_scc1 .LBB0_8
-; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-expansion-residual-body.preheader
+; CHECK-NEXT: s_cbranch_execnz .LBB0_5
+; CHECK-NEXT: ; %bb.3: ; %dynamic-memcpy-expansion-residual-body.preheader
; CHECK-NEXT: s_add_u32 s4, s16, 32
; CHECK-NEXT: s_addc_u32 s5, s17, 0
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_add_co_u32_e32 v2, vcc, s4, v4
; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: ; %bb.6: ; %dynamic-memcpy-expansion-residual-body
+; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: s_add_u32 s6, 32, s4
; CHECK-NEXT: s_addc_u32 s7, 0, s5
-; CHECK-NEXT: v_mov_b32_e32 v6, s6
-; CHECK-NEXT: v_mov_b32_e32 v7, s7
-; CHECK-NEXT: flat_load_ubyte v10, v[6:7]
-; CHECK-NEXT: v_mov_b32_e32 v7, s5
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s4, v2
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
+; CHECK-NEXT: v_mov_b32_e32 v9, s7
+; CHECK-NEXT: v_mov_b32_e32 v8, s6
+; CHECK-NEXT: flat_load_ubyte v10, v[8:9]
+; CHECK-NEXT: v_mov_b32_e32 v9, s5
+; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s4, v2
+; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v3, v9, vcc
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, 0, s5
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_byte v[6:7], v10
-; CHECK-NEXT: ; %bb.7:
-; CHECK-NEXT: v_mov_b32_e32 v7, v5
-; CHECK-NEXT: v_mov_b32_e32 v6, v4
-; CHECK-NEXT: .LBB0_8: ; %dynamic-memcpy-post-expansion
+; CHECK-NEXT: flat_store_byte v[8:9], v10
+; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-post-expansion
; CHECK-NEXT: v_and_b32_e32 v2, 15, v0
; CHECK-NEXT: v_and_b32_e32 v0, -16, v0
-; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, v6, v0
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, v4, v0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v1, vcc
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v1, vcc
; CHECK-NEXT: v_cmp_ne_u64_e64 s[4:5], 0, v[0:1]
; CHECK-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v6, s17
+; CHECK-NEXT: v_mov_b32_e32 v8, s17
; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, s16, v4
-; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v6, v5, vcc
-; CHECK-NEXT: s_branch .LBB0_11
-; CHECK-NEXT: .LBB0_9: ; %Flow14
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v8, v5, vcc
+; CHECK-NEXT: s_branch .LBB0_8
+; CHECK-NEXT: .LBB0_6: ; %Flow14
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_or_b64 exec, exec, s[10:11]
; CHECK-NEXT: s_mov_b64 s[8:9], 0
-; CHECK-NEXT: .LBB0_10: ; %Flow16
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: .LBB0_7: ; %Flow16
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_and_b64 s[8:9], s[8:9], exec
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_cmp_lg_u32 s8, 1
-; CHECK-NEXT: s_cbranch_scc0 .LBB0_18
-; CHECK-NEXT: .LBB0_11: ; %while.cond
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_15
+; CHECK-NEXT: .LBB0_8: ; %while.cond
; CHECK-NEXT: ; =>This Loop Header: Depth=1
-; CHECK-NEXT: ; Child Loop BB0_13 Depth 2
-; CHECK-NEXT: ; Child Loop BB0_17 Depth 2
+; CHECK-NEXT: ; Child Loop BB0_10 Depth 2
+; CHECK-NEXT: ; Child Loop BB0_14 Depth 2
; CHECK-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_14
-; CHECK-NEXT: ; %bb.12: ; %dynamic-memcpy-expansion-main-body2.preheader
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: s_cbranch_execz .LBB0_11
+; CHECK-NEXT: ; %bb.9: ; %dynamic-memcpy-expansion-main-body2.preheader
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_mov_b64 s[10:11], 0
; CHECK-NEXT: s_mov_b64 s[12:13], 0
-; CHECK-NEXT: .LBB0_13: ; %dynamic-memcpy-expansion-main-body2
-; CHECK-NEXT: ; Parent Loop BB0_11 Depth=1
+; CHECK-NEXT: .LBB0_10: ; %dynamic-memcpy-expansion-main-body2
+; CHECK-NEXT: ; Parent Loop BB0_8 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_mov_b32_e32 v6, s10
-; CHECK-NEXT: v_mov_b32_e32 v7, s11
-; CHECK-NEXT: flat_load_dwordx4 v[10:13], v[6:7]
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s10, v8
+; CHECK-NEXT: v_mov_b32_e32 v8, s10
+; CHECK-NEXT: v_mov_b32_e32 v9, s11
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; CHECK-NEXT: v_mov_b32_e32 v13, s11
+; CHECK-NEXT: v_add_co_u32_e32 v12, vcc, s10, v6
; CHECK-NEXT: s_add_u32 s10, s10, 16
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
+; CHECK-NEXT: v_addc_co_u32_e32 v13, vcc, v7, v13, vcc
; CHECK-NEXT: s_addc_u32 s11, s11, 0
; CHECK-NEXT: v_cmp_ge_u64_e32 vcc, s[10:11], v[0:1]
; CHECK-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13]
+; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
; CHECK-NEXT: s_andn2_b64 exec, exec, s[12:13]
-; CHECK-NEXT: s_cbranch_execnz .LBB0_13
-; CHECK-NEXT: .LBB0_14: ; %Flow15
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: s_cbranch_execnz .LBB0_10
+; CHECK-NEXT: .LBB0_11: ; %Flow15
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_or_b64 exec, exec, s[8:9]
; CHECK-NEXT: s_mov_b64 s[8:9], -1
-; CHECK-NEXT: s_cbranch_execz .LBB0_10
-; CHECK-NEXT: ; %bb.15: ; %dynamic-memcpy-expansion-residual-cond5
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: s_cbranch_execz .LBB0_7
+; CHECK-NEXT: ; %bb.12: ; %dynamic-memcpy-expansion-residual-cond5
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
-; CHECK-NEXT: s_cbranch_execz .LBB0_9
-; CHECK-NEXT: ; %bb.16: ; %dynamic-memcpy-expansion-residual-body4.preheader
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT: s_cbranch_execz .LBB0_6
+; CHECK-NEXT: ; %bb.13: ; %dynamic-memcpy-expansion-residual-body4.preheader
+; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1
; CHECK-NEXT: s_mov_b64 s[12:13], 0
; CHECK-NEXT: s_mov_b64 s[14:15], 0
-; CHECK-NEXT: .LBB0_17: ; %dynamic-memcpy-expansion-residual-body4
-; CHECK-NEXT: ; Parent Loop BB0_11 Depth=1
+; CHECK-NEXT: .LBB0_14: ; %dynamic-memcpy-expansion-residual-body4
+; CHECK-NEXT: ; Parent Loop BB0_8 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NEXT: v_mov_b32_e32 v10, s13
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s12, v0
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v10, vcc
-; CHECK-NEXT: flat_load_ubyte v11, v[6:7]
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s12, v4
+; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s12, v0
+; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v10, vcc
+; CHECK-NEXT: flat_load_ubyte v11, v[8:9]
+; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s12, v4
; CHECK-NEXT: s_add_u32 s12, s12, 1
; CHECK-NEXT: s_addc_u32 s13, s13, 0
; CHECK-NEXT: v_cmp_ge_u64_e64 s[8:9], s[12:13], v[2:3]
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v5, v10, vcc
+; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v5, v10, vcc
; CHECK-NEXT: s_or_b64 s[14:15], s[8:9], s[14:15]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_byte v[6:7], v11
+; CHECK-NEXT: flat_store_byte v[8:9], v11
; CHECK-NEXT: s_andn2_b64 exec, exec, s[14:15]
-; CHECK-NEXT: s_cbranch_execnz .LBB0_17
-; CHECK-NEXT: s_branch .LBB0_9
-; CHECK-NEXT: .LBB0_18: ; %DummyReturnBlock
+; CHECK-NEXT: s_cbranch_execnz .LBB0_14
+; CHECK-NEXT: s_branch .LBB0_6
+; CHECK-NEXT: .LBB0_15: ; %DummyReturnBlock
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll b/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll
index 5b7f893889744..a265f737deb34 100644
--- a/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll
+++ b/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -O3 | FileCheck %s --check-prefix=PIPELINE
-; RUN: opt < %s -passes='gvn<no-scalar-pre>' -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=NO-SCALAR-PRE
-; RUN: opt < %s -passes='gvn<scalar-pre>' -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=SCALAR-PRE
+; RUN: opt < %s -passes='gvn<no-scalar-pre>' --enable-simple-gvn-hoist=false -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=NO-SCALAR-PRE
+; RUN: opt < %s -passes='gvn<scalar-pre>' --enable-simple-gvn-hoist=false -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=SCALAR-PRE
; Scalar PRE inserts a critical-edge computation and a PHI for the common add.
; That shape needs more NVPTX virtual registers than keeping the duplicated adds.
diff --git a/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
index 205dff7968018..a8eae380b9033 100644
--- a/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
+++ b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes=gvn -S | FileCheck %s
+; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s
; PR12858
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll b/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll
index b6772725d2a88..4eb06478a55c1 100644
--- a/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
; Check that PRE-LOAD across backedge does not
; result in invalid dominator tree.
diff --git a/llvm/test/Transforms/GVN/PRE/local-pre.ll b/llvm/test/Transforms/GVN/PRE/local-pre.ll
index c67a5f1549f80..5d0914b52aa9c 100644
--- a/llvm/test/Transforms/GVN/PRE/local-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/local-pre.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s
-; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false -S | FileCheck %s
+; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s
+; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false --enable-simple-gvn-hoist=false -S | FileCheck %s
declare void @may_exit() nounwind
diff --git a/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll b/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll
index c817ac5d1a131..b682e594ad222 100644
--- a/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -enable-scalar-pre=false -passes=gvn -S < %s | FileCheck %s
-; RUN: opt -enable-scalar-pre=true -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED
-; RUN: opt -passes='gvn<no-scalar-pre>' -S < %s | FileCheck %s
-; RUN: opt -passes='gvn<scalar-pre>' -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED
+; RUN: opt -enable-scalar-pre=false -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s
+; RUN: opt -enable-scalar-pre=true -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED
+; RUN: opt -passes='gvn<no-scalar-pre>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s
+; RUN: opt -passes='gvn<scalar-pre>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED
define void @test_scalar_pre_option(ptr %arr, i8 %cond) {
; CHECK-LABEL: define void @test_scalar_pre_option(
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate.ll b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
index 1915244e9490e..5931735ac5cba 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
target datalayout = "e-p:64:64:64"
diff --git a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
index 92306015378cb..239e1bc5c0d3e 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
-; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false -S | FileCheck %s
+; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false --enable-simple-gvn-hoist=false -S | FileCheck %s
@H = common global i32 0 ; <ptr> [#uses=2]
@G = common global i32 0 ; <ptr> [#uses=1]
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll b/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll
index 2a2019e80323e..ccc760ddc085f 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn' -S %s | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn<memoryssa>' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn' --enable-simple-gvn-hoist=false -S %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn<memoryssa>' --enable-simple-gvn-hoist=false -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
define i32 @test_pointer_phi_select_simp_1(ptr %a, ptr %b, i1 %cond) {
; MDEP-LABEL: @test_pointer_phi_select_simp_1(
diff --git a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
index 22c628bb35464..329c6aee0e5e9 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=gvn -S | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt < %s -passes='gvn<memoryssa>' -S | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA
; This testcase tests insertion of no-cost phis. That is,
; when the value is already available in every predecessor,
; and we just need to insert a phi node to merge the available values.
diff --git a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
index a4ee356628f11..c328826f0bbb1 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
-; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA
@H = common global i32 0
@G = common global i32 0
diff --git a/llvm/test/Transforms/GVN/freeze.ll b/llvm/test/Transforms/GVN/freeze.ll
index de079fddb0dac..577cd017db22a 100644
--- a/llvm/test/Transforms/GVN/freeze.ll
+++ b/llvm/test/Transforms/GVN/freeze.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=gvn -S | FileCheck %s
+; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s
define i1 @f(i1 %a) {
; CHECK-LABEL: @f(
diff --git a/llvm/test/Transforms/GVN/gc_relocate.ll b/llvm/test/Transforms/GVN/gc_relocate.ll
index 6bc71f5da53fb..131b157c26d73 100644
--- a/llvm/test/Transforms/GVN/gc_relocate.ll
+++ b/llvm/test/Transforms/GVN/gc_relocate.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=gvn -S < %s | FileCheck %s
+; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s
declare void @func()
declare i32 @"personality_function"()
diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
index 0c0789e6e4144..25dbe8b80d5a6 100644
--- a/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
+++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll
@@ -49,12 +49,12 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
;
; MAX-DEPTH4-LABEL: @f(
; MAX-DEPTH4-NEXT: entry:
+; MAX-DEPTH4-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
; MAX-DEPTH4-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; MAX-DEPTH4: if.then:
; MAX-DEPTH4-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1
; MAX-DEPTH4-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1
; MAX-DEPTH4-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
-; MAX-DEPTH4-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
; MAX-DEPTH4-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]]
; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
; MAX-DEPTH4: if.then1:
@@ -67,8 +67,7 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
; MAX-DEPTH4-NEXT: [[R1:%.*]] = add i32 [[B]], 2
; MAX-DEPTH4-NEXT: [[AND1:%.*]] = and i32 [[A]], 1
; MAX-DEPTH4-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
-; MAX-DEPTH4-NEXT: [[D1:%.*]] = add i32 [[D]], 1
-; MAX-DEPTH4-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]]
+; MAX-DEPTH4-NEXT: [[S1:%.*]] = add i32 [[D0]], [[B]]
; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
; MAX-DEPTH4: if.then2:
; MAX-DEPTH4-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4
@@ -84,13 +83,13 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
;
; MAX-DEPTH5-LABEL: @f(
; MAX-DEPTH5-NEXT: entry:
+; MAX-DEPTH5-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH5-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B:%.*]]
; MAX-DEPTH5-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; MAX-DEPTH5: if.then:
-; MAX-DEPTH5-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH5-NEXT: [[R0:%.*]] = add i32 [[B]], 1
; MAX-DEPTH5-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1
; MAX-DEPTH5-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
-; MAX-DEPTH5-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1
-; MAX-DEPTH5-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]]
; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
; MAX-DEPTH5: if.then1:
; MAX-DEPTH5-NEXT: [[OR0:%.*]] = or i32 [[R0]], 1
@@ -102,8 +101,6 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
; MAX-DEPTH5-NEXT: [[R1:%.*]] = add i32 [[B]], 2
; MAX-DEPTH5-NEXT: [[AND1:%.*]] = and i32 [[A]], 1
; MAX-DEPTH5-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
-; MAX-DEPTH5-NEXT: [[D1:%.*]] = add i32 [[D]], 1
-; MAX-DEPTH5-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]]
; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
; MAX-DEPTH5: if.then2:
; MAX-DEPTH5-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4
@@ -113,7 +110,7 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
; MAX-DEPTH5-NEXT: br label [[EXIT]]
; MAX-DEPTH5: exit:
; MAX-DEPTH5-NEXT: [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
-; MAX-DEPTH5-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH5-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S0]], [[IF_THEN2]] ], [ [[S0]], [[IF_ELSE2]] ]
; MAX-DEPTH5-NEXT: [[R:%.*]] = add i32 [[OR]], [[S]]
; MAX-DEPTH5-NEXT: ret i32 [[R]]
;
diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
index e774f64d4a4e5..d35ca95cb87c5 100644
--- a/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
+++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll
@@ -7,19 +7,16 @@ target triple = "aarch64-unknown-linux"
define dso_local i32 @everything_hoisted(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @everything_hoisted(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @barrier(i32 [[A:%.*]])
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], [[B:%.*]]
; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP1]], [[C:%.*]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @barrier(i32 [[A]])
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[B]]
-; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP4]], [[C]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -46,15 +43,14 @@ if.end:
define dso_local i32 @spec_barrier_short_side(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @spec_barrier_short_side(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A]])
; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP0]]
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2
; CHECK-NEXT: br label [[IF_END]]
@@ -87,15 +83,14 @@ if.end:
define dso_local i32 @spec_barrier_long_side(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @spec_barrier_long_side(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A]])
-; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP0]]
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2
; CHECK-NEXT: br label [[IF_END]]
@@ -127,15 +122,14 @@ if.end:
define dso_local i32 @no_reorder_across_volatile(i1 %cc, i32 %a, i32 %b, i32 %c, ptr %p) {
; CHECK-LABEL: @no_reorder_across_volatile(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]]
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
; CHECK-NEXT: store volatile i32 0, ptr [[P:%.*]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[C]], [[TMP3]]
+; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[C]], [[TMP0]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ]
@@ -163,19 +157,16 @@ if.end:
define dso_local i32 @no_barrier_call(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @no_barrier_call(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
; CHECK-NEXT: call void @will_return()
; CHECK-NEXT: [[TMP0:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[C:%.*]], [[TMP0]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: call void @will_return()
-; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[C]], [[TMP2]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP3]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP1]], [[IF_ELSE]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -243,19 +234,16 @@ if.end:
define dso_local i32 @multiple_use(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @multiple_use(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i32 [[TMP0]], [[C:%.*]]
; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i32 [[TMP3]], [[C]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -282,19 +270,16 @@ if.end:
define dso_local i32 @commutative_ops(i1 %cc, i32 %a, i32 %b, i32 %c) {
; CHECK-LABEL: @commutative_ops(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP0]], [[C:%.*]]
; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[C]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP3]], [[TMP4]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -378,5 +363,57 @@ if.else:
ret i32 %1
}
+define dso_local i32 @no_hoist_nomerge(i1 %cc, i32 %x, ptr %p) {
+; CHECK-LABEL: @no_hoist_nomerge(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[W:%.*]] = add i32 [[V]], [[X:%.*]]
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @cannot_merge()
+; CHECK-NEXT: ret i32 [[TMP0]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @cannot_merge()
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %v = load i32, ptr %p
+ %w = add i32 %v, %x
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %0 = call i32 @cannot_merge()
+ ret i32 %0
+
+if.else:
+ %1 = call i32 @cannot_merge()
+ ret i32 %1
+}
+
+
+define i32 @no_hoist_bundles(i1 %cc, i32 %x, ptr %p) {
+; CHECK-LABEL: @no_hoist_bundles(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[U:%.*]] = call i32 @will_return(i1 true, i32 [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] [ "foo"(i32 [[X]], ptr [[P]]) ]
+; CHECK-NEXT: ret i32 [[U]]
+; CHECK: if.else:
+; CHECK-NEXT: [[V:%.*]] = call i32 @will_return(i1 false, i32 [[X]], ptr [[P]]) #[[ATTR0]] [ "bar"(i32 [[X]], ptr [[P]]) ]
+; CHECK-NEXT: ret i32 [[V]]
+;
+entry:
+ br i1 %cc, label %if.then, label %if.else
+
+if.then:
+ %u = call i32 @will_return(i1 %cc, i32 %x, ptr %p) memory(none) [ "foo"(i32 %x, ptr %p) ]
+ ret i32 %u
+
+if.else:
+ %v = call i32 @will_return(i1 %cc, i32 %x, ptr %p) memory(none) [ "bar"(i32 %x, ptr %p) ]
+ ret i32 %v
+}
+
declare i32 @barrier(i32) memory(none)
-declare void @will_return() nounwind willreturn
+declare void @will_return(i1, i32, ptr) memory(none) nounwind willreturn
+declare void @cannot_merge() memory(none) nounwind willreturn nomerge
More information about the cfe-commits
mailing list