[llvm-branch-commits] [llvm] [GVN] Assign unique VNs to calls with operand bundles (PR #210335)
Momchil Velikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 23 05:59:13 PDT 2026
https://github.com/momchil-velikov updated https://github.com/llvm/llvm-project/pull/210335
>From 6096261b4f019ad2a533b471ba426369158b8322 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 21 Jul 2026 16:27:12 +0100
Subject: [PATCH 1/7] [GVN] Restructure `GVN.h` to reduce its size (NFC)
* Rename `GVNPass::ValueTable` to `GVNValueTable`, and move it out to
the `llvm` and to its own file `GVNValueTable.h` (the type is also
used by `GVNHoistPass` and it makes sense to have it in a separate
file instead of `GVNHoistPass` peeking into `GVN.h`).
* Move `GVNPass::Expression` into `llvm::GVNValueTable`.
* Move `DepKind`, `ReachingMemVal`, and `DependencyBlockInfo` to `GVN.cpp`.
* Move `GVNHoistPass` and `GVNSinkPass` to their own headers.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 169 +-----------------
.../include/llvm/Transforms/Scalar/GVNHoist.h | 30 ++++
llvm/include/llvm/Transforms/Scalar/GVNSink.h | 30 ++++
.../llvm/Transforms/Scalar/GVNValueTable.h | 164 +++++++++++++++++
llvm/lib/Passes/PassBuilder.cpp | 2 +
llvm/lib/Passes/PassBuilderPipelines.cpp | 2 +
llvm/lib/Transforms/Scalar/GVN.cpp | 161 +++++++++++------
llvm/lib/Transforms/Scalar/GVNHoist.cpp | 14 +-
llvm/lib/Transforms/Scalar/GVNSink.cpp | 2 +-
9 files changed, 354 insertions(+), 220 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Scalar/GVNHoist.h
create mode 100644 llvm/include/llvm/Transforms/Scalar/GVNSink.h
create mode 100644 llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 383b265586674..4d8a2d3eda820 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -26,6 +26,8 @@
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
+#include "llvm/Transforms/Scalar/GVNValueTable.h"
+
#include <cstdint>
#include <optional>
#include <utility>
@@ -122,102 +124,15 @@ struct GVNOptions {
/// this particular pass here.
class GVNPass : public OptionalPassInfoMixin<GVNPass> {
public:
- struct Expression;
struct AvailableValue;
struct AvailableValueInBlock;
- /// This class holds the mapping between values and value numbers. It is used
- /// as an efficient mechanism to determine the expression-wise equivalence of
- /// two values.
- class ValueTable {
- DenseMap<Value *, uint32_t> ValueNumbering;
- DenseMap<Expression, uint32_t> ExpressionNumbering;
-
- // Expressions is the vector of Expression. ExprIdx is the mapping from
- // value number to the index of Expression in Expressions. We use it
- // instead of a DenseMap because filling such mapping is faster than
- // filling a DenseMap and the compile time is a little better.
- uint32_t NextExprNumber = 0;
-
- std::vector<Expression> Expressions;
- std::vector<uint32_t> ExprIdx;
-
- // Value number to PHINode mapping. Used for phi-translate in scalarpre.
- DenseMap<uint32_t, PHINode *> NumberingPhi;
-
- // Value number to BasicBlock mapping. Used for phi-translate across
- // MemoryPhis.
- DenseMap<uint32_t, BasicBlock *> NumberingBB;
-
- // Cache for phi-translate in scalarpre.
- using PhiTranslateMap =
- DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>;
- PhiTranslateMap PhiTranslateTable;
-
- AAResults *AA = nullptr;
- MemoryDependenceResults *MD = nullptr;
- bool IsMDEnabled = false;
- MemorySSA *MSSA = nullptr;
- bool IsMSSAEnabled = false;
- DominatorTree *DT = nullptr;
-
- uint32_t NextValueNumber = 1;
-
- Expression createExpr(Instruction *I);
- Expression createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate,
- Value *LHS, Value *RHS);
- Expression createExtractvalueExpr(ExtractValueInst *EI);
- Expression createGEPExpr(GetElementPtrInst *GEP);
- uint32_t lookupOrAddCall(CallInst *C);
- uint32_t computeLoadStoreVN(Instruction *I);
- uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN);
- bool areCallValsEqual(uint32_t Num, uint32_t NewNum, const BasicBlock *Pred,
- const BasicBlock *PhiBlock, GVNPass &GVN);
- std::pair<uint32_t, bool> assignExpNewValueNum(Expression &Exp);
- bool areAllValsInBB(uint32_t Num, const BasicBlock *BB, GVNPass &GVN);
- void addMemoryStateToExp(Instruction *I, Expression &Exp);
-
- public:
- LLVM_ABI ValueTable();
- LLVM_ABI ValueTable(const ValueTable &Arg);
- LLVM_ABI ValueTable(ValueTable &&Arg);
- LLVM_ABI ~ValueTable();
- LLVM_ABI ValueTable &operator=(const ValueTable &Arg);
-
- LLVM_ABI void add(Value *V, uint32_t Num);
- LLVM_ABI uint32_t lookupOrAdd(MemoryAccess *MA);
- LLVM_ABI uint32_t lookupOrAdd(Value *V);
- LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const;
- LLVM_ABI uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred,
- Value *LHS, Value *RHS);
- LLVM_ABI uint32_t lookupPtrToInt(Value *Ptr, Type *Ty);
- LLVM_ABI uint32_t phiTranslate(const BasicBlock *BB,
- const BasicBlock *PhiBlock, uint32_t Num,
- GVNPass &GVN);
- LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
- const BasicBlock &CurrBlock);
- LLVM_ABI bool exists(Value *V) const;
- LLVM_ABI void clear();
- LLVM_ABI void erase(Value *V);
- void setAliasAnalysis(AAResults *A) { AA = A; }
- AAResults *getAliasAnalysis() const { return AA; }
- void setMemDep(MemoryDependenceResults *M, bool MDEnabled = true) {
- MD = M;
- IsMDEnabled = MDEnabled;
- }
- void setMemorySSA(MemorySSA *M, bool MSSAEnabled = false) {
- MSSA = M;
- IsMSSAEnabled = MSSAEnabled;
- }
- void setDomTree(DominatorTree *D) { DT = D; }
- uint32_t getNextUnusedValueNumber() { return NextValueNumber; }
- LLVM_ABI void verifyRemoved(const Value *) const;
- };
+ struct ReachingMemVal;
+ struct DependencyBlockInfo;
-private:
+ friend class GVNValueTable;
friend class GVNLegacyPass;
- friend struct DenseMapInfo<Expression>;
+private:
GVNOptions Options;
MemoryDependenceResults *MD = nullptr;
DominatorTree *DT = nullptr;
@@ -229,7 +144,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoopInfo *LI = nullptr;
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
- ValueTable VN;
+ GVNValueTable VN;
/// A mapping from value numbers to lists of Value*'s that
/// have that value number. Use findLeader to query it.
@@ -349,62 +264,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
- enum class DepKind {
- Other = 0, // Unknown value.
- Def, // Exactly overlapping locations.
- Clobber, // Reaching value superset of needed bits.
- Select, // Reaching value is a select of two reaching addresses.
- };
-
- // Describe a memory location value, such that there exists a path to a point
- // in the program, along which that memory location is not modified.
- struct ReachingMemVal {
- DepKind Kind;
- BasicBlock *Block;
- const Value *Addr;
- Instruction *Inst;
- int32_t Offset;
- // For DepKind::Select only: the condition and the two addresses referenced
- // by the "true" and "false" side of the select-dependent load.
- const Value *SelCond = nullptr;
- const Value *SelTrueAddr = nullptr;
- const Value *SelFalseAddr = nullptr;
-
- static ReachingMemVal getUnknown(BasicBlock *BB, const Value *Addr,
- Instruction *Inst = nullptr) {
- return {DepKind::Other, BB, Addr, Inst, -1};
- }
-
- static ReachingMemVal getDef(const Value *Addr, Instruction *Inst) {
- return {DepKind::Def, Inst->getParent(), Addr, Inst, -1};
- }
-
- static ReachingMemVal getClobber(const Value *Addr, Instruction *Inst,
- int32_t Offset = -1) {
- return {DepKind::Clobber, Inst->getParent(), Addr, Inst, Offset};
- }
-
- static ReachingMemVal getSelect(BasicBlock *BB, const Value *Cond,
- const Value *TrueAddr,
- const Value *FalseAddr) {
- return {DepKind::Select, BB, nullptr, nullptr, -1, Cond,
- TrueAddr, FalseAddr};
- }
- };
-
- struct DependencyBlockInfo {
- DependencyBlockInfo() = delete;
- DependencyBlockInfo(const PHITransAddr &Addr, MemoryAccess *ClobberMA)
- : Addr(Addr), InitialClobberMA(ClobberMA), ClobberMA(ClobberMA),
- ForceUnknown(false), Visited(false) {}
- PHITransAddr Addr;
- MemoryAccess *InitialClobberMA;
- MemoryAccess *ClobberMA;
- std::optional<ReachingMemVal> MemVal;
- bool ForceUnknown : 1;
- bool Visited : 1;
- };
-
using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
/// Given a select-dependency for the load (the load address is a select of
@@ -539,20 +398,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI FunctionPass *createGVNPass(bool ScalarPRE);
LLVM_ABI FunctionPass *createGVNPass();
-/// A simple and fast domtree-based GVN pass to hoist common expressions
-/// from sibling branches.
-struct GVNHoistPass : OptionalPassInfoMixin<GVNHoistPass> {
- /// Run the pass over the function.
- LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-};
-
-/// Uses an "inverted" value numbering to decide the similarity of
-/// expressions and sinks similar expressions into successors.
-struct GVNSinkPass : OptionalPassInfoMixin<GVNSinkPass> {
- /// Run the pass over the function.
- LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-};
-
} // end namespace llvm
#endif // LLVM_TRANSFORMS_SCALAR_GVN_H
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNHoist.h b/llvm/include/llvm/Transforms/Scalar/GVNHoist.h
new file mode 100644
index 0000000000000..2ab562b13247a
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/GVNHoist.h
@@ -0,0 +1,30 @@
+//===- GVNHoist.h - Hoist scalar and load expressions ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides the interface for the GVNHoist pass.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_GVNHOIST_H
+#define LLVM_TRANSFORMS_SCALAR_GVNHOIST_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+/// A simple and fast domtree-based GVN pass to hoist common expressions
+/// from sibling branches.
+struct GVNHoistPass : OptionalPassInfoMixin<GVNHoistPass> {
+ /// Run the pass over the function.
+ LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_GVNHOIST_H
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNSink.h b/llvm/include/llvm/Transforms/Scalar/GVNSink.h
new file mode 100644
index 0000000000000..f8dafc87d716c
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/GVNSink.h
@@ -0,0 +1,30 @@
+//===- GVNSink.h - Sink expressions into successors -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides the interface for the GVNSink pass.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_GVNSINK_H
+#define LLVM_TRANSFORMS_SCALAR_GVNSINK_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+/// Uses an "inverted" value numbering to decide the similarity of
+/// expressions and sinks similar expressions into successors.
+struct GVNSinkPass : OptionalPassInfoMixin<GVNSinkPass> {
+ /// Run the pass over the function.
+ LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_GVNSINK_H
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h b/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
new file mode 100644
index 0000000000000..5d93461c1260c
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
@@ -0,0 +1,164 @@
+//===- GVNValueTable.h - Value table for GVN ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides a data structure for mapping values and expressions to
+/// congruence class IDs.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_GVNVALUETABLE_H
+#define LLVM_TRANSFORMS_SCALAR_GVNVALUETABLE_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/PHITransAddr.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Transforms/Scalar/GVNValueTable.h"
+
+#include <cstdint>
+#include <optional>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace llvm {
+
+class AAResults;
+class AssumeInst;
+class AssumptionCache;
+class BasicBlock;
+class BatchAAResults;
+class CallInst;
+class CondBrInst;
+class EarliestEscapeAnalysis;
+class ExtractValueInst;
+class Function;
+class FunctionPass;
+class GVNLegacyPass;
+class GVNPass;
+class GetElementPtrInst;
+class ImplicitControlFlowTracking;
+class LoadInst;
+class LoopInfo;
+class MemDepResult;
+class MemoryAccess;
+class MemoryDependenceResults;
+class MemoryLocation;
+class MemorySSA;
+class MemorySSAUpdater;
+class NonLocalDepResult;
+class OptimizationRemarkEmitter;
+class PHINode;
+class TargetLibraryInfo;
+class Value;
+class IntrinsicInst;
+
+/// This class holds the mapping between values and value numbers. It is used
+/// as an efficient mechanism to determine the expression-wise equivalence of
+/// two values.
+class GVNValueTable {
+public:
+ struct Expression;
+
+private:
+ DenseMap<Value *, uint32_t> ValueNumbering;
+ DenseMap<Expression, uint32_t> ExpressionNumbering;
+
+ // Expressions is the vector of Expression. ExprIdx is the mapping from
+ // value number to the index of Expression in Expressions. We use it
+ // instead of a DenseMap because filling such mapping is faster than
+ // filling a DenseMap and the compile time is a little better.
+ uint32_t NextExprNumber = 0;
+
+ std::vector<Expression> Expressions;
+ std::vector<uint32_t> ExprIdx;
+
+ // Value number to PHINode mapping. Used for phi-translate in scalarpre.
+ DenseMap<uint32_t, PHINode *> NumberingPhi;
+
+ // Value number to BasicBlock mapping. Used for phi-translate across
+ // MemoryPhis.
+ DenseMap<uint32_t, BasicBlock *> NumberingBB;
+
+ // Cache for phi-translate in scalarpre.
+ using PhiTranslateMap =
+ DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>;
+ PhiTranslateMap PhiTranslateTable;
+
+ AAResults *AA = nullptr;
+ MemoryDependenceResults *MD = nullptr;
+ bool IsMDEnabled = false;
+ MemorySSA *MSSA = nullptr;
+ bool IsMSSAEnabled = false;
+ DominatorTree *DT = nullptr;
+
+ uint32_t NextValueNumber = 1;
+
+ Expression createExpr(Instruction *I);
+ Expression createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate,
+ Value *LHS, Value *RHS);
+ Expression createExtractvalueExpr(ExtractValueInst *EI);
+ Expression createGEPExpr(GetElementPtrInst *GEP);
+ uint32_t lookupOrAddCall(CallInst *C);
+ uint32_t computeLoadStoreVN(Instruction *I);
+ uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN);
+ bool areCallValsEqual(uint32_t Num, uint32_t NewNum, const BasicBlock *Pred,
+ const BasicBlock *PhiBlock, GVNPass &GVN);
+ std::pair<uint32_t, bool> assignExpNewValueNum(Expression &Exp);
+ bool areAllValsInBB(uint32_t Num, const BasicBlock *BB, GVNPass &GVN);
+ void addMemoryStateToExp(Instruction *I, Expression &Exp);
+
+public:
+ LLVM_ABI GVNValueTable();
+ LLVM_ABI GVNValueTable(const GVNValueTable &Arg);
+ LLVM_ABI GVNValueTable(GVNValueTable &&Arg);
+ LLVM_ABI ~GVNValueTable();
+ LLVM_ABI GVNValueTable &operator=(const GVNValueTable &Arg);
+
+ LLVM_ABI void add(Value *V, uint32_t Num);
+ LLVM_ABI uint32_t lookupOrAdd(MemoryAccess *MA);
+ LLVM_ABI uint32_t lookupOrAdd(Value *V);
+ LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const;
+ LLVM_ABI uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred,
+ Value *LHS, Value *RHS);
+ LLVM_ABI uint32_t lookupPtrToInt(Value *Ptr, Type *Ty);
+ LLVM_ABI uint32_t phiTranslate(const BasicBlock *BB,
+ const BasicBlock *PhiBlock, uint32_t Num,
+ GVNPass &GVN);
+ LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
+ const BasicBlock &CurrBlock);
+ LLVM_ABI bool exists(Value *V) const;
+ LLVM_ABI void clear();
+ LLVM_ABI void erase(Value *V);
+ void setAliasAnalysis(AAResults *A) { AA = A; }
+ AAResults *getAliasAnalysis() const { return AA; }
+ void setMemDep(MemoryDependenceResults *M, bool MDEnabled = true) {
+ MD = M;
+ IsMDEnabled = MDEnabled;
+ }
+ void setMemorySSA(MemorySSA *M, bool MSSAEnabled = false) {
+ MSSA = M;
+ IsMSSAEnabled = MSSAEnabled;
+ }
+ void setDomTree(DominatorTree *D) { DT = D; }
+ uint32_t getNextUnusedValueNumber() { return NextValueNumber; }
+ LLVM_ABI void verifyRemoved(const Value *) const;
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_GVNVALUETABLE_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 17bc2a5a5afdd..2c9d32e758052 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -299,6 +299,8 @@
#include "llvm/Transforms/Scalar/FlattenCFG.h"
#include "llvm/Transforms/Scalar/Float2Int.h"
#include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/GVNHoist.h"
+#include "llvm/Transforms/Scalar/GVNSink.h"
#include "llvm/Transforms/Scalar/GuardWidening.h"
#include "llvm/Transforms/Scalar/IVUsersPrinter.h"
#include "llvm/Transforms/Scalar/IndVarSimplify.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index d62828c78bfe4..e46f98889e49d 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -102,6 +102,8 @@
#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
#include "llvm/Transforms/Scalar/Float2Int.h"
#include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/GVNHoist.h"
+#include "llvm/Transforms/Scalar/GVNSink.h"
#include "llvm/Transforms/Scalar/IndVarSimplify.h"
#include "llvm/Transforms/Scalar/InferAlignment.h"
#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 996ed2a72cdaf..d763bab153822 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -145,7 +145,7 @@ static cl::opt<uint32_t> MaxNumInsnsPerBlock(
cl::desc("Max number of instructions to scan in each basic block in GVN "
"(default = 100)"));
-struct llvm::GVNPass::Expression {
+struct llvm::GVNValueTable::Expression {
uint32_t Opcode;
bool Commutative = false;
// The type is not necessarily the result type of the expression, it may be
@@ -178,15 +178,15 @@ struct llvm::GVNPass::Expression {
}
};
-template <> struct llvm::DenseMapInfo<GVNPass::Expression> {
- static unsigned getHashValue(const GVNPass::Expression &E) {
+template <> struct llvm::DenseMapInfo<GVNValueTable::Expression> {
+ static unsigned getHashValue(const GVNValueTable::Expression &E) {
using llvm::hash_value;
return static_cast<unsigned>(hash_value(E));
}
- static bool isEqual(const GVNPass::Expression &LHS,
- const GVNPass::Expression &RHS) {
+ static bool isEqual(const GVNValueTable::Expression &LHS,
+ const GVNValueTable::Expression &RHS) {
return LHS == RHS;
}
};
@@ -391,7 +391,7 @@ struct llvm::GVNPass::AvailableValueInBlock {
// ValueTable Internal Functions
//===----------------------------------------------------------------------===//
-GVNPass::Expression GVNPass::ValueTable::createExpr(Instruction *I) {
+GVNValueTable::Expression GVNValueTable::createExpr(Instruction *I) {
Expression E;
E.Ty = I->getType();
E.Opcode = I->getOpcode();
@@ -438,8 +438,9 @@ GVNPass::Expression GVNPass::ValueTable::createExpr(Instruction *I) {
return E;
}
-GVNPass::Expression GVNPass::ValueTable::createCmpExpr(
- unsigned Opcode, CmpInst::Predicate Predicate, Value *LHS, Value *RHS) {
+GVNValueTable::Expression
+GVNValueTable::createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate,
+ Value *LHS, Value *RHS) {
assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
"Not a comparison!");
Expression E;
@@ -457,8 +458,8 @@ GVNPass::Expression GVNPass::ValueTable::createCmpExpr(
return E;
}
-GVNPass::Expression
-GVNPass::ValueTable::createExtractvalueExpr(ExtractValueInst *EI) {
+GVNValueTable::Expression
+GVNValueTable::createExtractvalueExpr(ExtractValueInst *EI) {
assert(EI && "Not an ExtractValueInst?");
Expression E;
E.Ty = EI->getType();
@@ -486,7 +487,7 @@ GVNPass::ValueTable::createExtractvalueExpr(ExtractValueInst *EI) {
return E;
}
-GVNPass::Expression GVNPass::ValueTable::createGEPExpr(GetElementPtrInst *GEP) {
+GVNValueTable::Expression GVNValueTable::createGEPExpr(GetElementPtrInst *GEP) {
Expression E;
Type *PtrTy = GEP->getType()->getScalarType();
const DataLayout &DL = GEP->getDataLayout();
@@ -518,7 +519,7 @@ GVNPass::Expression GVNPass::ValueTable::createGEPExpr(GetElementPtrInst *GEP) {
return E;
}
-uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) {
+uint32_t GVNValueTable::lookupOrAddCall(CallInst *C) {
// FIXME: Currently the calls which may access the thread id may
// be considered as not accessing the memory. But this is
// problematic for coroutines, since coroutines may resume in a
@@ -651,7 +652,7 @@ uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) {
}
/// Returns the value number for the specified load or store instruction.
-uint32_t GVNPass::ValueTable::computeLoadStoreVN(Instruction *I) {
+uint32_t GVNValueTable::computeLoadStoreVN(Instruction *I) {
if (!MSSA || !IsMSSAEnabled) {
ValueNumbering[I] = NextValueNumber;
return NextValueNumber++;
@@ -671,9 +672,9 @@ uint32_t GVNPass::ValueTable::computeLoadStoreVN(Instruction *I) {
/// Translate value number \p Num using phis, so that it has the values of
/// the phis in BB.
-uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
+uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock,
+ uint32_t Num, GVNPass &GVN) {
// See if we can refine the value number by looking at the PN incoming value
// for the given predecessor.
if (PHINode *PN = NumberingPhi[Num]) {
@@ -753,10 +754,9 @@ uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
// Return true if the value number \p Num and NewNum have equal value.
// Return false if the result is unknown.
-bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
- const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- GVNPass &GVN) {
+bool GVNValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
+ const BasicBlock *Pred,
+ const BasicBlock *PhiBlock, GVNPass &GVN) {
CallInst *Call = nullptr;
auto Leaders = GVN.LeaderTable.getLeaders(Num);
for (const auto &Entry : Leaders) {
@@ -788,8 +788,7 @@ bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
/// Return a pair the first field showing the value number of \p Exp and the
/// second field showing whether it is a value number newly created.
-std::pair<uint32_t, bool>
-GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
+std::pair<uint32_t, bool> GVNValueTable::assignExpNewValueNum(Expression &Exp) {
uint32_t &E = ExpressionNumbering[Exp];
bool CreateNewValNum = !E;
if (CreateNewValNum) {
@@ -804,11 +803,12 @@ GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) {
/// Return whether all the values related with the same \p num are
/// defined in \p BB.
-bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
- GVNPass &GVN) {
- return all_of(
- GVN.LeaderTable.getLeaders(Num),
- [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
+bool GVNValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
+ GVNPass &GVN) {
+ return all_of(GVN.LeaderTable.getLeaders(Num),
+ [=](const GVNPass::LeaderMap::LeaderTableEntry &L) {
+ return L.BB == BB;
+ });
}
/// Include the incoming memory state into the hash of the expression for the
@@ -817,7 +817,7 @@ bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
/// * a MemoryPhi, add the value number of the basic block corresponding to that
/// MemoryPhi,
/// * a MemoryDef, add the value number of the memory setting instruction.
-void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
+void GVNValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
@@ -828,21 +828,20 @@ void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
// ValueTable External Functions
//===----------------------------------------------------------------------===//
-GVNPass::ValueTable::ValueTable() = default;
-GVNPass::ValueTable::ValueTable(const ValueTable &) = default;
-GVNPass::ValueTable::ValueTable(ValueTable &&) = default;
-GVNPass::ValueTable::~ValueTable() = default;
-GVNPass::ValueTable &
-GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default;
+GVNValueTable::GVNValueTable() = default;
+GVNValueTable::GVNValueTable(const GVNValueTable &) = default;
+GVNValueTable::GVNValueTable(GVNValueTable &&) = default;
+GVNValueTable::~GVNValueTable() = default;
+GVNValueTable &GVNValueTable::operator=(const GVNValueTable &Arg) = default;
/// add - Insert a value into the table with a specified value number.
-void GVNPass::ValueTable::add(Value *V, uint32_t Num) {
+void GVNValueTable::add(Value *V, uint32_t Num) {
ValueNumbering.insert(std::make_pair(V, Num));
if (PHINode *PN = dyn_cast<PHINode>(V))
NumberingPhi[Num] = PN;
}
-uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
+uint32_t GVNValueTable::lookupOrAdd(MemoryAccess *MA) {
return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
? lookupOrAdd(MA->getBlock())
: lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
@@ -850,7 +849,7 @@ uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) {
/// lookupOrAdd - Returns the value number for the specified value, assigning
/// it a new number if it did not have one before.
-uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
+uint32_t GVNValueTable::lookupOrAdd(Value *V) {
auto VI = ValueNumbering.find(V);
if (VI != ValueNumbering.end())
return VI->second;
@@ -935,7 +934,7 @@ uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) {
/// Returns the value number of the specified value. Fails if
/// the value has not yet been numbered.
-uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
+uint32_t GVNValueTable::lookup(Value *V, bool Verify) const {
auto VI = ValueNumbering.find(V);
if (Verify) {
assert(VI != ValueNumbering.end() && "Value not numbered?");
@@ -948,15 +947,15 @@ uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const {
/// assigning it a new number if it did not have one before. Useful when
/// we deduced the result of a comparison, but don't immediately have an
/// instruction realizing that comparison to hand.
-uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode,
- CmpInst::Predicate Predicate,
- Value *LHS, Value *RHS) {
+uint32_t GVNValueTable::lookupOrAddCmp(unsigned Opcode,
+ CmpInst::Predicate Predicate, Value *LHS,
+ Value *RHS) {
Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
return assignExpNewValueNum(Exp).first;
}
/// Returns the value number of ptrtoint \p Ptr to \Ty.
-uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
+uint32_t GVNValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
Expression Exp(Instruction::PtrToInt);
Exp.Ty = Ty;
Exp.VarArgs.push_back(lookupOrAdd(Ptr));
@@ -964,9 +963,9 @@ uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
}
/// Wrap phiTranslateImpl to provide caching functionality.
-uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
- const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
+uint32_t GVNValueTable::phiTranslate(const BasicBlock *Pred,
+ const BasicBlock *PhiBlock, uint32_t Num,
+ GVNPass &GVN) {
auto FindRes = PhiTranslateTable.find({Num, Pred});
if (FindRes != PhiTranslateTable.end())
return FindRes->second;
@@ -977,19 +976,19 @@ uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred,
/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
/// again.
-void GVNPass::ValueTable::eraseTranslateCacheEntry(
- uint32_t Num, const BasicBlock &CurrBlock) {
+void GVNValueTable::eraseTranslateCacheEntry(uint32_t Num,
+ const BasicBlock &CurrBlock) {
for (const BasicBlock *Pred : predecessors(&CurrBlock))
PhiTranslateTable.erase({Num, Pred});
}
/// Returns true if a value number exists for the specified value.
-bool GVNPass::ValueTable::exists(Value *V) const {
+bool GVNValueTable::exists(Value *V) const {
return ValueNumbering.contains(V);
}
/// Remove all entries from the ValueTable.
-void GVNPass::ValueTable::clear() {
+void GVNValueTable::clear() {
ValueNumbering.clear();
ExpressionNumbering.clear();
NumberingPhi.clear();
@@ -1002,7 +1001,7 @@ void GVNPass::ValueTable::clear() {
}
/// Remove a value from the value numbering.
-void GVNPass::ValueTable::erase(Value *V) {
+void GVNValueTable::erase(Value *V) {
uint32_t Num = ValueNumbering.lookup(V);
ValueNumbering.erase(V);
// If V is PHINode, V <--> value number is an one-to-one mapping.
@@ -1014,7 +1013,7 @@ void GVNPass::ValueTable::erase(Value *V) {
/// verifyRemoved - Verify that the value is removed from all internal data
/// structures.
-void GVNPass::ValueTable::verifyRemoved(const Value *V) const {
+void GVNValueTable::verifyRemoved(const Value *V) const {
assert(!ValueNumbering.contains(V) &&
"Inst still occurs in value numbering map!");
}
@@ -1075,6 +1074,66 @@ void GVNPass::LeaderMap::erase(uint32_t N, Instruction *I,
}
}
+//===----------------------------------------------------------------------===//
+// Helper Dependency Information Classes
+//===----------------------------------------------------------------------===//
+
+enum class DepKind {
+ Other = 0, // Unknown value.
+ Def, // Exactly overlapping locations.
+ Clobber, // Reaching value superset of needed bits.
+ Select, // Reaching value is a select of two reaching addresses.
+};
+
+// Describe a memory location value, such that there exists a path to a point
+// in the program, along which that memory location is not modified.
+struct GVNPass::ReachingMemVal {
+ DepKind Kind;
+ BasicBlock *Block;
+ const Value *Addr;
+ Instruction *Inst;
+ int32_t Offset;
+ // For DepKind::Select only: the condition and the two addresses referenced
+ // by the "true" and "false" side of the select-dependent load.
+ const Value *SelCond = nullptr;
+ const Value *SelTrueAddr = nullptr;
+ const Value *SelFalseAddr = nullptr;
+
+ static ReachingMemVal getUnknown(BasicBlock *BB, const Value *Addr,
+ Instruction *Inst = nullptr) {
+ return {DepKind::Other, BB, Addr, Inst, -1};
+ }
+
+ static ReachingMemVal getDef(const Value *Addr, Instruction *Inst) {
+ return {DepKind::Def, Inst->getParent(), Addr, Inst, -1};
+ }
+
+ static ReachingMemVal getClobber(const Value *Addr, Instruction *Inst,
+ int32_t Offset = -1) {
+ return {DepKind::Clobber, Inst->getParent(), Addr, Inst, Offset};
+ }
+
+ static ReachingMemVal getSelect(BasicBlock *BB, const Value *Cond,
+ const Value *TrueAddr,
+ const Value *FalseAddr) {
+ return {DepKind::Select, BB, nullptr, nullptr, -1, Cond,
+ TrueAddr, FalseAddr};
+ }
+};
+
+struct GVNPass::DependencyBlockInfo {
+ DependencyBlockInfo() = delete;
+ DependencyBlockInfo(const PHITransAddr &Addr, MemoryAccess *ClobberMA)
+ : Addr(Addr), InitialClobberMA(ClobberMA), ClobberMA(ClobberMA),
+ ForceUnknown(false), Visited(false) {}
+ PHITransAddr Addr;
+ MemoryAccess *InitialClobberMA;
+ MemoryAccess *ClobberMA;
+ std::optional<ReachingMemVal> MemVal;
+ bool ForceUnknown : 1;
+ bool Visited : 1;
+};
+
//===----------------------------------------------------------------------===//
// GVN Pass
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index 3022f4a8481d0..da605664df87b 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -66,8 +66,10 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/GVNValueTable.h"
+#include "llvm/Transforms/Scalar/GVNHoist.h"
#include "llvm/Transforms/Utils/Local.h"
+
#include <algorithm>
#include <cassert>
#include <memory>
@@ -163,7 +165,7 @@ class InsnInfo {
public:
// Inserts I and its value number in VNtoScalars.
- void insert(Instruction *I, GVNPass::ValueTable &VN) {
+ void insert(Instruction *I, GVNValueTable &VN) {
// Scalar instruction.
unsigned V = VN.lookupOrAdd(I);
VNtoScalars[{V, InvalidVN}].push_back(I);
@@ -178,7 +180,7 @@ class LoadInfo {
public:
// Insert Load and the value number of its memory address in VNtoLoads.
- void insert(LoadInst *Load, GVNPass::ValueTable &VN) {
+ void insert(LoadInst *Load, GVNValueTable &VN) {
if (Load->isSimple()) {
unsigned V = VN.lookupOrAdd(Load->getPointerOperand());
// With opaque pointers we may have loads from the same pointer with
@@ -197,7 +199,7 @@ class StoreInfo {
public:
// Insert the Store and a hash number of the store address and the stored
// value in VNtoStores.
- void insert(StoreInst *Store, GVNPass::ValueTable &VN) {
+ void insert(StoreInst *Store, GVNValueTable &VN) {
if (!Store->isSimple())
return;
// Hash the store address and the stored value.
@@ -217,7 +219,7 @@ class CallInfo {
public:
// Insert Call and its value numbering in one of the VNtoCalls* containers.
- void insert(CallInst *Call, GVNPass::ValueTable &VN) {
+ void insert(CallInst *Call, GVNValueTable &VN) {
// A call that doesNotAccessMemory is handled as a Scalar,
// onlyReadsMemory will be handled as a Load instruction,
// all other calls will be handled as stores.
@@ -260,7 +262,7 @@ class GVNHoist {
unsigned int rank(const Value *V) const;
private:
- GVNPass::ValueTable VN;
+ GVNValueTable VN;
DominatorTree *DT;
PostDominatorTree *PDT;
AliasAnalysis *AA;
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 67196ef9715f1..48782be4bbcc0 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -61,8 +61,8 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/GVNExpression.h"
+#include "llvm/Transforms/Scalar/GVNSink.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LockstepReverseIterator.h"
>From 3c4ea8c411ee80ade64b75a48b40beae90a68423 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 22 Jul 2026 11:08:35 +0100
Subject: [PATCH 2/7] [fixup] Fix formatting
---
llvm/lib/Transforms/Scalar/GVNHoist.cpp | 2 +-
llvm/lib/Transforms/Scalar/GVNSink.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index da605664df87b..756315ad3e761 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -33,6 +33,7 @@
// 2. geps when corresponding load/store cannot be hoisted.
//===----------------------------------------------------------------------===//
+#include "llvm/Transforms/Scalar/GVNHoist.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
@@ -67,7 +68,6 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar/GVNValueTable.h"
-#include "llvm/Transforms/Scalar/GVNHoist.h"
#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 48782be4bbcc0..a54538170a82a 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -33,6 +33,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/Transforms/Scalar/GVNSink.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Hashing.h"
@@ -62,7 +63,6 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar/GVNExpression.h"
-#include "llvm/Transforms/Scalar/GVNSink.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LockstepReverseIterator.h"
>From 88462d9c50637a08a0473bd57ce0b52d7f675d6f Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 22 Jul 2026 16:51:16 +0100
Subject: [PATCH 3/7] [GVN] Decouple GVNValueTable and GVNPass
`GVNValueTable` has several member functions taking a `GVNPass`
reference as an argument. This prevents moving `GVNPass`
out of `GVN.h` and into `GVN.cpp.`
The `GVNPass` reference is only used to access the `LeaderMap`. This patch moves
the `LeaderMap` type into its own header file, `GVNLeaderMap.h`, under class
name `GVNLeaderMap` and changes the `GVNValueTable` member functions to take a
`GVNLeaderMap` reference instead of a `GVNPass` reference.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 82 +-------------
.../llvm/Transforms/Scalar/GVNLeaderMap.h | 102 ++++++++++++++++++
.../llvm/Transforms/Scalar/GVNValueTable.h | 13 ++-
llvm/lib/Transforms/Scalar/GVN.cpp | 38 +++----
4 files changed, 129 insertions(+), 106 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Scalar/GVNLeaderMap.h
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 4d8a2d3eda820..f5816313bf214 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -26,6 +26,7 @@
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
+#include "llvm/Transforms/Scalar/GVNLeaderMap.h"
#include "llvm/Transforms/Scalar/GVNValueTable.h"
#include <cstdint>
@@ -145,86 +146,7 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
GVNValueTable VN;
-
- /// A mapping from value numbers to lists of Value*'s that
- /// have that value number. Use findLeader to query it.
- class LeaderMap {
- public:
- struct LeaderTableEntry {
- // Use AssertingVH here to catch dangling Value*'s in the leader table.
- // Will crash if the value gets deleted before the AssertingVH is
- // destroyed.
- AssertingVH<Value> Val;
- const BasicBlock *BB;
- LeaderTableEntry(Value *V, const BasicBlock *BB) : Val(V), BB(BB) {}
- };
-
- private:
- struct LeaderListNode {
- LeaderTableEntry Entry;
- LeaderListNode *Next;
- LeaderListNode(Value *V, const BasicBlock *BB, LeaderListNode *Next)
- : Entry(V, BB), Next(Next) {}
- };
- DenseMap<uint32_t, LeaderListNode> NumToLeaders;
- BumpPtrAllocator TableAllocator;
-
- public:
- class leader_iterator {
- const LeaderListNode *Current;
-
- public:
- using iterator_category = std::forward_iterator_tag;
- using value_type = const LeaderTableEntry;
- using difference_type = std::ptrdiff_t;
- using pointer = value_type *;
- using reference = value_type &;
-
- leader_iterator(const LeaderListNode *C) : Current(C) {}
- leader_iterator &operator++() {
- assert(Current && "Dereferenced end of leader list!");
- Current = Current->Next;
- return *this;
- }
- bool operator==(const leader_iterator &Other) const {
- return Current == Other.Current;
- }
- bool operator!=(const leader_iterator &Other) const {
- return Current != Other.Current;
- }
- reference operator*() const { return Current->Entry; }
- };
-
- iterator_range<leader_iterator> getLeaders(uint32_t N) {
- auto I = NumToLeaders.find(N);
- if (I == NumToLeaders.end()) {
- return iterator_range(leader_iterator(nullptr),
- leader_iterator(nullptr));
- }
-
- return iterator_range(leader_iterator(&I->second),
- leader_iterator(nullptr));
- }
-
- LLVM_ABI void insert(uint32_t N, Value *V, const BasicBlock *BB);
- LLVM_ABI void erase(uint32_t N, Instruction *I, const BasicBlock *BB);
- void clear() {
- // Manually destroy non-head nodes (in BumpPtrAllocator) to properly
- // clean up AssertingVH handles before Reset(). Head nodes are destroyed
- // by NumToLeaders.clear() below.
- for (auto &[_, HeadNode] : NumToLeaders) {
- LeaderListNode *N = HeadNode.Next;
- while (N) {
- auto *Next = N->Next;
- N->~LeaderListNode();
- N = Next;
- }
- }
- NumToLeaders.clear();
- TableAllocator.Reset();
- }
- };
- LeaderMap LeaderTable;
+ GVNLeaderMap LeaderTable;
// Map the block to reversed postorder traversal number. It is used to
// find back edge easily.
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNLeaderMap.h b/llvm/include/llvm/Transforms/Scalar/GVNLeaderMap.h
new file mode 100644
index 0000000000000..5b04346ece6e6
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/GVNLeaderMap.h
@@ -0,0 +1,102 @@
+//===- GVNLeaderMap.h - Leader map for GVN --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file / This file provides a data structure for mapping value numbers to
+/// lists of values that have that value number.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_GVNLEADERMAP_H
+#define LLVM_TRANSFORMS_SCALAR_GVNLEADERMAP_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Allocator.h"
+
+namespace llvm {
+
+class GVNLeaderMap {
+public:
+ struct LeaderTableEntry {
+ // Use AssertingVH here to catch dangling Value*'s in the leader table.
+ // Will crash if the value gets deleted before the AssertingVH is
+ // destroyed.
+ AssertingVH<Value> Val;
+ const BasicBlock *BB;
+ LeaderTableEntry(Value *V, const BasicBlock *BB) : Val(V), BB(BB) {}
+ };
+
+private:
+ struct LeaderListNode {
+ LeaderTableEntry Entry;
+ LeaderListNode *Next;
+ LeaderListNode(Value *V, const BasicBlock *BB, LeaderListNode *Next)
+ : Entry(V, BB), Next(Next) {}
+ };
+ DenseMap<uint32_t, LeaderListNode> NumToLeaders;
+ BumpPtrAllocator TableAllocator;
+
+public:
+ class leader_iterator {
+ const LeaderListNode *Current;
+
+ public:
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = const LeaderTableEntry;
+ using difference_type = std::ptrdiff_t;
+ using pointer = value_type *;
+ using reference = value_type &;
+
+ leader_iterator(const LeaderListNode *C) : Current(C) {}
+ leader_iterator &operator++() {
+ assert(Current && "Dereferenced end of leader list!");
+ Current = Current->Next;
+ return *this;
+ }
+ bool operator==(const leader_iterator &Other) const {
+ return Current == Other.Current;
+ }
+ bool operator!=(const leader_iterator &Other) const {
+ return Current != Other.Current;
+ }
+ reference operator*() const { return Current->Entry; }
+ };
+
+ iterator_range<leader_iterator> getLeaders(uint32_t N) {
+ auto I = NumToLeaders.find(N);
+ if (I == NumToLeaders.end()) {
+ return iterator_range(leader_iterator(nullptr), leader_iterator(nullptr));
+ }
+
+ return iterator_range(leader_iterator(&I->second),
+ leader_iterator(nullptr));
+ }
+
+ LLVM_ABI void insert(uint32_t N, Value *V, const BasicBlock *BB);
+ LLVM_ABI void erase(uint32_t N, Instruction *I, const BasicBlock *BB);
+ void clear() {
+ // Manually destroy non-head nodes (in BumpPtrAllocator) to properly
+ // clean up AssertingVH handles before Reset(). Head nodes are destroyed
+ // by NumToLeaders.clear() below.
+ for (auto &[_, HeadNode] : NumToLeaders) {
+ LeaderListNode *N = HeadNode.Next;
+ while (N) {
+ auto *Next = N->Next;
+ N->~LeaderListNode();
+ N = Next;
+ }
+ }
+ NumToLeaders.clear();
+ TableAllocator.Reset();
+ }
+};
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_GVNLEADERMAP_H
\ No newline at end of file
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h b/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
index 5d93461c1260c..9ee4b6b125643 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVNValueTable.h
@@ -26,7 +26,6 @@
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
-#include "llvm/Transforms/Scalar/GVNValueTable.h"
#include <cstdint>
#include <optional>
@@ -47,8 +46,7 @@ class EarliestEscapeAnalysis;
class ExtractValueInst;
class Function;
class FunctionPass;
-class GVNLegacyPass;
-class GVNPass;
+class GVNLeaderMap;
class GetElementPtrInst;
class ImplicitControlFlowTracking;
class LoadInst;
@@ -115,11 +113,12 @@ class GVNValueTable {
uint32_t lookupOrAddCall(CallInst *C);
uint32_t computeLoadStoreVN(Instruction *I);
uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN);
+ uint32_t Num, GVNLeaderMap &LeaderTable);
bool areCallValsEqual(uint32_t Num, uint32_t NewNum, const BasicBlock *Pred,
- const BasicBlock *PhiBlock, GVNPass &GVN);
+ const BasicBlock *PhiBlock, GVNLeaderMap &LeaderTable);
std::pair<uint32_t, bool> assignExpNewValueNum(Expression &Exp);
- bool areAllValsInBB(uint32_t Num, const BasicBlock *BB, GVNPass &GVN);
+ bool areAllValsInBB(uint32_t Num, const BasicBlock *BB,
+ GVNLeaderMap &LeaderTable);
void addMemoryStateToExp(Instruction *I, Expression &Exp);
public:
@@ -138,7 +137,7 @@ class GVNValueTable {
LLVM_ABI uint32_t lookupPtrToInt(Value *Ptr, Type *Ty);
LLVM_ABI uint32_t phiTranslate(const BasicBlock *BB,
const BasicBlock *PhiBlock, uint32_t Num,
- GVNPass &GVN);
+ GVNLeaderMap &LeaderTable);
LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
const BasicBlock &CurrBlock);
LLVM_ABI bool exists(Value *V) const;
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index d763bab153822..35e21e0b63017 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -674,7 +674,8 @@ uint32_t GVNValueTable::computeLoadStoreVN(Instruction *I) {
/// the phis in BB.
uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
const BasicBlock *PhiBlock,
- uint32_t Num, GVNPass &GVN) {
+ uint32_t Num,
+ GVNLeaderMap &LeaderTable) {
// See if we can refine the value number by looking at the PN incoming value
// for the given predecessor.
if (PHINode *PN = NumberingPhi[Num]) {
@@ -714,7 +715,7 @@ uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
// If there is any value related with Num is defined in a BB other than
// PhiBlock, it cannot depend on a phi in PhiBlock without going through
// a backedge. We can do an early exit in that case to save compile time.
- if (!areAllValsInBB(Num, PhiBlock, GVN))
+ if (!areAllValsInBB(Num, PhiBlock, LeaderTable))
return Num;
if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
@@ -729,7 +730,7 @@ uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
(I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
(I > 1 && Exp.Opcode == Instruction::ShuffleVector))
continue;
- Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN);
+ Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], LeaderTable);
}
if (Exp.Commutative) {
@@ -746,7 +747,8 @@ uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
if (uint32_t NewNum = ExpressionNumbering[Exp]) {
if (Exp.Opcode == Instruction::Call && NewNum != Num)
- return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num;
+ return areCallValsEqual(Num, NewNum, Pred, PhiBlock, LeaderTable) ? NewNum
+ : Num;
return NewNum;
}
return Num;
@@ -756,9 +758,10 @@ uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
// Return false if the result is unknown.
bool GVNValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
const BasicBlock *Pred,
- const BasicBlock *PhiBlock, GVNPass &GVN) {
+ const BasicBlock *PhiBlock,
+ GVNLeaderMap &LeaderTable) {
CallInst *Call = nullptr;
- auto Leaders = GVN.LeaderTable.getLeaders(Num);
+ auto Leaders = LeaderTable.getLeaders(Num);
for (const auto &Entry : Leaders) {
Call = dyn_cast<CallInst>(&*Entry.Val);
if (Call && Call->getParent() == PhiBlock)
@@ -804,11 +807,10 @@ std::pair<uint32_t, bool> GVNValueTable::assignExpNewValueNum(Expression &Exp) {
/// Return whether all the values related with the same \p num are
/// defined in \p BB.
bool GVNValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
- GVNPass &GVN) {
- return all_of(GVN.LeaderTable.getLeaders(Num),
- [=](const GVNPass::LeaderMap::LeaderTableEntry &L) {
- return L.BB == BB;
- });
+ GVNLeaderMap &LeaderTable) {
+ return all_of(
+ LeaderTable.getLeaders(Num),
+ [=](const GVNLeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
}
/// Include the incoming memory state into the hash of the expression for the
@@ -965,11 +967,11 @@ uint32_t GVNValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
/// Wrap phiTranslateImpl to provide caching functionality.
uint32_t GVNValueTable::phiTranslate(const BasicBlock *Pred,
const BasicBlock *PhiBlock, uint32_t Num,
- GVNPass &GVN) {
+ GVNLeaderMap &LeaderTable) {
auto FindRes = PhiTranslateTable.find({Num, Pred});
if (FindRes != PhiTranslateTable.end())
return FindRes->second;
- uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN);
+ uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, LeaderTable);
PhiTranslateTable.insert({{Num, Pred}, NewNum});
return NewNum;
}
@@ -1023,7 +1025,7 @@ void GVNValueTable::verifyRemoved(const Value *V) const {
//===----------------------------------------------------------------------===//
/// Push a new Value to the LeaderTable onto the list for its value number.
-void GVNPass::LeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
+void GVNLeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
const auto &[It, Inserted] = NumToLeaders.try_emplace(N, V, BB, nullptr);
if (!Inserted) {
// Key already exists: insert new node after the head.
@@ -1035,8 +1037,7 @@ void GVNPass::LeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
/// Scan the list of values corresponding to a given
/// value number, and remove the given instruction if encountered.
-void GVNPass::LeaderMap::erase(uint32_t N, Instruction *I,
- const BasicBlock *BB) {
+void GVNLeaderMap::erase(uint32_t N, Instruction *I, const BasicBlock *BB) {
auto It = NumToLeaders.find(N);
if (It == NumToLeaders.end())
return;
@@ -3559,8 +3560,7 @@ bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
Success = false;
break;
}
- uint32_t TValNo =
- VN.phiTranslate(Pred, Curr, VN.lookup(Op), *this);
+ uint32_t TValNo = VN.phiTranslate(Pred, Curr, VN.lookup(Op), LeaderTable);
if (Value *V = findLeader(Pred, TValNo)) {
Instr->setOperand(I, V);
} else {
@@ -3652,7 +3652,7 @@ bool GVNPass::performScalarPRE(Instruction *CurInst) {
break;
}
- uint32_t TValNo = VN.phiTranslate(P, CurrentBlock, ValNo, *this);
+ uint32_t TValNo = VN.phiTranslate(P, CurrentBlock, ValNo, LeaderTable);
Value *PredV = findLeader(P, TValNo);
if (!PredV) {
PredMap.push_back(std::make_pair(static_cast<Value *>(nullptr), P));
>From a30a4522486b2442c672118a0db61ad49263ef64 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Thu, 23 Jul 2026 11:10:42 +0100
Subject: [PATCH 4/7] [GVN] More restructuring of `GVN.h` to reduce its size
(NFC)
* `GVNPass` left as an interface for the pass manager.
Actually `GVNPass` moved under the name `GVNPassImpl` to `GVN.cpp`.
* Various helper types moved out of `GVNPassImpl` and into an anonymous
namespace in `GVN.cpp`
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 190 +---
llvm/lib/Transforms/Scalar/GVN.cpp | 1076 +++++++++++++--------
2 files changed, 656 insertions(+), 610 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index f5816313bf214..08af9e2f8eeae 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -123,45 +123,18 @@ struct GVNOptions {
///
/// FIXME: We should have a good summary of the GVN algorithm implemented by
/// this particular pass here.
+class GVNPassImpl;
class GVNPass : public OptionalPassInfoMixin<GVNPass> {
-public:
- struct AvailableValue;
- struct AvailableValueInBlock;
- struct ReachingMemVal;
- struct DependencyBlockInfo;
-
- friend class GVNValueTable;
- friend class GVNLegacyPass;
-
-private:
- GVNOptions Options;
- MemoryDependenceResults *MD = nullptr;
- DominatorTree *DT = nullptr;
- const TargetLibraryInfo *TLI = nullptr;
- AssumptionCache *AC = nullptr;
- SetVector<BasicBlock *> DeadBlocks;
- OptimizationRemarkEmitter *ORE = nullptr;
- ImplicitControlFlowTracking *ICF = nullptr;
- LoopInfo *LI = nullptr;
- AAResults *AA = nullptr;
- MemorySSAUpdater *MSSAU = nullptr;
- GVNValueTable VN;
- GVNLeaderMap LeaderTable;
-
- // Map the block to reversed postorder traversal number. It is used to
- // find back edge easily.
- DenseMap<AssertingVH<BasicBlock>, uint32_t> BlockRPONumber;
-
- // This is set 'true' initially and also when new blocks have been added to
- // the function being analyzed. This boolean is used to control the updating
- // of BlockRPONumber prior to accessing the contents of BlockRPONumber.
- bool InvalidBlockRPONumbers = true;
-
- // List of critical edges to be split between iterations.
- SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
+ std::unique_ptr<GVNPassImpl> Impl;
public:
- GVNPass(GVNOptions Options = {}) : Options(Options) {}
+ GVNPass(GVNOptions Options = {});
+ ~GVNPass();
+
+ GVNPass(const GVNPass &) = delete;
+ GVNPass(GVNPass &&);
+ GVNPass &operator=(const GVNPass &) = delete;
+ GVNPass &operator=(GVNPass &&);
/// Run the pass over the function.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
@@ -169,151 +142,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LLVM_ABI void
printPipeline(raw_ostream &OS,
function_ref<StringRef(StringRef)> MapClassName2PassName);
-
-private:
- DominatorTree &getDominatorTree() const { return *DT; }
- AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
- MemoryDependenceResults &getMemDep() const { return *MD; }
-
- bool isScalarPREEnabled() const;
- bool isLoadPREEnabled() const;
- bool isLoadInLoopPREEnabled() const;
- bool isLoadPRESplitBackedgeEnabled() const;
- bool isMemDepEnabled() const;
- bool isMemorySSAEnabled() const;
-
- using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
- using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
- using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
-
- using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
-
- /// Given a select-dependency for the load (the load address is a select of
- /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
- /// value is available by finding dominating values for both addresses. If
- /// so, the load can be rematerialized as a select of those two values.
- std::optional<AvailableValue>
- analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
- Value *FalseAddr, Instruction *From);
-
- /// Given a local dependency (Def or Clobber) determine if a value is
- /// available for the load.
- std::optional<AvailableValue>
- analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
- Value *Address);
-
- /// Given a list of non-local dependencies, determine if a value is
- /// available for the load in each specified block. If it is, add it to
- /// ValuesPerBlock. If not, add it to UnavailableBlocks.
- void analyzeLoadAvailability(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks);
-
- /// Given a critical edge from Pred to LoadBB, find a load instruction
- /// which is identical to Load from another successor of Pred.
- LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
- LoadInst *Load);
-
- /// Eliminates partially redundant \p Load, replacing it with \p
- /// AvailableLoads (connected by Phis if needed).
- void eliminatePartiallyRedundantLoad(
- LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- MapVector<BasicBlock *, Value *> &AvailableLoads,
- MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
-
- // Helper functions for d etermining load dependencies.
- std::optional<GVNPass::ReachingMemVal> scanMemoryAccessesUsers(
- const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
- const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
- BatchAAResults &AA, LoadInst *L = nullptr);
-
- std::optional<GVNPass::ReachingMemVal>
- accessMayModifyLocation(MemoryAccess *ClobberMA, const MemoryLocation &Loc,
- bool IsInvariantLoad, BasicBlock *BB, MemorySSA &MSSA,
- BatchAAResults &AA);
-
- bool collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
- MemoryAccess *ClobberMA, DependencyBlockSet &Blocks,
- SmallVectorImpl<BasicBlock *> &Worklist);
-
- void collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
- BasicBlock *BB, const DependencyBlockInfo &StartInfo,
- const DependencyBlockSet &Blocks, MemorySSA &MSSA);
-
- bool findReachingValuesForLoad(LoadInst *Inst,
- SmallVectorImpl<ReachingMemVal> &Values,
- MemorySSA &MSSA, AAResults &AA);
-
- bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks);
-
- /// Try to replace a load which executes on each loop iteraiton with Phi
- /// translation of load in preheader and load(s) in conditionally executed
- /// paths.
- bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks);
-
- // Try to eliminate redundent loades with non-local dependencies.
- bool processNonLocalLoad(LoadInst *L);
- bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
-
- /// Add any blocks determined to be unreachable by a conditional branch with a
- /// constant condition to the dead blocks.
- bool processFoldableCondBr(CondBrInst *BI);
-
- /// Propagate equalities derived from llvm.assume intrinsics.
- bool processAssumeIntrinsic(AssumeInst *II);
-
- /// Try to eliminate redundant loads.
- bool processLoad(LoadInst *L);
-
- /// Try to eliminate masked loads which have loaded from
- /// masked stores with the same mask.
- bool processMaskedLoad(IntrinsicInst *I);
-
- /// Propagate value of a condition to blocks dominated by "then" and "else"
- /// edges, as well as certains derived equalities.
- bool
- propagateEquality(Value *LHS, Value *RHS,
- const std::variant<BasicBlockEdge, Instruction *> &Root);
-
- // Pass iteration helper functions.
- bool processInstruction(Instruction *I);
- bool processBlock(BasicBlock *BB);
- bool iterateOnFunction(Function &F);
-
- // Scalar PRE helper functions
- bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
- BasicBlock *Curr, unsigned int ValNo);
- bool performScalarPRE(Instruction *I);
- bool performPRE(Function &F);
-
- /// Main entry point for the GVN pass. Also used by the GVNLegacyPass.
- bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
- const TargetLibraryInfo &RunTLI, AAResults &RunAA,
- MemoryDependenceResults *RunMD, LoopInfo &LI,
- OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
-
- // Other helper routines.
-
- Value *findLeader(const BasicBlock *BB, uint32_t Num);
- void cleanupGlobalSets();
-
- void removeInstruction(Instruction *I);
-
- /// This removes the specified instruction from
- /// our various maps and marks it for deletion.
- void salvageAndRemoveInstruction(Instruction *I);
-
- void verifyRemoved(const Instruction *I) const;
-
- bool splitCriticalEdges();
- BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
-
- void addDeadBlock(BasicBlock *BB);
- void assignValNumForDeadCode();
- void assignBlockRPONumber(Function &F);
};
/// Create a legacy GVN pass.
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 35e21e0b63017..ce39ddabfd665 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -84,9 +84,6 @@ using namespace llvm;
using namespace llvm::VNCoercion;
using namespace PatternMatch;
-using AvailableValue = GVNPass::AvailableValue;
-using AvailableValueInBlock = GVNPass::AvailableValueInBlock;
-
#define DEBUG_TYPE "gvn"
STATISTIC(NumGVNInstr, "Number of instructions deleted");
@@ -191,202 +188,6 @@ template <> struct llvm::DenseMapInfo<GVNValueTable::Expression> {
}
};
-/// Represents a particular available value that we know how to materialize.
-/// Materialization of an AvailableValue never fails. An AvailableValue is
-/// implicitly associated with a rematerialization point which is the
-/// location of the instruction from which it was formed.
-struct llvm::GVNPass::AvailableValue {
- enum class ValType {
- SimpleVal, // A simple offsetted value that is accessed.
- LoadVal, // A value produced by a load.
- MemIntrin, // A memory intrinsic which is loaded from.
- UndefVal, // A UndefValue representing a value from dead block (which
- // is not yet physically removed from the CFG).
- SelectVal, // A pointer select which is loaded from and for which the load
- // can be replace by a value select.
- };
-
- /// Val - The value that is live out of the block.
- Value *Val;
- /// Kind of the live-out value.
- ValType Kind;
-
- /// Offset - The byte offset in Val that is interesting for the load query.
- unsigned Offset = 0;
- /// V1, V2 - The dominating non-clobbered values of SelectVal.
- Value *V1 = nullptr, *V2 = nullptr;
-
- static AvailableValue get(Value *V, unsigned Offset = 0) {
- AvailableValue Res;
- Res.Val = V;
- Res.Kind = ValType::SimpleVal;
- Res.Offset = Offset;
- return Res;
- }
-
- static AvailableValue getMI(MemIntrinsic *MI, unsigned Offset = 0) {
- AvailableValue Res;
- Res.Val = MI;
- Res.Kind = ValType::MemIntrin;
- Res.Offset = Offset;
- return Res;
- }
-
- static AvailableValue getLoad(LoadInst *Load, unsigned Offset = 0) {
- AvailableValue Res;
- Res.Val = Load;
- Res.Kind = ValType::LoadVal;
- Res.Offset = Offset;
- return Res;
- }
-
- static AvailableValue getUndef() {
- AvailableValue Res;
- Res.Val = nullptr;
- Res.Kind = ValType::UndefVal;
- Res.Offset = 0;
- return Res;
- }
-
- static AvailableValue getSelect(Value *Cond, Value *V1, Value *V2) {
- AvailableValue Res;
- Res.Val = Cond;
- Res.Kind = ValType::SelectVal;
- Res.Offset = 0;
- Res.V1 = V1;
- Res.V2 = V2;
- return Res;
- }
-
- bool isSimpleValue() const { return Kind == ValType::SimpleVal; }
- bool isCoercedLoadValue() const { return Kind == ValType::LoadVal; }
- bool isMemIntrinValue() const { return Kind == ValType::MemIntrin; }
- bool isUndefValue() const { return Kind == ValType::UndefVal; }
- bool isSelectValue() const { return Kind == ValType::SelectVal; }
-
- Value *getSimpleValue() const {
- assert(isSimpleValue() && "Wrong accessor");
- return Val;
- }
-
- LoadInst *getCoercedLoadValue() const {
- assert(isCoercedLoadValue() && "Wrong accessor");
- return cast<LoadInst>(Val);
- }
-
- MemIntrinsic *getMemIntrinValue() const {
- assert(isMemIntrinValue() && "Wrong accessor");
- return cast<MemIntrinsic>(Val);
- }
-
- Value *getSelectCondition() const {
- assert(isSelectValue() && "Wrong accessor");
- return Val;
- }
-
- /// Emit code at the specified insertion point to adjust the value defined
- /// here to the specified type. This handles various coercion cases.
- Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const;
-};
-
-Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
- Instruction *InsertPt) const {
- Value *Res;
- Type *LoadTy = Load->getType();
- const DataLayout &DL = Load->getDataLayout();
- if (isSimpleValue()) {
- Res = getSimpleValue();
- if (Res->getType() != LoadTy) {
- Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
-
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
- << " " << *getSimpleValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isCoercedLoadValue()) {
- LoadInst *CoercedLoad = getCoercedLoadValue();
- if (CoercedLoad->getType() == LoadTy && Offset == 0) {
- Res = CoercedLoad;
- combineMetadataForCSE(CoercedLoad, Load, false);
- } else {
- Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
- Load->getFunction());
- // We are adding a new user for this load, for which the original
- // metadata may not hold. Additionally, the new load may have a different
- // size and type, so their metadata cannot be combined in any
- // straightforward way.
- // Drop all metadata that is not known to cause immediate UB on violation,
- // unless the load has !noundef, in which case all metadata violations
- // will be promoted to UB.
- // !noalias and !alias.scope are kept: the load is not moved and still
- // accesses the same memory, and these are independent of the load type
- // and offset, so they remain valid for the coerced result.
- if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
- CoercedLoad->dropUnknownNonDebugMetadata(
- {LLVMContext::MD_dereferenceable,
- LLVMContext::MD_dereferenceable_or_null,
- LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
- LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
- << " " << *getCoercedLoadValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- }
- } else if (isMemIntrinValue()) {
- Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, InsertPt,
- DL);
- LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
- << " " << *getMemIntrinValue() << '\n'
- << *Res << '\n'
- << "\n\n\n");
- } else if (isSelectValue()) {
- // Introduce a new value select for a load from an eligible pointer select.
- Value *Cond = getSelectCondition();
- assert(V1 && V2 && "both value operands of the select must be present");
- Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
- // We use the DebugLoc from the original load here, as this instruction
- // materializes the value that would previously have been loaded.
- cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
- } else {
- llvm_unreachable("Should not materialize value from dead block");
- }
- assert(Res && "failed to materialize?");
- return Res;
-}
-
-/// Represents an AvailableValue which can be rematerialized at the end of
-/// the associated BasicBlock.
-struct llvm::GVNPass::AvailableValueInBlock {
- /// BB - The basic block in question.
- BasicBlock *BB = nullptr;
-
- /// AV - The actual available value.
- AvailableValue AV;
-
- static AvailableValueInBlock get(BasicBlock *BB, AvailableValue &&AV) {
- AvailableValueInBlock Res;
- Res.BB = BB;
- Res.AV = std::move(AV);
- return Res;
- }
-
- static AvailableValueInBlock get(BasicBlock *BB, Value *V,
- unsigned Offset = 0) {
- return get(BB, AvailableValue::get(V, Offset));
- }
-
- static AvailableValueInBlock getUndef(BasicBlock *BB) {
- return get(BB, AvailableValue::getUndef());
- }
-
- /// Emit code at the end of this block to adjust the value defined here to
- /// the specified type. This handles various coercion cases.
- Value *MaterializeAdjustedValue(LoadInst *Load) const {
- return AV.MaterializeAdjustedValue(Load, BB->getTerminator());
- }
-};
-
//===----------------------------------------------------------------------===//
// ValueTable Internal Functions
//===----------------------------------------------------------------------===//
@@ -1013,221 +814,565 @@ void GVNValueTable::erase(Value *V) {
NumberingBB.erase(Num);
}
-/// verifyRemoved - Verify that the value is removed from all internal data
-/// structures.
-void GVNValueTable::verifyRemoved(const Value *V) const {
- assert(!ValueNumbering.contains(V) &&
- "Inst still occurs in value numbering map!");
-}
+/// verifyRemoved - Verify that the value is removed from all internal data
+/// structures.
+void GVNValueTable::verifyRemoved(const Value *V) const {
+ assert(!ValueNumbering.contains(V) &&
+ "Inst still occurs in value numbering map!");
+}
+
+//===----------------------------------------------------------------------===//
+// LeaderMap External Functions
+//===----------------------------------------------------------------------===//
+
+/// Push a new Value to the LeaderTable onto the list for its value number.
+void GVNLeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
+ const auto &[It, Inserted] = NumToLeaders.try_emplace(N, V, BB, nullptr);
+ if (!Inserted) {
+ // Key already exists: insert new node after the head.
+ auto *NewSlot = TableAllocator.Allocate<LeaderListNode>();
+ new (NewSlot) LeaderListNode(V, BB, It->second.Next);
+ It->second.Next = NewSlot;
+ }
+}
+
+/// Scan the list of values corresponding to a given
+/// value number, and remove the given instruction if encountered.
+void GVNLeaderMap::erase(uint32_t N, Instruction *I, const BasicBlock *BB) {
+ auto It = NumToLeaders.find(N);
+ if (It == NumToLeaders.end())
+ return;
+
+ LeaderListNode *Prev = nullptr;
+ LeaderListNode *Curr = &It->second;
+
+ while (Curr && (Curr->Entry.Val != I || Curr->Entry.BB != BB)) {
+ Prev = Curr;
+ Curr = Curr->Next;
+ }
+
+ if (!Curr)
+ return;
+
+ if (Prev) {
+ // Non-head node: unlink and destroy.
+ Prev->Next = Curr->Next;
+ Curr->~LeaderListNode();
+ TableAllocator.Deallocate<LeaderListNode>(Curr);
+ } else {
+ // Head node (stored by value in DenseMap).
+ if (!Curr->Next) {
+ // Only node; erase from map (DenseMap calls the destructor).
+ NumToLeaders.erase(It);
+ } else {
+ // Move second node's data into head, then destroy second node.
+ LeaderListNode *Next = Curr->Next;
+ Curr->Entry.Val = std::move(Next->Entry.Val);
+ Curr->Entry.BB = Next->Entry.BB;
+ Curr->Next = Next->Next;
+ Next->~LeaderListNode();
+ TableAllocator.Deallocate<LeaderListNode>(Next);
+ }
+ }
+}
+
+namespace {
+//===----------------------------------------------------------------------===//
+// Helper Dependency Information Classes
+//===----------------------------------------------------------------------===//
+
+enum class DepKind {
+ Other = 0, // Unknown value.
+ Def, // Exactly overlapping locations.
+ Clobber, // Reaching value superset of needed bits.
+ Select, // Reaching value is a select of two reaching addresses.
+};
+
+// Describe a memory location value, such that there exists a path to a point
+// in the program, along which that memory location is not modified.
+struct ReachingMemVal {
+ DepKind Kind;
+ BasicBlock *Block;
+ const Value *Addr;
+ Instruction *Inst;
+ int32_t Offset;
+ // For DepKind::Select only: the condition and the two addresses referenced
+ // by the "true" and "false" side of the select-dependent load.
+ const Value *SelCond = nullptr;
+ const Value *SelTrueAddr = nullptr;
+ const Value *SelFalseAddr = nullptr;
+
+ static ReachingMemVal getUnknown(BasicBlock *BB, const Value *Addr,
+ Instruction *Inst = nullptr) {
+ return {DepKind::Other, BB, Addr, Inst, -1};
+ }
+
+ static ReachingMemVal getDef(const Value *Addr, Instruction *Inst) {
+ return {DepKind::Def, Inst->getParent(), Addr, Inst, -1};
+ }
+
+ static ReachingMemVal getClobber(const Value *Addr, Instruction *Inst,
+ int32_t Offset = -1) {
+ return {DepKind::Clobber, Inst->getParent(), Addr, Inst, Offset};
+ }
+
+ static ReachingMemVal getSelect(BasicBlock *BB, const Value *Cond,
+ const Value *TrueAddr,
+ const Value *FalseAddr) {
+ return {DepKind::Select, BB, nullptr, nullptr, -1, Cond,
+ TrueAddr, FalseAddr};
+ }
+};
+
+struct DependencyBlockInfo {
+ DependencyBlockInfo() = delete;
+ DependencyBlockInfo(const PHITransAddr &Addr, MemoryAccess *ClobberMA)
+ : Addr(Addr), InitialClobberMA(ClobberMA), ClobberMA(ClobberMA),
+ ForceUnknown(false), Visited(false) {}
+ PHITransAddr Addr;
+ MemoryAccess *InitialClobberMA;
+ MemoryAccess *ClobberMA;
+ std::optional<ReachingMemVal> MemVal;
+ bool ForceUnknown : 1;
+ bool Visited : 1;
+};
+
+enum class AvailabilityState : char {
+ /// We know the block *is not* fully available. This is a fixpoint.
+ Unavailable = 0,
+ /// We know the block *is* fully available. This is a fixpoint.
+ Available = 1,
+ /// We do not know whether the block is fully available or not,
+ /// but we are currently speculating that it will be.
+ /// If it would have turned out that the block was, in fact, not fully
+ /// available, this would have been cleaned up into an Unavailable.
+ SpeculativelyAvailable = 2,
+};
+
+/// Represents a particular available value that we know how to materialize.
+/// Materialization of an AvailableValue never fails. An AvailableValue is
+/// implicitly associated with a rematerialization point which is the
+/// location of the instruction from which it was formed.
+struct AvailableValue {
+ enum class ValType {
+ SimpleVal, // A simple offsetted value that is accessed.
+ LoadVal, // A value produced by a load.
+ MemIntrin, // A memory intrinsic which is loaded from.
+ UndefVal, // A UndefValue representing a value from dead block (which
+ // is not yet physically removed from the CFG).
+ SelectVal, // A pointer select which is loaded from and for which the load
+ // can be replace by a value select.
+ };
+
+ /// Val - The value that is live out of the block.
+ Value *Val;
+ /// Kind of the live-out value.
+ ValType Kind;
+
+ /// Offset - The byte offset in Val that is interesting for the load query.
+ unsigned Offset = 0;
+ /// V1, V2 - The dominating non-clobbered values of SelectVal.
+ Value *V1 = nullptr, *V2 = nullptr;
+
+ static AvailableValue get(Value *V, unsigned Offset = 0) {
+ AvailableValue Res;
+ Res.Val = V;
+ Res.Kind = ValType::SimpleVal;
+ Res.Offset = Offset;
+ return Res;
+ }
+
+ static AvailableValue getMI(MemIntrinsic *MI, unsigned Offset = 0) {
+ AvailableValue Res;
+ Res.Val = MI;
+ Res.Kind = ValType::MemIntrin;
+ Res.Offset = Offset;
+ return Res;
+ }
+
+ static AvailableValue getLoad(LoadInst *Load, unsigned Offset = 0) {
+ AvailableValue Res;
+ Res.Val = Load;
+ Res.Kind = ValType::LoadVal;
+ Res.Offset = Offset;
+ return Res;
+ }
+
+ static AvailableValue getUndef() {
+ AvailableValue Res;
+ Res.Val = nullptr;
+ Res.Kind = ValType::UndefVal;
+ Res.Offset = 0;
+ return Res;
+ }
+
+ static AvailableValue getSelect(Value *Cond, Value *V1, Value *V2) {
+ AvailableValue Res;
+ Res.Val = Cond;
+ Res.Kind = ValType::SelectVal;
+ Res.Offset = 0;
+ Res.V1 = V1;
+ Res.V2 = V2;
+ return Res;
+ }
+
+ bool isSimpleValue() const { return Kind == ValType::SimpleVal; }
+ bool isCoercedLoadValue() const { return Kind == ValType::LoadVal; }
+ bool isMemIntrinValue() const { return Kind == ValType::MemIntrin; }
+ bool isUndefValue() const { return Kind == ValType::UndefVal; }
+ bool isSelectValue() const { return Kind == ValType::SelectVal; }
+
+ Value *getSimpleValue() const {
+ assert(isSimpleValue() && "Wrong accessor");
+ return Val;
+ }
+
+ LoadInst *getCoercedLoadValue() const {
+ assert(isCoercedLoadValue() && "Wrong accessor");
+ return cast<LoadInst>(Val);
+ }
+
+ MemIntrinsic *getMemIntrinValue() const {
+ assert(isMemIntrinValue() && "Wrong accessor");
+ return cast<MemIntrinsic>(Val);
+ }
+
+ Value *getSelectCondition() const {
+ assert(isSelectValue() && "Wrong accessor");
+ return Val;
+ }
+
+ /// Emit code at the specified insertion point to adjust the value defined
+ /// here to the specified type. This handles various coercion cases.
+ Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const;
+};
+
+Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
+ Instruction *InsertPt) const {
+ Value *Res;
+ Type *LoadTy = Load->getType();
+ const DataLayout &DL = Load->getDataLayout();
+ if (isSimpleValue()) {
+ Res = getSimpleValue();
+ if (Res->getType() != LoadTy) {
+ Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
+
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
+ << " " << *getSimpleValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isCoercedLoadValue()) {
+ LoadInst *CoercedLoad = getCoercedLoadValue();
+ if (CoercedLoad->getType() == LoadTy && Offset == 0) {
+ Res = CoercedLoad;
+ combineMetadataForCSE(CoercedLoad, Load, false);
+ } else {
+ Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
+ Load->getFunction());
+ // We are adding a new user for this load, for which the original
+ // metadata may not hold. Additionally, the new load may have a different
+ // size and type, so their metadata cannot be combined in any
+ // straightforward way.
+ // Drop all metadata that is not known to cause immediate UB on violation,
+ // unless the load has !noundef, in which case all metadata violations
+ // will be promoted to UB.
+ // !noalias and !alias.scope are kept: the load is not moved and still
+ // accesses the same memory, and these are independent of the load type
+ // and offset, so they remain valid for the coerced result.
+ if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
+ CoercedLoad->dropUnknownNonDebugMetadata(
+ {LLVMContext::MD_dereferenceable,
+ LLVMContext::MD_dereferenceable_or_null,
+ LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
+ LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
+ << " " << *getCoercedLoadValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ }
+ } else if (isMemIntrinValue()) {
+ Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, InsertPt,
+ DL);
+ LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
+ << " " << *getMemIntrinValue() << '\n'
+ << *Res << '\n'
+ << "\n\n\n");
+ } else if (isSelectValue()) {
+ // Introduce a new value select for a load from an eligible pointer select.
+ Value *Cond = getSelectCondition();
+ assert(V1 && V2 && "both value operands of the select must be present");
+ Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator());
+ // We use the DebugLoc from the original load here, as this instruction
+ // materializes the value that would previously have been loaded.
+ cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
+ } else {
+ llvm_unreachable("Should not materialize value from dead block");
+ }
+ assert(Res && "failed to materialize?");
+ return Res;
+}
+
+/// Represents an AvailableValue which can be rematerialized at the end of
+/// the associated BasicBlock.
+struct AvailableValueInBlock {
+ /// BB - The basic block in question.
+ BasicBlock *BB = nullptr;
-//===----------------------------------------------------------------------===//
-// LeaderMap External Functions
-//===----------------------------------------------------------------------===//
+ /// AV - The actual available value.
+ AvailableValue AV;
-/// Push a new Value to the LeaderTable onto the list for its value number.
-void GVNLeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
- const auto &[It, Inserted] = NumToLeaders.try_emplace(N, V, BB, nullptr);
- if (!Inserted) {
- // Key already exists: insert new node after the head.
- auto *NewSlot = TableAllocator.Allocate<LeaderListNode>();
- new (NewSlot) LeaderListNode(V, BB, It->second.Next);
- It->second.Next = NewSlot;
+ static AvailableValueInBlock get(BasicBlock *BB, AvailableValue &&AV) {
+ AvailableValueInBlock Res;
+ Res.BB = BB;
+ Res.AV = std::move(AV);
+ return Res;
}
-}
-
-/// Scan the list of values corresponding to a given
-/// value number, and remove the given instruction if encountered.
-void GVNLeaderMap::erase(uint32_t N, Instruction *I, const BasicBlock *BB) {
- auto It = NumToLeaders.find(N);
- if (It == NumToLeaders.end())
- return;
-
- LeaderListNode *Prev = nullptr;
- LeaderListNode *Curr = &It->second;
- while (Curr && (Curr->Entry.Val != I || Curr->Entry.BB != BB)) {
- Prev = Curr;
- Curr = Curr->Next;
+ static AvailableValueInBlock get(BasicBlock *BB, Value *V,
+ unsigned Offset = 0) {
+ return get(BB, AvailableValue::get(V, Offset));
}
- if (!Curr)
- return;
+ static AvailableValueInBlock getUndef(BasicBlock *BB) {
+ return get(BB, AvailableValue::getUndef());
+ }
- if (Prev) {
- // Non-head node: unlink and destroy.
- Prev->Next = Curr->Next;
- Curr->~LeaderListNode();
- TableAllocator.Deallocate<LeaderListNode>(Curr);
- } else {
- // Head node (stored by value in DenseMap).
- if (!Curr->Next) {
- // Only node; erase from map (DenseMap calls the destructor).
- NumToLeaders.erase(It);
- } else {
- // Move second node's data into head, then destroy second node.
- LeaderListNode *Next = Curr->Next;
- Curr->Entry.Val = std::move(Next->Entry.Val);
- Curr->Entry.BB = Next->Entry.BB;
- Curr->Next = Next->Next;
- Next->~LeaderListNode();
- TableAllocator.Deallocate<LeaderListNode>(Next);
- }
+ /// Emit code at the end of this block to adjust the value defined here to
+ /// the specified type. This handles various coercion cases.
+ Value *MaterializeAdjustedValue(LoadInst *Load) const {
+ return AV.MaterializeAdjustedValue(Load, BB->getTerminator());
}
-}
+};
+
+} // namespace
//===----------------------------------------------------------------------===//
-// Helper Dependency Information Classes
+// GVN Pass
//===----------------------------------------------------------------------===//
-enum class DepKind {
- Other = 0, // Unknown value.
- Def, // Exactly overlapping locations.
- Clobber, // Reaching value superset of needed bits.
- Select, // Reaching value is a select of two reaching addresses.
-};
+/// The core GVN pass object.
+///
+/// FIXME: We should have a good summary of the GVN algorithm implemented by
+/// this particular pass here.
+class llvm::GVNPassImpl {
+public:
+ friend class GVNValueTable;
+ friend class GVNLegacyPass;
-// Describe a memory location value, such that there exists a path to a point
-// in the program, along which that memory location is not modified.
-struct GVNPass::ReachingMemVal {
- DepKind Kind;
- BasicBlock *Block;
- const Value *Addr;
- Instruction *Inst;
- int32_t Offset;
- // For DepKind::Select only: the condition and the two addresses referenced
- // by the "true" and "false" side of the select-dependent load.
- const Value *SelCond = nullptr;
- const Value *SelTrueAddr = nullptr;
- const Value *SelFalseAddr = nullptr;
+private:
+ GVNOptions Options;
+ MemoryDependenceResults *MD = nullptr;
+ DominatorTree *DT = nullptr;
+ const TargetLibraryInfo *TLI = nullptr;
+ AssumptionCache *AC = nullptr;
+ SetVector<BasicBlock *> DeadBlocks;
+ OptimizationRemarkEmitter *ORE = nullptr;
+ ImplicitControlFlowTracking *ICF = nullptr;
+ LoopInfo *LI = nullptr;
+ AAResults *AA = nullptr;
+ MemorySSAUpdater *MSSAU = nullptr;
+ GVNValueTable VN;
+ GVNLeaderMap LeaderTable;
+
+ // Map the block to reversed postorder traversal number. It is used to
+ // find back edge easily.
+ DenseMap<AssertingVH<BasicBlock>, uint32_t> BlockRPONumber;
+
+ // This is set 'true' initially and also when new blocks have been added to
+ // the function being analyzed. This boolean is used to control the updating
+ // of BlockRPONumber prior to accessing the contents of BlockRPONumber.
+ bool InvalidBlockRPONumbers = true;
+
+ // List of critical edges to be split between iterations.
+ SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
- static ReachingMemVal getUnknown(BasicBlock *BB, const Value *Addr,
- Instruction *Inst = nullptr) {
- return {DepKind::Other, BB, Addr, Inst, -1};
- }
+public:
+ GVNPassImpl(GVNOptions Options = {}) : Options(Options) {}
- static ReachingMemVal getDef(const Value *Addr, Instruction *Inst) {
- return {DepKind::Def, Inst->getParent(), Addr, Inst, -1};
- }
+ void printPipeline(raw_ostream &OS,
+ function_ref<StringRef(StringRef)> MapClassName2PassName);
- static ReachingMemVal getClobber(const Value *Addr, Instruction *Inst,
- int32_t Offset = -1) {
- return {DepKind::Clobber, Inst->getParent(), Addr, Inst, Offset};
- }
+ bool isScalarPREEnabled() const;
+ bool isLoadPREEnabled() const;
+ bool isLoadInLoopPREEnabled() const;
+ bool isLoadPRESplitBackedgeEnabled() const;
+ bool isMemDepEnabled() const;
+ bool isMemorySSAEnabled() const;
- static ReachingMemVal getSelect(BasicBlock *BB, const Value *Cond,
- const Value *TrueAddr,
- const Value *FalseAddr) {
- return {DepKind::Select, BB, nullptr, nullptr, -1, Cond,
- TrueAddr, FalseAddr};
+ /// Main entry point for the GVN pass. Also used by the GVNLegacyPass.
+ bool run(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+ const TargetLibraryInfo &RunTLI, AAResults &RunAA,
+ MemoryDependenceResults *RunMD, LoopInfo &LI,
+ OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
+
+private:
+ DominatorTree &getDominatorTree() const { return *DT; }
+ AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
+ MemoryDependenceResults &getMemDep() const { return *MD; }
+
+ using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
+ using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
+ using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
+
+ using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
+
+ /// Given a select-dependency for the load (the load address is a select of
+ /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a
+ /// value is available by finding dominating values for both addresses. If
+ /// so, the load can be rematerialized as a select of those two values.
+ std::optional<AvailableValue>
+ analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
+ Value *FalseAddr, Instruction *From);
+
+ /// Given a local dependency (Def or Clobber) determine if a value is
+ /// available for the load.
+ std::optional<AvailableValue>
+ analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ Value *Address);
+
+ /// Given a list of non-local dependencies, determine if a value is
+ /// available for the load in each specified block. If it is, add it to
+ /// ValuesPerBlock. If not, add it to UnavailableBlocks.
+ void analyzeLoadAvailability(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks);
+
+ /// Given a critical edge from Pred to LoadBB, find a load instruction
+ /// which is identical to Load from another successor of Pred.
+ LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
+ LoadInst *Load);
+
+ /// Eliminates partially redundant \p Load, replacing it with \p
+ /// AvailableLoads (connected by Phis if needed).
+ void eliminatePartiallyRedundantLoad(
+ LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ MapVector<BasicBlock *, Value *> &AvailableLoads,
+ MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
+
+ // Helper functions for d etermining load dependencies.
+ std::optional<ReachingMemVal> scanMemoryAccessesUsers(
+ const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
+ const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
+ BatchAAResults &AA, LoadInst *L = nullptr);
+
+ std::optional<ReachingMemVal>
+ accessMayModifyLocation(MemoryAccess *ClobberMA, const MemoryLocation &Loc,
+ bool IsInvariantLoad, BasicBlock *BB, MemorySSA &MSSA,
+ BatchAAResults &AA);
+
+ bool collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
+ MemoryAccess *ClobberMA, DependencyBlockSet &Blocks,
+ SmallVectorImpl<BasicBlock *> &Worklist);
+
+ void collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
+ BasicBlock *BB, const DependencyBlockInfo &StartInfo,
+ const DependencyBlockSet &Blocks, MemorySSA &MSSA);
+
+ bool findReachingValuesForLoad(LoadInst *Inst,
+ SmallVectorImpl<ReachingMemVal> &Values,
+ MemorySSA &MSSA, AAResults &AA);
+
+ bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks);
+
+ /// Try to replace a load which executes on each loop iteraiton with Phi
+ /// translation of load in preheader and load(s) in conditionally executed
+ /// paths.
+ bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks);
+
+ // Try to eliminate redundent loades with non-local dependencies.
+ bool processNonLocalLoad(LoadInst *L);
+ bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
+
+ /// Add any blocks determined to be unreachable by a conditional branch with a
+ /// constant condition to the dead blocks.
+ bool processFoldableCondBr(CondBrInst *BI);
+
+ /// Propagate equalities derived from llvm.assume intrinsics.
+ bool processAssumeIntrinsic(AssumeInst *II);
+
+ /// Try to eliminate redundant loads.
+ bool processLoad(LoadInst *L);
+
+ /// Try to eliminate masked loads which have loaded from
+ /// masked stores with the same mask.
+ bool processMaskedLoad(IntrinsicInst *I);
+
+ /// Propagate value of a condition to blocks dominated by "then" and "else"
+ /// edges, as well as certains derived equalities.
+ bool
+ propagateEquality(Value *LHS, Value *RHS,
+ const std::variant<BasicBlockEdge, Instruction *> &Root);
+
+ // Pass iteration helper functions.
+ bool processInstruction(Instruction *I);
+ bool processBlock(BasicBlock *BB);
+ bool iterateOnFunction(Function &F);
+
+ // Scalar PRE helper functions
+ bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
+ BasicBlock *Curr, unsigned int ValNo);
+ bool performScalarPRE(Instruction *I);
+ bool performPRE(Function &F);
+
+ // Other helper routines.
+
+ Value *findLeader(const BasicBlock *BB, uint32_t Num);
+
+ /// Return whether all the values related with the same \p num are
+ /// defined in \p BB.
+ bool areAllValsInBB(uint32_t Num, const BasicBlock *BB) {
+ return all_of(
+ LeaderTable.getLeaders(Num),
+ [=](const GVNLeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
}
-};
-struct GVNPass::DependencyBlockInfo {
- DependencyBlockInfo() = delete;
- DependencyBlockInfo(const PHITransAddr &Addr, MemoryAccess *ClobberMA)
- : Addr(Addr), InitialClobberMA(ClobberMA), ClobberMA(ClobberMA),
- ForceUnknown(false), Visited(false) {}
- PHITransAddr Addr;
- MemoryAccess *InitialClobberMA;
- MemoryAccess *ClobberMA;
- std::optional<ReachingMemVal> MemVal;
- bool ForceUnknown : 1;
- bool Visited : 1;
-};
+ void cleanupGlobalSets();
-//===----------------------------------------------------------------------===//
-// GVN Pass
-//===----------------------------------------------------------------------===//
+ void removeInstruction(Instruction *I);
-PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
- // FIXME: The order of evaluation of these 'getResult' calls is very
- // significant! Re-ordering these variables will cause GVN when run alone to
- // be less effective! We should fix memdep and basic-aa to not exhibit this
- // behavior, but until then don't change the order here.
- auto &AC = AM.getResult<AssumptionAnalysis>(F);
- auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
- auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
- auto &AA = AM.getResult<AAManager>(F);
- auto *MemDep =
- isMemDepEnabled() ? &AM.getResult<MemoryDependenceAnalysis>(F) : nullptr;
- auto &LI = AM.getResult<LoopAnalysis>(F);
- auto *MSSA = AM.getCachedResult<MemorySSAAnalysis>(F);
- if (isMemorySSAEnabled() && !MSSA) {
- assert(!MemDep &&
- "On-demand computation of MemSSA implies that MemDep is disabled!");
- MSSA = &AM.getResult<MemorySSAAnalysis>(F);
- }
- auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
- bool Changed = runImpl(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
- MSSA ? &MSSA->getMSSA() : nullptr);
- if (!Changed)
- return PreservedAnalyses::all();
- PreservedAnalyses PA;
- PA.preserve<DominatorTreeAnalysis>();
- PA.preserve<TargetLibraryAnalysis>();
- if (MSSA)
- PA.preserve<MemorySSAAnalysis>();
- PA.preserve<LoopAnalysis>();
- return PA;
-}
+ /// This removes the specified instruction from
+ /// our various maps and marks it for deletion.
+ void salvageAndRemoveInstruction(Instruction *I);
-void GVNPass::printPipeline(
- raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
- static_cast<PassInfoMixin<GVNPass> *>(this)->printPipeline(
- OS, MapClassName2PassName);
+ void verifyRemoved(const Instruction *I) const;
- OS << '<';
- if (Options.AllowScalarPRE != std::nullopt)
- OS << (*Options.AllowScalarPRE ? "" : "no-") << "scalar-pre;";
- if (Options.AllowLoadPRE != std::nullopt)
- OS << (*Options.AllowLoadPRE ? "" : "no-") << "load-pre;";
- if (Options.AllowLoadPRESplitBackedge != std::nullopt)
- OS << (*Options.AllowLoadPRESplitBackedge ? "" : "no-")
- << "split-backedge-load-pre;";
- if (Options.AllowMemDep != std::nullopt)
- OS << (*Options.AllowMemDep ? "" : "no-") << "memdep;";
- if (Options.AllowMemorySSA != std::nullopt)
- OS << (*Options.AllowMemorySSA ? "" : "no-") << "memoryssa";
- OS << '>';
-}
+ bool splitCriticalEdges();
+ BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
+
+ void addDeadBlock(BasicBlock *BB);
+ void assignValNumForDeadCode();
+ void assignBlockRPONumber(Function &F);
+};
-bool GVNPass::isScalarPREEnabled() const {
+bool GVNPassImpl::isScalarPREEnabled() const {
return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE);
}
-bool GVNPass::isLoadPREEnabled() const {
+bool GVNPassImpl::isLoadPREEnabled() const {
return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE);
}
-bool GVNPass::isLoadInLoopPREEnabled() const {
+bool GVNPassImpl::isLoadInLoopPREEnabled() const {
return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE);
}
-bool GVNPass::isLoadPRESplitBackedgeEnabled() const {
+bool GVNPassImpl::isLoadPRESplitBackedgeEnabled() const {
return Options.AllowLoadPRESplitBackedge.value_or(
GVNEnableSplitBackedgeInLoadPRE);
}
-bool GVNPass::isMemDepEnabled() const {
+bool GVNPassImpl::isMemDepEnabled() const {
return Options.AllowMemDep.value_or(GVNEnableMemDep);
}
-bool GVNPass::isMemorySSAEnabled() const {
+bool GVNPassImpl::isMemorySSAEnabled() const {
return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA);
}
-enum class AvailabilityState : char {
- /// We know the block *is not* fully available. This is a fixpoint.
- Unavailable = 0,
- /// We know the block *is* fully available. This is a fixpoint.
- Available = 1,
- /// We do not know whether the block is fully available or not,
- /// but we are currently speculating that it will be.
- /// If it would have turned out that the block was, in fact, not fully
- /// available, this would have been cleaned up into an Unavailable.
- SpeculativelyAvailable = 2,
-};
-
/// Return true if we can prove that the value
/// we're analyzing is fully available in the specified block. As we go, keep
/// track of which blocks we know are fully alive in FullyAvailableBlocks. This
@@ -1534,8 +1679,9 @@ static Value *findDominatingValue(const MemoryLocation &Loc, Type *LoadTy,
}
std::optional<AvailableValue>
-GVNPass::analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
- Value *FalseAddr, Instruction *From) {
+GVNPassImpl::analyzeSelectAvailability(LoadInst *Load, Value *Cond,
+ Value *TrueAddr, Value *FalseAddr,
+ Instruction *From) {
assert(TrueAddr->getType() == Load->getPointerOperandType() &&
"Invalid address type of true side of select dependency");
assert(FalseAddr->getType() == Load->getPointerOperandType() &&
@@ -1556,8 +1702,8 @@ GVNPass::analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr,
}
std::optional<AvailableValue>
-GVNPass::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
- Value *Address) {
+GVNPassImpl::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
+ Value *Address) {
assert(Load->isUnordered() && "rules below are incorrect for ordered access");
assert((Dep.Kind == DepKind::Def || Dep.Kind == DepKind::Clobber) &&
"expected a local dependence");
@@ -1698,10 +1844,10 @@ GVNPass::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
return std::nullopt;
}
-void GVNPass::analyzeLoadAvailability(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
+void GVNPassImpl::analyzeLoadAvailability(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
// Filter out useless results (non-locals, etc). Keep track of the blocks
// where we have a value available in repl, also keep track of whether we see
// dependencies that produce an unknown value for the load (such as a call
@@ -1775,8 +1921,9 @@ void GVNPass::analyzeLoadAvailability(LoadInst *Load,
/// v2 = load %addr
/// ...
///
-LoadInst *GVNPass::findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
- LoadInst *Load) {
+LoadInst *GVNPassImpl::findLoadToHoistIntoPred(BasicBlock *Pred,
+ BasicBlock *LoadBB,
+ LoadInst *Load) {
// For simplicity we handle a Pred has 2 successors only.
auto *Term = Pred->getTerminator();
if (Term->getNumSuccessors() != 2 || Term->isSpecialTerminator())
@@ -1825,7 +1972,7 @@ LoadInst *GVNPass::findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
return nullptr;
}
-void GVNPass::eliminatePartiallyRedundantLoad(
+void GVNPassImpl::eliminatePartiallyRedundantLoad(
LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
MapVector<BasicBlock *, Value *> &AvailableLoads,
MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad) {
@@ -2010,7 +2157,7 @@ maybeLoadStoreLocation(Instruction *I, bool AllowStores,
/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
/// looking for memory reads whose location aliases `Loc` and dominates our
/// load.
-std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
+std::optional<ReachingMemVal> GVNPassImpl::scanMemoryAccessesUsers(
const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
BatchAAResults &AA, LoadInst *L) {
@@ -2084,7 +2231,7 @@ std::optional<GVNPass::ReachingMemVal> GVNPass::scanMemoryAccessesUsers(
/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
/// given location. Returns a ReachingMemVal describing the dependency.
-std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
+std::optional<ReachingMemVal> GVNPassImpl::accessMayModifyLocation(
MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad,
BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
assert(ClobberMA->getBlock() == BB);
@@ -2185,10 +2332,10 @@ std::optional<GVNPass::ReachingMemVal> GVNPass::accessMayModifyLocation(
/// Collect the predecessors of block, while doing phi-translation of the memory
/// address and the memory clobber. Return false if the block should be marked
/// as clobbering the memory location in an unknown way.
-bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
- MemoryAccess *ClobberMA,
- DependencyBlockSet &Blocks,
- SmallVectorImpl<BasicBlock *> &Worklist) {
+bool GVNPassImpl::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
+ MemoryAccess *ClobberMA,
+ DependencyBlockSet &Blocks,
+ SmallVectorImpl<BasicBlock *> &Worklist) {
if (Addr.needsPHITranslationFromBlock(BB) &&
!Addr.isPotentiallyPHITranslatable())
return false;
@@ -2243,11 +2390,11 @@ bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
/// walk the use-def chain to the final clobber. If the chain extends beyond
/// `BB`, continue into that block but only if it is in the previously collected
/// set.
-void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
- BasicBlock *BB,
- const DependencyBlockInfo &StartInfo,
- const DependencyBlockSet &Blocks,
- MemorySSA &MSSA) {
+void GVNPassImpl::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
+ BasicBlock *BB,
+ const DependencyBlockInfo &StartInfo,
+ const DependencyBlockSet &Blocks,
+ MemorySSA &MSSA) {
MemoryAccess *MA = StartInfo.InitialClobberMA;
MemoryAccess *LastMA = StartInfo.ClobberMA;
@@ -2293,9 +2440,9 @@ void GVNPass::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
/// * Other: we know which block defines the memory location in some way, but
/// could not identify a precise instruction (e.g., memory already live at
/// function entry).
-bool GVNPass::findReachingValuesForLoad(LoadInst *L,
- SmallVectorImpl<ReachingMemVal> &Values,
- MemorySSA &MSSA, AAResults &AAR) {
+bool GVNPassImpl::findReachingValuesForLoad(
+ LoadInst *L, SmallVectorImpl<ReachingMemVal> &Values, MemorySSA &MSSA,
+ AAResults &AAR) {
EarliestEscapeAnalysis EA(*DT, LI);
BatchAAResults AA(AAR, &EA);
BasicBlock *StartBlock = L->getParent();
@@ -2470,8 +2617,9 @@ bool GVNPass::findReachingValuesForLoad(LoadInst *L,
return true;
}
-bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
+bool GVNPassImpl::performLoadPRE(LoadInst *Load,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
// Okay, we have *some* definitions of the value. This means that the value
// is available in some of our (transitive) predecessors. Lets think about
// doing PRE of this load. This will involve inserting a new load into the
@@ -2723,9 +2871,9 @@ bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
return true;
}
-bool GVNPass::performLoopLoadPRE(LoadInst *Load,
- AvailValInBlkVect &ValuesPerBlock,
- UnavailBlkVect &UnavailableBlocks) {
+bool GVNPassImpl::performLoopLoadPRE(LoadInst *Load,
+ AvailValInBlkVect &ValuesPerBlock,
+ UnavailBlkVect &UnavailableBlocks) {
const Loop *L = LI->getLoopFor(Load->getParent());
// TODO: Generalize to other loop blocks that dominate the latch.
if (!L || L->getHeader() != Load->getParent())
@@ -2802,7 +2950,7 @@ bool GVNPass::performLoopLoadPRE(LoadInst *Load,
/// Attempt to eliminate a load whose dependencies are
/// non-local by performing PHI construction.
-bool GVNPass::processNonLocalLoad(LoadInst *Load) {
+bool GVNPassImpl::processNonLocalLoad(LoadInst *Load) {
// Non-local speculations are not allowed under asan.
if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
@@ -2845,8 +2993,8 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load) {
return processNonLocalLoad(Load, MemVals);
}
-bool GVNPass::processNonLocalLoad(LoadInst *Load,
- SmallVectorImpl<ReachingMemVal> &Deps) {
+bool GVNPassImpl::processNonLocalLoad(LoadInst *Load,
+ SmallVectorImpl<ReachingMemVal> &Deps) {
// If we had a phi translation failure, we'll have a single entry which is a
// clobber in the current block. Reject this early.
if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
@@ -2922,7 +3070,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load,
/// Attempt to eliminate a load, first by eliminating it
/// locally, and then attempting non-local elimination if that fails.
-bool GVNPass::processLoad(LoadInst *L) {
+bool GVNPassImpl::processLoad(LoadInst *L) {
if (!MD && !isMemorySSAEnabled())
return false;
@@ -2996,7 +3144,7 @@ bool GVNPass::processLoad(LoadInst *L) {
// Attempt to process masked loads which have loaded from
// masked stores with the same mask
-bool GVNPass::processMaskedLoad(IntrinsicInst *I) {
+bool GVNPassImpl::processMaskedLoad(IntrinsicInst *I) {
if (!MD)
return false;
MemDepResult Dep = MD->getDependency(I);
@@ -3036,7 +3184,7 @@ bool GVNPass::processMaskedLoad(IntrinsicInst *I) {
// dead blocks with "UndefVal" in an hope these PHIs will optimized away.
//
// Return true iff *NEW* dead code are found.
-bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
+bool GVNPassImpl::processFoldableCondBr(CondBrInst *BI) {
// If a branch has two identical successors, we cannot declare either dead.
if (BI->getSuccessor(0) == BI->getSuccessor(1))
return false;
@@ -3057,7 +3205,7 @@ bool GVNPass::processFoldableCondBr(CondBrInst *BI) {
return true;
}
-bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
+bool GVNPassImpl::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
Value *V = IntrinsicI->getArgOperand(0);
if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
@@ -3139,7 +3287,7 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E,
/// 'RHS' everywhere in the scope. Returns whether a change was made.
/// The Root may either be a basic block edge (for conditions) or an
/// instruction (for assumes).
-bool GVNPass::propagateEquality(
+bool GVNPassImpl::propagateEquality(
Value *LHS, Value *RHS,
const std::variant<BasicBlockEdge, Instruction *> &Root) {
SmallVector<std::pair<Value*, Value*>, 4> Worklist;
@@ -3348,7 +3496,7 @@ static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
/// When calculating availability, handle an instruction
/// by inserting it into the appropriate sets.
-bool GVNPass::processInstruction(Instruction *I) {
+bool GVNPassImpl::processInstruction(Instruction *I) {
// If the instruction can be easily simplified then do so now in preference
// to value numbering it. Value numbering often exposes redundancies, for
// example if it determines that %y is equal to %x then the instruction
@@ -3503,7 +3651,7 @@ bool GVNPass::processInstruction(Instruction *I) {
return true;
}
-bool GVNPass::processBlock(BasicBlock *BB) {
+bool GVNPassImpl::processBlock(BasicBlock *BB) {
if (DeadBlocks.count(BB))
return false;
@@ -3524,7 +3672,7 @@ bool GVNPass::processBlock(BasicBlock *BB) {
}
/// Executes one iteration of GVN.
-bool GVNPass::iterateOnFunction(Function &F) {
+bool GVNPassImpl::iterateOnFunction(Function &F) {
cleanupGlobalSets();
// Top-down walk of the dominator tree.
@@ -3541,8 +3689,9 @@ bool GVNPass::iterateOnFunction(Function &F) {
}
// Instantiate an expression in a predecessor that lacked it.
-bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
- BasicBlock *Curr, unsigned int ValNo) {
+bool GVNPassImpl::performScalarPREInsertion(Instruction *Instr,
+ BasicBlock *Pred, BasicBlock *Curr,
+ unsigned int ValNo) {
// Because we are going top-down through the block, all value numbers
// will be available in the predecessor by the time we need them. Any
// that weren't originally present will have been instantiated earlier
@@ -3589,7 +3738,7 @@ bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
return true;
}
-bool GVNPass::performScalarPRE(Instruction *CurInst) {
+bool GVNPassImpl::performScalarPRE(Instruction *CurInst) {
if (isa<AllocaInst>(CurInst) || CurInst->isTerminator() ||
isa<PHINode>(CurInst) || CurInst->getType()->isVoidTy() ||
CurInst->mayReadFromMemory() || CurInst->mayHaveSideEffects() ||
@@ -3751,7 +3900,7 @@ bool GVNPass::performScalarPRE(Instruction *CurInst) {
/// Perform a purely local form of PRE that looks for diamond
/// control flow patterns and attempts to perform simple PRE at the join point.
-bool GVNPass::performPRE(Function &F) {
+bool GVNPassImpl::performPRE(Function &F) {
bool Changed = false;
for (BasicBlock *CurrentBlock : depth_first(&F.getEntryBlock())) {
// Nothing to PRE in the entry block.
@@ -3776,8 +3925,26 @@ bool GVNPass::performPRE(Function &F) {
return Changed;
}
+void GVNPassImpl::printPipeline(
+ raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
+
+ OS << '<';
+ if (Options.AllowScalarPRE != std::nullopt)
+ OS << (*Options.AllowScalarPRE ? "" : "no-") << "scalar-pre;";
+ if (Options.AllowLoadPRE != std::nullopt)
+ OS << (*Options.AllowLoadPRE ? "" : "no-") << "load-pre;";
+ if (Options.AllowLoadPRESplitBackedge != std::nullopt)
+ OS << (*Options.AllowLoadPRESplitBackedge ? "" : "no-")
+ << "split-backedge-load-pre;";
+ if (Options.AllowMemDep != std::nullopt)
+ OS << (*Options.AllowMemDep ? "" : "no-") << "memdep;";
+ if (Options.AllowMemorySSA != std::nullopt)
+ OS << (*Options.AllowMemorySSA ? "" : "no-") << "memoryssa";
+ OS << '>';
+}
+
/// runOnFunction - This is the main transformation entry point for a function.
-bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
+bool GVNPassImpl::run(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
const TargetLibraryInfo &RunTLI, AAResults &RunAA,
MemoryDependenceResults *RunMD, LoopInfo &LI,
OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
@@ -3857,7 +4024,7 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
// and then scan the list to find one whose block dominates the block in
// question. This is fast because dominator tree queries consist of only
// a few comparisons of DFS numbers.
-Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
+Value *GVNPassImpl::findLeader(const BasicBlock *BB, uint32_t Num) {
auto Leaders = LeaderTable.getLeaders(Num);
if (Leaders.empty())
return nullptr;
@@ -3874,7 +4041,7 @@ Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) {
return Val;
}
-void GVNPass::cleanupGlobalSets() {
+void GVNPassImpl::cleanupGlobalSets() {
VN.clear();
LeaderTable.clear();
BlockRPONumber.clear();
@@ -3882,7 +4049,7 @@ void GVNPass::cleanupGlobalSets() {
InvalidBlockRPONumbers = true;
}
-void GVNPass::removeInstruction(Instruction *I) {
+void GVNPassImpl::removeInstruction(Instruction *I) {
VN.erase(I);
if (MD) MD->removeInstruction(I);
if (MSSAU)
@@ -3895,7 +4062,7 @@ void GVNPass::removeInstruction(Instruction *I) {
++NumGVNInstr;
}
-void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
+void GVNPassImpl::salvageAndRemoveInstruction(Instruction *I) {
salvageKnowledge(I, AC);
salvageDebugInfo(*I);
removeInstruction(I);
@@ -3903,13 +4070,13 @@ void GVNPass::salvageAndRemoveInstruction(Instruction *I) {
/// Verify that the specified instruction does not occur in our
/// internal data structures.
-void GVNPass::verifyRemoved(const Instruction *Inst) const {
+void GVNPassImpl::verifyRemoved(const Instruction *Inst) const {
VN.verifyRemoved(Inst);
}
/// Split critical edges found during the previous
/// iteration that may enable further optimization.
-bool GVNPass::splitCriticalEdges() {
+bool GVNPassImpl::splitCriticalEdges() {
if (ToSplit.empty())
return false;
@@ -3930,7 +4097,8 @@ bool GVNPass::splitCriticalEdges() {
/// Split the critical edge connecting the given two blocks, and return
/// the block inserted to the critical edge.
-BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
+BasicBlock *GVNPassImpl::splitCriticalEdges(BasicBlock *Pred,
+ BasicBlock *Succ) {
// GVN does not require loop-simplify, do not try to preserve it if it is not
// possible.
BasicBlock *BB = SplitCriticalEdge(
@@ -3948,7 +4116,7 @@ BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
/// function is to add all these blocks to "DeadBlocks". For the dead blocks'
/// live successors, update their phi nodes by replacing the operands
/// corresponding to dead blocks with UndefVal.
-void GVNPass::addDeadBlock(BasicBlock *BB) {
+void GVNPassImpl::addDeadBlock(BasicBlock *BB) {
SmallVector<BasicBlock *, 4> NewDead;
SmallSetVector<BasicBlock *, 4> DF;
@@ -4027,7 +4195,7 @@ void GVNPass::addDeadBlock(BasicBlock *BB) {
// associated val-num. As it normally has far more live instructions than dead
// instructions, it makes more sense just to "fabricate" a val-number for the
// dead code than checking if instruction involved is dead or not.
-void GVNPass::assignValNumForDeadCode() {
+void GVNPassImpl::assignValNumForDeadCode() {
for (BasicBlock *BB : DeadBlocks) {
for (Instruction &Inst : *BB) {
unsigned ValNum = VN.lookupOrAdd(&Inst);
@@ -4036,7 +4204,7 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-void GVNPass::assignBlockRPONumber(Function &F) {
+void GVNPassImpl::assignBlockRPONumber(Function &F) {
BlockRPONumber.clear();
uint32_t NextBlockNumber = 1;
ReversePostOrderTraversal<Function *> RPOT(&F);
@@ -4045,6 +4213,56 @@ void GVNPass::assignBlockRPONumber(Function &F) {
InvalidBlockRPONumbers = false;
}
+GVNPass::GVNPass(GVNOptions Options)
+ : Impl(std::make_unique<GVNPassImpl>(Options)) {}
+
+GVNPass::~GVNPass() = default;
+
+GVNPass::GVNPass(GVNPass &&) = default;
+
+GVNPass &GVNPass::operator=(GVNPass &&) = default;
+
+PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
+ // FIXME: The order of evaluation of these 'getResult' calls is very
+ // significant! Re-ordering these variables will cause GVN when run alone to
+ // be less effective! We should fix memdep and basic-aa to not exhibit this
+ // behavior, but until then don't change the order here.
+ auto &AC = AM.getResult<AssumptionAnalysis>(F);
+ auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+ auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
+ auto &AA = AM.getResult<AAManager>(F);
+ auto *MemDep = Impl->isMemDepEnabled()
+ ? &AM.getResult<MemoryDependenceAnalysis>(F)
+ : nullptr;
+ auto &LI = AM.getResult<LoopAnalysis>(F);
+ auto *MSSA = AM.getCachedResult<MemorySSAAnalysis>(F);
+ if (Impl->isMemorySSAEnabled() && !MSSA) {
+ assert(!MemDep &&
+ "On-demand computation of MemSSA implies that MemDep is disabled!");
+ MSSA = &AM.getResult<MemorySSAAnalysis>(F);
+ }
+ auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ bool Changed = Impl->run(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
+ MSSA ? &MSSA->getMSSA() : nullptr);
+ if (!Changed)
+ return PreservedAnalyses::all();
+ PreservedAnalyses PA;
+ PA.preserve<DominatorTreeAnalysis>();
+ PA.preserve<TargetLibraryAnalysis>();
+ if (MSSA)
+ PA.preserve<MemorySSAAnalysis>();
+ PA.preserve<LoopAnalysis>();
+ return PA;
+}
+
+void GVNPass::printPipeline(
+ raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
+ static_cast<PassInfoMixin<GVNPass> *>(this)->printPipeline(
+ OS, MapClassName2PassName);
+
+ Impl->printPipeline(OS, MapClassName2PassName);
+}
+
class llvm::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
@@ -4067,7 +4285,7 @@ class llvm::GVNLegacyPass : public FunctionPass {
if (Impl.isMemorySSAEnabled() && !MSSAWP)
MSSAWP = &getAnalysis<MemorySSAWrapperPass>();
- return Impl.runImpl(
+ return Impl.run(
F, getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
@@ -4099,7 +4317,7 @@ class llvm::GVNLegacyPass : public FunctionPass {
}
private:
- GVNPass Impl;
+ GVNPassImpl Impl;
};
char GVNLegacyPass::ID = 0;
>From e9a91581b2748b6b51fa7e8bdf5f430fc81da053 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 15 Jul 2026 15:41:09 +0100
Subject: [PATCH 5/7] [GVN] Assign unique VNs to calls with operand bundles
Call instructions with operand bundles may be assigned the same value number,
even if operand bundles differ. The GVN may eliminate one of the calls in favour
of another and drop one of the operand bundles.
Work around this by assigning unique value numbers to calls with operand
bundles.
---
llvm/lib/Transforms/Scalar/GVN.cpp | 6 ++++++
.../Transforms/GVN/operand-bundle-unique-vn.ll | 18 ++++++++++++++++++
2 files changed, 24 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index ce39ddabfd665..60d3f46bd5e45 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -341,6 +341,12 @@ uint32_t GVNValueTable::lookupOrAddCall(CallInst *C) {
return NextValueNumber++;
}
+ // Conservatively assign unique value numbers to calls with operand bundles.
+ if (C->hasOperandBundles()) {
+ ValueNumbering[C] = NextValueNumber;
+ return NextValueNumber++;
+ }
+
if (AA->doesNotAccessMemory(C)) {
Expression Exp = createExpr(C);
uint32_t E = assignExpNewValueNum(Exp).first;
diff --git a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
new file mode 100644
index 0000000000000..d027a875310f9
--- /dev/null
+++ b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -p gvn < %s | FileCheck %s
+
+; Check GVN does not eliminate the second call bacause of operand bundle presence.
+
+define i32 @f(ptr %p) {
+; CHECK-LABEL: define i32 @f(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1:[0-9]+]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: ret i32 [[V]]
+;
+ %u = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ %v = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ ret i32 %v
+}
+
+declare void @g(i1) nounwind willreturn
>From 6a7bde880ec9a227fe1f33b3baa03edcd371bdb5 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Tue, 21 Jul 2026 17:38:27 +0100
Subject: [PATCH 6/7] [fixup] Correct test to use a distinct operand bundle tag
for the second call
---
llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
index d027a875310f9..78876c709d505 100644
--- a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
+++ b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
@@ -7,11 +7,11 @@ define i32 @f(ptr %p) {
; CHECK-LABEL: define i32 @f(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1:[0-9]+]] [ "foo"(ptr [[P]]) ]
-; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "bar"(ptr [[P]]) ]
; CHECK-NEXT: ret i32 [[V]]
;
%u = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
- %v = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ %v = call i32 @g(i1 true) memory(none) ["bar"(ptr %p)]
ret i32 %v
}
>From f84302ff9d59edba980fec8b96a468a8b7fa32bb Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Wed, 22 Jul 2026 11:00:11 +0100
Subject: [PATCH 7/7] [fixup] Update tests
---
llvm/lib/Transforms/Scalar/GVN.cpp | 2 +
.../GVN/operand-bundle-unique-vn.ll | 62 +++++++++++++++++--
2 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 60d3f46bd5e45..845f9dc3011ba 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -342,6 +342,8 @@ uint32_t GVNValueTable::lookupOrAddCall(CallInst *C) {
}
// Conservatively assign unique value numbers to calls with operand bundles.
+ // TODO: Bundle names could be included in the value numbering expression to
+ // allow combining calls with identical bundles.
if (C->hasOperandBundles()) {
ValueNumbering[C] = NextValueNumber;
return NextValueNumber++;
diff --git a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
index 78876c709d505..b79512d84e93c 100644
--- a/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
+++ b/llvm/test/Transforms/GVN/operand-bundle-unique-vn.ll
@@ -1,18 +1,70 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -p gvn < %s | FileCheck %s
-; Check GVN does not eliminate the second call bacause of operand bundle presence.
-define i32 @f(ptr %p) {
-; CHECK-LABEL: define i32 @f(
+; Check that the calls are not CSEd because they have different operand bundle
+; tags.
+define i32 @no_drop_bundle(ptr %p) {
+; CHECK-LABEL: define i32 @no_drop_bundle(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1:[0-9]+]] [ "foo"(ptr [[P]]) ]
; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "bar"(ptr [[P]]) ]
-; CHECK-NEXT: ret i32 [[V]]
+; CHECK-NEXT: [[W:%.*]] = add i32 [[U]], [[V]]
+; CHECK-NEXT: ret i32 [[W]]
;
%u = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
%v = call i32 @g(i1 true) memory(none) ["bar"(ptr %p)]
- ret i32 %v
+ %w = add i32 %u, %v
+ ret i32 %w
+}
+
+; Check that the calls are not CSEd because they have different operand bundle
+; arguments.
+define i32 @diff_args(ptr %p, ptr %q) {
+; CHECK-LABEL: define i32 @diff_args(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "foo"(ptr [[Q]]) ]
+; CHECK-NEXT: [[W:%.*]] = add i32 [[U]], [[V]]
+; CHECK-NEXT: ret i32 [[W]]
+;
+ %u = call i32 @g(i1 true) memory(none) ["foo"(ptr %p)]
+ %v = call i32 @g(i1 true) memory(none) ["foo"(ptr %q)]
+ %w = add i32 %u, %v
+ ret i32 %w
+}
+
+; Check that the calls are not CSEd because they access memory in unknown ways.
+define i32 @mem_access(ptr %p) {
+; CHECK-LABEL: define i32 @mem_access(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) [ "foo"(ptr [[P]]) ]
+; CHECK-NEXT: [[W:%.*]] = add i32 [[U]], [[V]]
+; CHECK-NEXT: ret i32 [[W]]
+;
+ %u = call i32 @g(i1 true) ["foo"(ptr %p)]
+ %v = call i32 @g(i1 true) ["foo"(ptr %p)]
+ %w = add i32 %u, %v
+ ret i32 %w
+}
+
+; Check that the calls are CSEd because of the conservative treatment of operand
+; bundles.
+; TODO: Perhaps this can be made more precise by including the bundle name in
+; the value numbering expression.
+define i32 @no_mem_access(ptr %p) {
+; CHECK-LABEL: define i32 @no_mem_access(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[U:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "align"(ptr [[P]], i32 8) ]
+; CHECK-NEXT: [[V:%.*]] = call i32 @g(i1 true) #[[ATTR1]] [ "align"(ptr [[P]], i32 8) ]
+; CHECK-NEXT: [[W:%.*]] = add i32 [[U]], [[V]]
+; CHECK-NEXT: ret i32 [[W]]
+;
+ %u = call i32 @g(i1 true) memory(none) ["align"(ptr %p, i32 8)]
+ %v = call i32 @g(i1 true) memory(none) ["align"(ptr %p, i32 8)]
+ %w = add i32 %u, %v
+ ret i32 %w
}
declare void @g(i1) nounwind willreturn
More information about the llvm-branch-commits
mailing list