[llvm] [NewGVN][2/3] Load coercion between loads that have live-on-entry definitions (PR #68666)
Konstantina Mitropoulou via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 9 23:38:49 PDT 2023
https://github.com/kmitropoulou created https://github.com/llvm/llvm-project/pull/68666
In the following example, both %V1 and %V2 have live-on-entry definitions and
their memory locations are overlapping. After load coercion the value of %V2
is extracted from %V1 and the uses of %V2 are updated accordingly.
```
Before load coercion
BB1
%V1 = load <2 x i32>, ptr %P, align 1
%V2 = load i32, ptr %P, align 1
%V3 = add i32 %V2, 42
After load coercion
BB1
%V1 = load <2 x i32>, ptr %P, align 1
%0 = bitcast <2 x i32> %V1 to i64
%1 = trunc i64 %0 to i32
%V3 = add i32 %1, 42
```
>From 78ca18546aa9af1c333fbe7f9b9bd03e0fa36b02 Mon Sep 17 00:00:00 2001
From: Konstantina Mitropoulou <Konstantina.Mitropoulou at amd.com>
Date: Mon, 9 Oct 2023 21:05:14 -0700
Subject: [PATCH 1/2] [NewGVN][1/3] Load coercion between load and store
Load coercion consists of two phases:
1. Collection of the load instructions that can be optiimized with load coercion.
We collect pairs of candidate load and its depending instructions. The candidate
load is the laod that will be eliminated by the value that we will extract from
the depending instruction. The reason that we can eliminate the candidate load is
because its memory location overlaps with the memory location of the depending
instruction.
For example, in the following snippet, the candidate load is %V2 and the
depending instruction is the store.
```
Beofre load coercion After load coercion
BB1: BB1:
store i32 100, ptr %P store i32 100, ptr %P
%V1 = ... => %V1 = ...
%V2 = load i32, ptr %P %V3 = add i32 %V1, 100
%V3 = add i32 %V1, %V2
```
2. Code generation for load coercion: This phase updatest the IR by eliminating
the candidate load and by updating its uses.
This patch implements load coercion between a load candidate and a store
depending instruction. The follow-up patches implement load coercion support for
instructions that have live on entry definitions and MemoryPhi definitions.
---
llvm/lib/Transforms/Scalar/NewGVN.cpp | 478 ++++++++++++++++--
.../load_coercion_between_store_and_load.ll | 341 +++++++++++++
llvm/test/Transforms/NewGVN/pr14166-xfail.ll | 1 -
3 files changed, 780 insertions(+), 40 deletions(-)
create mode 100644 llvm/test/Transforms/NewGVN/load_coercion_between_store_and_load.ll
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 19ac9526b5f88b6..f4c84bd747bb417 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -76,6 +76,7 @@
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Argument.h"
@@ -154,6 +155,10 @@ static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true),
cl::Hidden);
+// Enables load coercion for non-constant values.
+static cl::opt<bool> EnableLoadCoercion("enable-load-coercion", cl::init(true),
+ cl::Hidden);
+
//===----------------------------------------------------------------------===//
// GVN Pass
//===----------------------------------------------------------------------===//
@@ -653,6 +658,16 @@ class NewGVN {
// Deletion info.
SmallPtrSet<Instruction *, 8> InstructionsToErase;
+ // Map candidate load to their depending instructions.
+ mutable std::map<Value *, DenseSet<std::pair<Instruction *, BasicBlock *>>>
+ LoadCoercion;
+
+ // Keep newly generated loads.
+ SmallVector<Instruction *, 2> NewLoadsInLoadCoercion;
+
+ // Keep newly generated instructions.
+ SmallVector<Instruction *, 2> NewlyGeneratedInsns;
+
public:
NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
@@ -776,9 +791,9 @@ class NewGVN {
ExprResult checkExprResults(Expression *, Instruction *, Value *) const;
ExprResult performSymbolicEvaluation(Instruction *,
SmallPtrSetImpl<Value *> &) const;
- const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
- Instruction *,
- MemoryAccess *) const;
+ const Expression *createLoadExpAndUpdateMemUses(LoadInst *, Value *,
+ MemoryAccess *,
+ MemoryAccess *) const;
const Expression *performSymbolicLoadEvaluation(Instruction *) const;
const Expression *performSymbolicStoreEvaluation(Instruction *) const;
ExprResult performSymbolicCallEvaluation(Instruction *) const;
@@ -853,6 +868,7 @@ class NewGVN {
// Utilities.
void cleanupTables();
std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
+ unsigned updateDFSNumbers(unsigned);
void updateProcessedCount(const Value *V);
void verifyMemoryCongruency() const;
void verifyIterationSettled(Function &F);
@@ -893,6 +909,37 @@ class NewGVN {
// Debug counter info. When verifying, we have to reset the value numbering
// debug counter to the same state it started in to get the same results.
int64_t StartingVNCounter = 0;
+
+ // The following functions are used in load coercion:
+ // Try to add the load along with the depending instruction(s) in
+ // LoadCoercion map.
+ bool tryAddLoadDepInsnIntoLoadCoercionMap(LoadInst *, Instruction *,
+ BasicBlock *) const;
+ // Collect the load instructions that can be optimized with load coercion.
+ // The filtering of the load instructions is based the type of their memory
+ // access.
+ bool performSymbolicLoadCoercionForNonConstantMemoryDef(LoadInst *,
+ StoreInst *,
+ MemoryAccess *) const;
+ const Expression *performSymbolicLoadCoercionForConstantMemoryDef(
+ Type *, Value *, LoadInst *, Instruction *, MemoryAccess *) const;
+ // Code generation for load coercion. Replaces the load with the right
+ // instruction or the right sequence of instructions.
+ bool implementLoadCoercion();
+ // Update MemorySSA with the load instructions that are emitted during load
+ // coercion.
+ void updateMemorySSA(Instruction *, Instruction *);
+ // Extract the value that will replace the load from the depending
+ // instruction.
+ Value *getExtractedValue(LoadInst *, Instruction *);
+ // If load coercion is successful, the uses of the optimized load might need
+ // to be added to new congruence classes in order to optimize the code
+ // further. For this reason, we run value numbering for all the uses of the
+ // optimized load. If load coercion has failed, then we need to add the load
+ // (and its uses) to the right congruence class.
+ void updateUsesAfterLoadCoercionImpl(LoadInst *,
+ SmallVectorImpl<Instruction *> &);
+ void updateUsesAfterLoadCoercion(LoadInst *, Value *);
};
} // end anonymous namespace
@@ -1439,12 +1486,122 @@ const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
return createStoreExpression(SI, StoreAccess);
}
+// A load can have one or more dependencies as the following examples show:
+//
+// Example 1:
+// BB1:
+// ...
+// store i32 %V1, ptr %P
+// ...
+// %V2 = load i32, ptr %P
+// ...
+//
+// Example 2:
+// BB1: BB2:
+// store i32 %V1, ptr %P %V2 = load i32, ptr %P
+// br label %BB3 br label %BB3
+// \ /
+// BB3:
+// %V3 = load i32, ptr %P
+//
+// In the first example, the load (%V2) has only one dependency. In the second
+// example, the load (%V3) has two dependencies. Therefore, we add the load
+// along with its two dependencies in LoadCoercion map. However, this is not
+// always the case as it is shown below:
+//
+// Example 3:
+// BB1:
+// %V1 = load <4 x i32>, ptr %P
+// br i1 %cond, label %BB2, label %BB3
+// / \
+// BB2: BB3:
+// %V2 = load <2 x i32>, ptr %P %V3 = load i32, ptr %P
+// br label %BB4 br label %BB4
+// \ /
+// BB4:
+// %V4 = load i32, ptr %P
+//
+// The %V4 load can be optimized by any of the loads (%V1, %V2, %V3). The loads
+// %V2 and %V3 can also be optimized by %V1. For this reason, we need to do an
+// extra check before we add the load in the map. Hence, we check if the load is
+// already in the map and if the existing depending instruction dominates the
+// current depending instruction. If so, then we do not add the new depending
+// instruction in LoadCoercion map. If the current depending instruction
+// dominates the existing depending instruction, then we remove the existing
+// depending instruction from LoadCoercion map and we add the current depending
+// instruction. In Example 3, the %V4 load has only one dependency (%V1) and we
+// add only this one in LoadCoercion map.
+bool NewGVN::tryAddLoadDepInsnIntoLoadCoercionMap(
+ LoadInst *LI, Instruction *CurrentDepI, BasicBlock *CurrentDepIBB) const {
+ // Can't forward from non-atomic to atomic without violating memory model.
+ if (LI->isAtomic() > CurrentDepI->isAtomic())
+ return false;
+
+ if (auto *DepLI = dyn_cast<LoadInst>(CurrentDepI))
+ if (LI->getAlign() < DepLI->getAlign())
+ return false;
+
+ if (auto *DepSI = dyn_cast<StoreInst>(CurrentDepI))
+ if (LI->getAlign() < DepSI->getAlign())
+ return false;
+
+ // Check if LI already exists in LoadCoercion map.
+ auto It = LoadCoercion.find(LI);
+ if (It != LoadCoercion.end()) {
+ auto &ExistingDepInsns = It->second;
+ // Iterate over all the existing depending instructions of LI.
+ for (auto &P : llvm::make_early_inc_range(ExistingDepInsns)) {
+ Instruction *ExistingDepI = P.first;
+ if (MSSAWalker->getClobberingMemoryAccess(getMemoryAccess(CurrentDepI)) ==
+ MSSAWalker->getClobberingMemoryAccess(
+ getMemoryAccess(ExistingDepI)) &&
+ isa<LoadInst>(ExistingDepI) && isa<LoadInst>(CurrentDepI)) {
+ // If the existing depending instruction dominates the current depending
+ // instruction, then we should not add the current depending instruction
+ // in LoadCoercion map (Example 3).
+ if (DT->dominates(ExistingDepI, CurrentDepI))
+ return true;
+
+ // If the current depending instruction dominates the existing one, then
+ // we remove the existing depending instruction from the LoadCoercion
+ // map. Next, we add the current depending instruction in LoadCoercion
+ // map.
+ if (DT->dominates(CurrentDepI, ExistingDepI))
+ ExistingDepInsns.erase(P);
+ }
+ }
+ }
+ // Add the load and the corresponding depending instruction in LoadCoercion
+ // map.
+ LoadCoercion[LI].insert(std::make_pair(CurrentDepI, CurrentDepIBB));
+ return true;
+}
+
+// Find load coercion opportunities between load (LI) and store instructions
+// (DepSI).
+bool NewGVN::performSymbolicLoadCoercionForNonConstantMemoryDef(
+ LoadInst *LI, StoreInst *DepSI, MemoryAccess *DefiningAccess) const {
+ Type *LoadType = LI->getType();
+ bool IsLoadCoercionCandidate = false;
+ if (LI->isAtomic() > DepSI->isAtomic() ||
+ LoadType == DepSI->getValueOperand()->getType())
+ return false;
+
+ int Offset = analyzeLoadFromClobberingStore(
+ LoadType, lookupOperandLeader(LI->getPointerOperand()), DepSI, DL);
+ if (Offset >= 0) {
+ IsLoadCoercionCandidate |=
+ tryAddLoadDepInsnIntoLoadCoercionMap(LI, DepSI, DepSI->getParent());
+ }
+
+ return IsLoadCoercionCandidate;
+}
+
// See if we can extract the value of a loaded pointer from a load, a store, or
// a memory instruction.
-const Expression *
-NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
- LoadInst *LI, Instruction *DepInst,
- MemoryAccess *DefiningAccess) const {
+const Expression *NewGVN::performSymbolicLoadCoercionForConstantMemoryDef(
+ Type *LoadType, Value *LoadPtr, LoadInst *LI, Instruction *DepInst,
+ MemoryAccess *DefiningAccess) const {
assert((!LI || LI->isSimple()) && "Not a simple load");
if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
// Can't forward from non-atomic to atomic without violating memory model.
@@ -1464,21 +1621,6 @@ NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
}
}
}
- } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) {
- // Can't forward from non-atomic to atomic without violating memory model.
- if (LI->isAtomic() > DepLI->isAtomic())
- return nullptr;
- int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
- if (Offset >= 0) {
- // We can coerce a constant load into a load.
- if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
- if (auto *PossibleConstant =
- getConstantValueForLoad(C, Offset, LoadType, DL)) {
- LLVM_DEBUG(dbgs() << "Coercing load from load " << *LI
- << " to constant " << *PossibleConstant << "\n");
- return createConstantExpression(PossibleConstant);
- }
- }
} else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
if (Offset >= 0) {
@@ -1510,11 +1652,24 @@ NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
return createConstantExpression(UndefValue::get(LoadType));
} else if (auto *InitVal =
getInitialValueOfAllocation(DepInst, TLI, LoadType))
- return createConstantExpression(InitVal);
+ return createConstantExpression(InitVal);
return nullptr;
}
+const Expression *
+NewGVN::createLoadExpAndUpdateMemUses(LoadInst *LI, Value *LoadAddressLeader,
+ MemoryAccess *OriginalAccess,
+ MemoryAccess *DefiningAccess) const {
+ const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,
+ DefiningAccess);
+ // If our MemoryLeader is not our defining access, add a use to the
+ // MemoryLeader, so that we get reprocessed when it changes.
+ if (LE->getMemoryLeader() != DefiningAccess)
+ addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);
+ return LE;
+}
+
const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
auto *LI = cast<LoadInst>(I);
@@ -1531,6 +1686,22 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
MemoryAccess *DefiningAccess =
MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
+ // Do not apply load coercion to load instructions that are candidates of
+ // phi-of-ops optimization.
+ if (TempToBlock.count(LI))
+ return createLoadExpAndUpdateMemUses(LI, LoadAddressLeader, OriginalAccess,
+ DefiningAccess);
+
+ // Do not apply load coercion to load isntructions that are generated during
+ // load coercion.
+ auto It = llvm::find(NewLoadsInLoadCoercion, LI);
+ if (It != NewLoadsInLoadCoercion.end())
+ return createLoadExpAndUpdateMemUses(LI, LoadAddressLeader, OriginalAccess,
+ DefiningAccess);
+
+ // Check if we can apply load coercion.
+ bool IsLoadCoercionCandidate = false;
+
if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
Instruction *DefiningInst = MD->getMemoryInst();
@@ -1542,19 +1713,34 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
// certain memory operations that cause the memory to have a fixed value
// (IE things like calloc).
if (const auto *CoercionResult =
- performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
- DefiningInst, DefiningAccess))
+ performSymbolicLoadCoercionForConstantMemoryDef(
+ LI->getType(), LoadAddressLeader, LI, DefiningInst,
+ DefiningAccess))
return CoercionResult;
+
+ if (EnableLoadCoercion) {
+ if (auto *DepSI = dyn_cast<StoreInst>(DefiningInst)) {
+ if (!isa<Constant>(lookupOperandLeader(DepSI->getValueOperand()))) {
+ IsLoadCoercionCandidate =
+ performSymbolicLoadCoercionForNonConstantMemoryDef(
+ LI, DepSI, DefiningAccess);
+ }
+ }
+ }
}
}
- const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,
- DefiningAccess);
- // If our MemoryLeader is not our defining access, add a use to the
- // MemoryLeader, so that we get reprocessed when it changes.
- if (LE->getMemoryLeader() != DefiningAccess)
- addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);
- return LE;
+ // If LI is a candidate for load coercion, then we do not create a load
+ // expression and we remove it from PHINodeUses which keeps the candidates of
+ // phi-of-ops optimization.
+ if (EnableLoadCoercion && IsLoadCoercionCandidate) {
+ if (PHINodeUses.count(LI))
+ const_cast<NewGVN *>(this)->PHINodeUses.erase(LI);
+ return nullptr;
+ }
+ // Otherwise, we create a load expression.
+ return createLoadExpAndUpdateMemUses(LI, LoadAddressLeader, OriginalAccess,
+ DefiningAccess);
}
NewGVN::ExprResult
@@ -3021,6 +3207,17 @@ std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
return std::make_pair(Start, End);
}
+unsigned NewGVN::updateDFSNumbers(unsigned ICount) {
+ // Now a standard depth first ordering of the domtree is equivalent to RPO.
+ for (auto DTN : depth_first(DT->getRootNode())) {
+ BasicBlock *B = DTN->getBlock();
+ const auto &BlockRange = assignDFSNumbers(B, ICount);
+ BlockInstRange.insert({B, BlockRange});
+ ICount += BlockRange.second - BlockRange.first;
+ }
+ return ICount;
+}
+
void NewGVN::updateProcessedCount(const Value *V) {
#ifndef NDEBUG
if (ProcessedCount.count(V) == 0) {
@@ -3458,13 +3655,7 @@ bool NewGVN::runGVN() {
});
}
- // Now a standard depth first ordering of the domtree is equivalent to RPO.
- for (auto *DTN : depth_first(DT->getRootNode())) {
- BasicBlock *B = DTN->getBlock();
- const auto &BlockRange = assignDFSNumbers(B, ICount);
- BlockInstRange.insert({B, BlockRange});
- ICount += BlockRange.second - BlockRange.first;
- }
+ ICount = updateDFSNumbers(ICount);
initializeCongruenceClasses(F);
TouchedInstructions.resize(ICount);
@@ -3485,6 +3676,15 @@ bool NewGVN::runGVN() {
verifyIterationSettled(F);
verifyStoreExpressions();
+ if (EnableLoadCoercion && implementLoadCoercion()) {
+ // Update the newly generated instructions with the correct DFS numbers.
+ // TODO: Update DFS numbers faster.
+ InstrDFS.clear();
+ DFSToInstr.clear();
+ RevisitOnReachabilityChange.clear();
+ ICount = updateDFSNumbers(0);
+ }
+
Changed |= eliminateInstructions(F);
// Delete all instructions marked for deletion.
@@ -3821,6 +4021,206 @@ Value *NewGVN::findPHIOfOpsLeader(const Expression *E,
return nullptr;
}
+// Update MemorySSA for the newly emitted load instruction.
+void NewGVN::updateMemorySSA(Instruction *LoadToOptimize,
+ Instruction *NewLoad) {
+ MemorySSAUpdater MemSSAUpdater(MSSA);
+ MemoryAccess *DefiningAccess = MSSA->getLiveOnEntryDef();
+ MemoryAccess *NewAccess = MemSSAUpdater.createMemoryAccessInBB(
+ NewLoad, DefiningAccess, NewLoad->getParent(),
+ MemorySSA::BeforeTerminator);
+ if (auto *NewDef = dyn_cast<MemoryDef>(NewAccess))
+ MemSSAUpdater.insertDef(NewDef, /*RenameUses=*/true);
+ else
+ MemSSAUpdater.insertUse(cast<MemoryUse>(NewAccess),
+ /*RenameUses=*/true);
+}
+
+// Extract the correct value from the depending instruction.
+Value *NewGVN::getExtractedValue(LoadInst *LI, Instruction *DepI) {
+
+ Type *LoadTy = LI->getType();
+ Value *NewValue = nullptr;
+ Instruction *InsertPtr = nullptr;
+ // Emit the instructions that extract the coerced value from the depending
+ // instruction.
+ if (auto *Store = dyn_cast<StoreInst>(DepI)) {
+ int Offset = analyzeLoadFromClobberingStore(LoadTy, LI->getPointerOperand(),
+ Store, DL);
+ InsertPtr = Store->getNextNode();
+ NewValue = getValueForLoad(Store->getValueOperand(), Offset, LoadTy,
+ InsertPtr, DL);
+ } else if (LoadInst *Load = dyn_cast<LoadInst>(DepI)) {
+ int Offset = analyzeLoadFromClobberingLoad(LoadTy, LI->getPointerOperand(),
+ Load, DL);
+ InsertPtr = Load->getNextNode();
+ NewValue = getValueForLoad(Load, Offset, LoadTy, InsertPtr, DL);
+ }
+
+ // Get the newly generated instructions and add them to NewLoadsInLoadCoercion
+ // and NewlyGeneratedInsns.
+ if (!isa<Constant>(NewValue) && !isa<Argument>(NewValue))
+ for (Instruction *CurInsn = DepI->getNextNode(); CurInsn != InsertPtr;
+ CurInsn = CurInsn->getNextNode()) {
+ if (LoadInst *NewLI = dyn_cast<LoadInst>(CurInsn)) {
+ updateMemorySSA(LI, NewLI);
+ NewLoadsInLoadCoercion.push_back(LI);
+ }
+ NewlyGeneratedInsns.push_back(CurInsn);
+ }
+
+ return NewValue;
+}
+
+void NewGVN::updateUsesAfterLoadCoercionImpl(
+ LoadInst *LI, SmallVectorImpl<Instruction *> &LIUses) {
+ // Run value numbering for the users of the candidate load instruction.
+ while (!LIUses.empty()) {
+ Instruction *I = LIUses.front();
+ assert(I != LI &&
+ "Vanity check that we do not process the optimized load.\n");
+ LIUses.erase(&*(LIUses.begin()));
+ if (InstructionsToErase.count(I))
+ continue;
+ CongruenceClass *OrigClass = ValueToClass.lookup(I);
+ valueNumberInstruction(I);
+ updateProcessedCount(I);
+ CongruenceClass *NewClass = ValueToClass.lookup(I);
+ if (OrigClass != NewClass) {
+ for (auto *User : I->users()) {
+ auto *UserI = cast<Instruction>(User);
+ LIUses.push_back(UserI);
+ }
+ if (auto *NewLI = dyn_cast<LoadInst>(I))
+ if (MemoryAccess *MA = getMemoryAccess(NewLI))
+ for (auto *MAU : MA->users()) {
+ Instruction *MAUInst = cast<Instruction>(MAU);
+ LIUses.push_back(MAUInst);
+ }
+ auto It = PredicateToUsers.find(I);
+ if (It != PredicateToUsers.end())
+ for (auto *PredI : PredicateToUsers[I]) {
+ LIUses.push_back(PredI);
+ }
+ }
+ }
+}
+
+void NewGVN::updateUsesAfterLoadCoercion(LoadInst *LI, Value *NewValue) {
+ SmallVector<Instruction *, 2> LIUses;
+ if (!NewValue) {
+ Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
+ MemoryAccess *OriginalAccess = getMemoryAccess(LI);
+ MemoryAccess *DefiningAccess =
+ MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
+ const Expression *Symbolized = createLoadExpAndUpdateMemUses(
+ LI, LoadAddressLeader, OriginalAccess, DefiningAccess);
+ performCongruenceFinding(LI, Symbolized);
+ for (Use &U : LI->uses())
+ LIUses.push_back(cast<Instruction>(U.getUser()));
+ updateUsesAfterLoadCoercionImpl(LI, LIUses);
+ } else {
+ CongruenceClass *LIClass = ValueToClass.lookup(LI);
+ InstructionsToErase.insert(LI);
+ LI->replaceAllUsesWith(NewValue);
+ // Collect the instructions for which we have to run value numbering.
+ // These are all the instructions that are in the same congruence class as
+ // LI and all the newly generated instructions.
+ for (auto *Member : *LIClass) {
+ auto *I = cast<Instruction>(Member);
+ if (LI == I)
+ continue;
+ LIUses.push_back(I);
+ }
+ for (Instruction *I : NewlyGeneratedInsns) {
+ TOPClass->insert(I);
+ ValueToClass[I] = TOPClass;
+ LIUses.push_back(I);
+ }
+
+ updateUsesAfterLoadCoercionImpl(LI, LIUses);
+
+ if (isa<PHINode>(NewValue)) {
+ NewValue->takeName(LI);
+ }
+ if (Instruction *I = dyn_cast<Instruction>(NewValue))
+ I->setDebugLoc(LI->getDebugLoc());
+ LLVM_DEBUG(dbgs() << "Load coersion: The load " << *LI
+ << " was eliminated and its uses were replaced by "
+ << *NewValue << "\n");
+ }
+ LIUses.clear();
+ NewlyGeneratedInsns.clear();
+}
+
+// Iterate over the load instructions of LoadCoercion map and replace them with
+// the right sequence of instructions.
+bool NewGVN::implementLoadCoercion() {
+ bool AnythingReplaced = false;
+ for (const auto &P : LoadCoercion) {
+ LoadInst *LI = cast<LoadInst>(P.first);
+ auto DependingInsns = P.second;
+ Value *NewValue = nullptr;
+ Instruction *FirstDepI = (*DependingInsns.begin()).first;
+ MemoryAccess *OriginalAccess = getMemoryAccess(LI);
+ MemoryAccess *DefiningAccess =
+ MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
+ // Firstly, we check if we can extract the correct value from the depending
+ // instruction. This happens when LI is dominated by its only depending
+ // instruction:
+ //
+ // Example 1
+ // BB1: BB1:
+ // store i32 100, ptr %P store i32 100, ptr %P
+ // %V1 = ... => %V1 = ...
+ // %V2 = load i32, ptr %P %V3 = add i32 %V1, 100
+ // %V3 = add i32 %V1, %V2
+ //
+ // Example 2
+ // Before load coercion
+ // BB1:
+ // store i32 100, ptr %P
+ // br i1 %Cond, label %BB2, label %BB3
+ // / \
+ // BB2 BB3
+ // ... ...
+ // br label %BB4 br label %BB4
+ // \ /
+ // BB4:
+ // %V1 = ...
+ // %V2 = load i32, ptr %P
+ // %V3 = add i32 %V1, %V2
+ //
+ // After load coercion
+ // BB1:
+ // store i32 100, ptr %P
+ // br i1 %Cond, label %BB2, label %BB3
+ // / \
+ // BB2 BB3
+ // ... ...
+ // br label %BB4 br label %BB4
+ // \ /
+ // BB4:
+ // %V1 = ...
+ // %V3 = add i32 %V1, 100
+ //
+ if (DependingInsns.size() == 1 && DT->dominates(FirstDepI, LI) &&
+ !isa<MemoryPhi>(DefiningAccess))
+ NewValue = getExtractedValue(LI, FirstDepI);
+ // If we could not eliminate the load, then we need run value numbering for
+ // the load (the load does not have an expression up to this point) and its
+ // uses.
+ if (!NewValue)
+ updateUsesAfterLoadCoercion(LI, NewValue);
+ else {
+ // Run value numbering for all the new instructions and their uses.
+ updateUsesAfterLoadCoercion(LI, NewValue);
+ AnythingReplaced = true;
+ }
+ }
+ return AnythingReplaced;
+}
+
bool NewGVN::eliminateInstructions(Function &F) {
// This is a non-standard eliminator. The normal way to eliminate is
// to walk the dominator tree in order, keeping track of available
diff --git a/llvm/test/Transforms/NewGVN/load_coercion_between_store_and_load.ll b/llvm/test/Transforms/NewGVN/load_coercion_between_store_and_load.ll
new file mode 100644
index 000000000000000..dcb166f6d29c1a6
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/load_coercion_between_store_and_load.ll
@@ -0,0 +1,341 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -p=gvn < %s | FileCheck %s -check-prefixes=GVN,OLDGVN
+; RUN: opt -S -p=newgvn < %s | FileCheck %s -check-prefixes=GVN,NEWGVN
+
+define float @test1(i32 %V1, ptr %P) {
+; GVN-LABEL: @test1(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = bitcast i32 [[V1]] to float
+; GVN-NEXT: ret float [[TMP1]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = load float, ptr %P, align 1
+ ret float %V2
+}
+
+define float @test2(ptr %V1, ptr %P) {
+; GVN-LABEL: @test2(
+; GVN-NEXT: store ptr [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V1]] to i64
+; GVN-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; GVN-NEXT: [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
+; GVN-NEXT: ret float [[TMP3]]
+;
+ store ptr %V1, ptr %P, align 1
+ %V2 = load float, ptr %P, align 1
+ ret float %V2
+}
+
+define i8 @test3(i32 %V1, ptr %P) {
+; GVN-LABEL: @test3(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V1]] to i8
+; GVN-NEXT: ret i8 [[TMP1]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ ret i8 %V2
+}
+
+define float @test4(i64 %V1, ptr %P) {
+; GVN-LABEL: @test4(
+; GVN-NEXT: store i64 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[V1]] to i32
+; GVN-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; GVN-NEXT: ret float [[TMP2]]
+;
+ store i64 %V1, ptr %P, align 1
+ %V2 = load float, ptr %P, align 1
+ ret float %V2
+}
+
+define i8 @test5(ptr %P, ptr %T) {
+; GVN-LABEL: @test5(
+; GVN-NEXT: [[V1:%.*]] = load i8, ptr [[T:%.*]], align 1
+; GVN-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 2
+; GVN-NEXT: store i8 [[V1]], ptr [[P2]], align 1
+; GVN-NEXT: ret i8 [[V1]]
+;
+ %V1 = load i8, ptr %T, align 1
+ %P2 = getelementptr i8, ptr %P, i32 2
+ store i8 %V1, ptr %P2, align 1
+ %V2 = load i8, ptr %P2, align 1
+ ret i8 %V2
+}
+
+define ptr @test6(i64 %V1, ptr %P) {
+; GVN-LABEL: @test6(
+; GVN-NEXT: store i64 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = inttoptr i64 [[V1]] to ptr
+; GVN-NEXT: ret ptr [[TMP1]]
+;
+ store i64 %V1, ptr %P, align 1
+ %V2 = load ptr, ptr %P, align 1
+ ret ptr %V2
+}
+
+define i32 @test7(double %V1, ptr %P) {
+; GVN-LABEL: @test7(
+; GVN-NEXT: store double [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = bitcast double [[V1]] to i64
+; GVN-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; GVN-NEXT: ret i32 [[TMP2]]
+;
+ store double %V1, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ ret i32 %V2
+}
+
+define i8 @test8(i32 %V1, ptr %P) {
+; OLDGVN-LABEL: @test8(
+; OLDGVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; OLDGVN-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; OLDGVN-NEXT: [[TMP1:%.*]] = lshr i32 [[V1]], 16
+; OLDGVN-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+; OLDGVN-NEXT: ret i8 [[TMP2]]
+;
+; NEWGVN-LABEL: @test8(
+; NEWGVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; NEWGVN-NEXT: [[TMP1:%.*]] = lshr i32 [[V1]], 16
+; NEWGVN-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+; NEWGVN-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; NEWGVN-NEXT: ret i8 [[TMP2]]
+;
+ store i32 %V1, ptr %P, align 1
+ %P1 = getelementptr i8, ptr %P, i32 2
+ %V2 = load i8, ptr %P1, align 1
+ ret i8 %V2
+}
+
+define double @test9(i64 %V, ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; GVN-LABEL: @test9(
+; GVN-NEXT: Entry:
+; GVN-NEXT: store i64 [[V:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast i64 [[V]] to double
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: ret double [[TMP0]]
+; GVN: F:
+; GVN-NEXT: ret double [[TMP0]]
+;
+Entry:
+ %A = load i64 , ptr %P, align 1
+ store i64 %V, ptr %P, align 1
+ br i1 %cond, label %T, label %F
+T:
+ %B = load double, ptr %P, align 1
+ ret double %B
+
+F:
+ %C = load double, ptr %P, align 1
+ ret double %C
+}
+
+define <{i8, float}> @test10(i32 %V0, ptr %P) {
+; OLDGVN-LABEL: @test10(
+; OLDGVN-NEXT: store i32 [[V0:%.*]], ptr [[P:%.*]], align 1
+; OLDGVN-NEXT: [[TMP1:%.*]] = bitcast i32 [[V0]] to float
+; OLDGVN-NEXT: [[TMP2:%.*]] = trunc i32 [[V0]] to i8
+; OLDGVN-NEXT: [[I1:%.*]] = insertvalue <{ i8, float }> poison, i8 [[TMP2]], 0
+; OLDGVN-NEXT: [[I2:%.*]] = insertvalue <{ i8, float }> [[I1]], float [[TMP1]], 1
+; OLDGVN-NEXT: ret <{ i8, float }> [[I2]]
+;
+; NEWGVN-LABEL: @test10(
+; NEWGVN-NEXT: store i32 [[V0:%.*]], ptr [[P:%.*]], align 1
+; NEWGVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V0]] to i8
+; NEWGVN-NEXT: [[TMP2:%.*]] = bitcast i32 [[V0]] to float
+; NEWGVN-NEXT: [[I1:%.*]] = insertvalue <{ i8, float }> poison, i8 [[TMP1]], 0
+; NEWGVN-NEXT: [[I2:%.*]] = insertvalue <{ i8, float }> [[I1]], float [[TMP2]], 1
+; NEWGVN-NEXT: ret <{ i8, float }> [[I2]]
+;
+ store i32 %V0, ptr %P, align 1
+ %V1 = load float, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ %I1 = insertvalue <{i8, float}> poison, i8 %V2, 0
+ %I2 = insertvalue <{i8, float}> %I1, float %V1, 1
+ ret <{i8, float}> %I2
+}
+
+define <{i8, float}> @test11(i32 %V0, ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; GVN-LABEL: @test11(
+; GVN-NEXT: Entry:
+; GVN-NEXT: store i32 [[V0:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = trunc i32 [[V0]] to i8
+; GVN-NEXT: [[TMP1:%.*]] = bitcast i32 [[V0]] to float
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ i8, float }> poison, float [[TMP1]], 1
+; GVN-NEXT: ret <{ i8, float }> [[I1]]
+; GVN: F:
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ i8, float }> poison, i8 [[TMP0]], 0
+; GVN-NEXT: ret <{ i8, float }> [[I2]]
+;
+Entry:
+ store i32 %V0, ptr %P, align 1
+ br i1 %cond, label %T, label %F
+
+T:
+ %V1 = load float, ptr %P, align 1
+ %I1 = insertvalue <{i8, float}> poison, float %V1, 1
+ ret <{i8, float}> %I1
+
+F:
+ %V2 = load i8, ptr %P, align 1
+ %I2 = insertvalue <{i8, float}> poison, i8 %V2, 0
+ ret <{i8, float}> %I2
+}
+
+define <{float, float}> @test12(i32 %V0, ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; GVN-LABEL: @test12(
+; GVN-NEXT: Entry:
+; GVN-NEXT: store i32 [[V0:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast i32 [[V0]] to float
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ float, float }> poison, float [[TMP0]], 1
+; GVN-NEXT: ret <{ float, float }> [[I1]]
+; GVN: F:
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ float, float }> poison, float [[TMP0]], 0
+; GVN-NEXT: ret <{ float, float }> [[I2]]
+;
+Entry:
+ store i32 %V0, ptr %P, align 1
+ br i1 %cond, label %T, label %F
+
+T:
+ %V1 = load float, ptr %P, align 1
+ %I1 = insertvalue <{float, float}> poison, float %V1, 1
+ ret <{float, float}> %I1
+
+F:
+ %V2 = load float, ptr %P, align 1
+ %I2 = insertvalue <{float, float}> poison, float %V2, 0
+ ret <{float, float}> %I2
+}
+
+define i8 @test13(ptr %P, i32 %V1) {
+; GVN-LABEL: @test13(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V1]] to i8
+; GVN-NEXT: [[V5:%.*]] = add i8 [[TMP1]], [[TMP1]]
+; GVN-NEXT: ret i8 [[V5]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ %V3 = load i64, ptr %P, align 1
+ %V4 = trunc i32 %V1 to i8
+ %V5 = add i8 %V2, %V4
+ ret i8 %V5
+}
+
+define i8 @test14(ptr %P, i32 %V1) {
+; GVN-LABEL: @test14(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V1]] to i8
+; GVN-NEXT: [[V5:%.*]] = add i8 [[TMP1]], [[TMP1]]
+; GVN-NEXT: ret i8 [[V5]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ %V3 = load i8, ptr %P, align 1
+ %V5 = add i8 %V2, %V3
+ ret i8 %V5
+}
+
+define i16 @test15(ptr %P, i1 %Cond) {
+; Entry
+; / \
+; T F
+; \ /
+; vv
+; Exit
+;
+; GVN-LABEL: @test15(
+; GVN-NEXT: Entry:
+; GVN-NEXT: store i32 13, ptr [[P:%.*]], align 1
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: br label [[EXIT:%.*]]
+; GVN: F:
+; GVN-NEXT: br label [[EXIT]]
+; GVN: Exit:
+; GVN-NEXT: ret i16 13
+;
+Entry:
+ store i32 13, ptr %P, align 1
+ br i1 %Cond, label %T, label %F
+
+T:
+ %V1 = load i16, ptr %P, align 1
+ br label %Exit
+
+F:
+ %V2 = load i32, ptr %P, align 1
+ br label %Exit
+
+Exit:
+ %V3 = load i16, ptr %P, align 1
+ ret i16 %V3
+}
+
+define i64 @test16(ptr %V1) {
+; GVN-LABEL: @test16(
+; GVN-NEXT: Entry:
+; GVN-NEXT: store ptr [[V1:%.*]], ptr inttoptr (i64 16 to ptr), align 8
+; GVN-NEXT: [[V3:%.*]] = load i64, ptr [[V1]], align 4
+; GVN-NEXT: ret i64 [[V3]]
+;
+Entry:
+ store ptr %V1, ptr inttoptr (i64 16 to ptr), align 8
+ %V2 = load ptr, ptr inttoptr (i64 16 to ptr), align 8
+ %V3 = load i64, ptr %V2
+ ret i64 %V3
+}
+
+declare void @foo1(ptr, i32) #0
+
+define i32 @test17(ptr %P, i32 %V1) {
+; GVN-LABEL: @test17(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V2:%.*]] = call i32 @foo1(ptr [[P]], i32 [[V1]]) #[[ATTR0:[0-9]+]]
+; GVN-NEXT: [[V3:%.*]] = load i32, ptr [[P]], align 1
+; GVN-NEXT: [[V4:%.*]] = add i32 [[V2]], [[V3]]
+; GVN-NEXT: ret i32 [[V4]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = call i32 @foo1(ptr %P, i32 %V1) #0
+ %V3 = load i32, ptr %P, align 1
+ %V4 = add i32 %V2, %V3
+ ret i32 %V4
+}
+
+declare void @foo2(ptr, i32) #1
+
+define i32 @test18(ptr %P, i32 %V1) {
+; GVN-LABEL: @test18(
+; GVN-NEXT: store i32 [[V1:%.*]], ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V2:%.*]] = call i32 @foo2(ptr [[P]], i32 [[V1]]) #[[ATTR1:[0-9]+]]
+; GVN-NEXT: [[V4:%.*]] = add i32 [[V2]], [[V1]]
+; GVN-NEXT: ret i32 [[V4]]
+;
+ store i32 %V1, ptr %P, align 1
+ %V2 = call i32 @foo2(ptr %P, i32 %V1) #1
+ %V3 = load i32, ptr %P, align 1
+ %V4 = add i32 %V2, %V3
+ ret i32 %V4
+}
+
+attributes #0 = { willreturn }
+attributes #1 = { readonly }
diff --git a/llvm/test/Transforms/NewGVN/pr14166-xfail.ll b/llvm/test/Transforms/NewGVN/pr14166-xfail.ll
index 1e722361d7c545f..ceb9d2c67e02c4d 100644
--- a/llvm/test/Transforms/NewGVN/pr14166-xfail.ll
+++ b/llvm/test/Transforms/NewGVN/pr14166-xfail.ll
@@ -1,4 +1,3 @@
-; XFAIL: *
; RUN: opt -disable-basic-aa -passes=newgvn -S < %s | FileCheck %s
; NewGVN fails this due to missing load coercion
target datalayout = "e-p:32:32:32"
>From 8a312c05d320cea6d42d7d01e622cb6b883be874 Mon Sep 17 00:00:00 2001
From: Konstantina Mitropoulou <Konstantina.Mitropoulou at amd.com>
Date: Mon, 9 Oct 2023 23:19:32 -0700
Subject: [PATCH 2/2] [NewGVN][2/3] Load coercion between loads that have
live-on-entry definitions
In the following example, both %V1 and %V2 have live-on-entry definitions and
their memory locations are overlapping. After load coercion the value of %V2
is extracted from %V1 and the uses of %V2 are updated accordingly.
```
Before load coercion
BB1
%V1 = load <2 x i32>, ptr %P, align 1
%V2 = load i32, ptr %P, align 1
%V3 = add i32 %V2, 42
After load coercion
BB1
%V1 = load <2 x i32>, ptr %P, align 1
%0 = bitcast <2 x i32> %V1 to i64
%1 = trunc i64 %0 to i32
%V3 = add i32 %1, 42
```
---
llvm/lib/Transforms/Scalar/NewGVN.cpp | 185 +++++++-
.../NewGVN/load_coercion_between_loads.ll | 424 ++++++++++++++++++
2 files changed, 586 insertions(+), 23 deletions(-)
create mode 100644 llvm/test/Transforms/NewGVN/load_coercion_between_loads.ll
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index f4c84bd747bb417..140ec02572db7de 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -73,6 +73,7 @@
#include "llvm/Analysis/CFGPrinter.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InstructionPrecedenceTracking.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemorySSA.h"
@@ -500,6 +501,7 @@ class NewGVN {
AssumptionCache *AC = nullptr;
const DataLayout &DL;
std::unique_ptr<PredicateInfo> PredInfo;
+ ImplicitControlFlowTracking *ICF = nullptr;
// These are the only two things the create* functions should have
// side-effects on due to allocating memory.
@@ -915,6 +917,10 @@ class NewGVN {
// LoadCoercion map.
bool tryAddLoadDepInsnIntoLoadCoercionMap(LoadInst *, Instruction *,
BasicBlock *) const;
+ // Check if the candidate load can be optimized by another load which is also
+ // a live of entry definition and add it in LoadCoercion map.
+ bool findLiveOnEntryDependency(LoadInst *, LoadInst *, ArrayRef<BasicBlock *>,
+ bool) const;
// Collect the load instructions that can be optimized with load coercion.
// The filtering of the load instructions is based the type of their memory
// access.
@@ -923,6 +929,8 @@ class NewGVN {
MemoryAccess *) const;
const Expression *performSymbolicLoadCoercionForConstantMemoryDef(
Type *, Value *, LoadInst *, Instruction *, MemoryAccess *) const;
+ bool performSymbolicLoadCoercionForLiveOnEntryDef(LoadInst *,
+ MemoryAccess *) const;
// Code generation for load coercion. Replaces the load with the right
// instruction or the right sequence of instructions.
bool implementLoadCoercion();
@@ -1577,6 +1585,133 @@ bool NewGVN::tryAddLoadDepInsnIntoLoadCoercionMap(
return true;
}
+// Check if it is possible to apply load coercion between CandidateLI and
+// DependingLoad.
+bool NewGVN::findLiveOnEntryDependency(LoadInst *CandidateLI,
+ LoadInst *DependingLoad,
+ ArrayRef<BasicBlock *> DependingBlocks,
+ bool IsMemoryPhiDep) const {
+ int Offset = -1;
+
+ if (!DependingLoad || CandidateLI == DependingLoad ||
+ DependingLoad->getNumUses() == 0)
+ return false;
+
+ BasicBlock *DependingLoadBB = DependingLoad->getParent();
+ if (!ReachableBlocks.count(DependingLoadBB) ||
+ ICF->isDominatedByICFIFromSameBlock(CandidateLI))
+ return false;
+
+ if (InstructionsToErase.count(DependingLoad))
+ return false;
+
+ // We do not look deep in the CFG. We consider either instructions that
+ // dominate CandidateLI or instructions that are in one of the predecessors of
+ // CandidateLI.
+ if (DT->dominates(DependingLoad, CandidateLI))
+ Offset = analyzeLoadFromClobberingLoad(CandidateLI->getType(),
+ CandidateLI->getPointerOperand(),
+ DependingLoad, DL);
+ else {
+ BasicBlock *CandidateLIBB = CandidateLI->getParent();
+ auto It1 = llvm::find(DependingBlocks, CandidateLIBB);
+ auto It2 = llvm::find(DependingBlocks, DependingLoadBB);
+ auto Ite = DependingBlocks.end();
+ if (It1 == Ite && It2 != Ite && !isBackedge(DependingLoadBB, CandidateLIBB))
+ Offset = analyzeLoadFromClobberingLoad(CandidateLI->getType(),
+ CandidateLI->getPointerOperand(),
+ DependingLoad, DL);
+ }
+
+ bool IsLoadCoercionCandidate = false;
+ if (Offset >= 0) {
+ // If the candidate load depends on a MemoryPhi, then we do not consider the
+ // parent block of the depending instruction, but instead it is more
+ // convenient to consider the basic block of the MemoryPhi from which the
+ // value comes e.g.:
+ // BB1:
+ // %V1 = load i32, ptr %P
+ // br i1 %Cond, label %BB2, label %BB3
+ // / \
+ // BB2: BB3:
+ // store i32 100, ptr %P br label %BB4
+ // br label %BB4 /
+ // \ /
+ // BB4:
+ // %V2 = load i32, ptr %P
+ //
+ BasicBlock *BB = IsMemoryPhiDep ? DependingBlocks.back() : DependingLoadBB;
+ IsLoadCoercionCandidate |=
+ tryAddLoadDepInsnIntoLoadCoercionMap(CandidateLI, DependingLoad, BB);
+ }
+ return IsLoadCoercionCandidate;
+}
+
+// Find load coercion opportunities between instructions with live on entry
+// definitions.
+bool NewGVN::performSymbolicLoadCoercionForLiveOnEntryDef(
+ LoadInst *LI, MemoryAccess *DefiningAccess) const {
+ bool IsLoadCoercionCandidate = false;
+ for (const auto &U : MSSA->getLiveOnEntryDef()->uses()) {
+ if (auto *MemUse = dyn_cast<MemoryUse>(U.getUser())) {
+ // TODO: Add support for calls.
+ LoadInst *DependingLoad = dyn_cast<LoadInst>(MemUse->getMemoryInst());
+ if (!DependingLoad || LI == DependingLoad)
+ continue;
+
+ // If the two instructions have the same type, then there is a load
+ // coercion opportunity only if the LI and the DependingLoad are in
+ // different basic blocks and the basic block of the DependingLoad is one
+ // of the predecessors of the basic block of the LI. For any other case,
+ // the LI will be eliminated by adding the two loads in the same
+ // congruence class.
+ //
+ // Example 1: Here, we do not need to apply load coercion. The two load
+ // will be added in the same congruence class and %V2 will be eliminated.
+ //
+ // BB1:
+ // ...
+ // %V1 = load i32, ptr %P
+ // br label %BB2
+ //
+ // BB2
+ // ...
+ // %V2 = load i32, ptr %P
+ // ...
+ //
+ // Example 2: Here, %V2 can be replaced by a phi node.
+ // BB1: BB2:
+ // %V1 = load <2 x i32>, ptr %P br label %BB3
+ // br label %BB3 /
+ // \ /
+ // BB3:
+ // %V2 = load i32, ptr %P
+ //
+ // Hence, the code will become:
+ // BB1: BB2:
+ // %V1 = load <2 x i32>, ptr %P %V2' = load i32, ptr %P
+ // %0 = bitcast <2 x i32> %V1 to i64 br label %BB3
+ // %1 = trunc i64 %0 to i32 /
+ // br label %BB3 /
+ // \ /
+ // BB3:
+ // %V2 = phi i32 [ %1, %BB1], [ %V2', %BB2 ]
+ //
+ if (DependingLoad->getType() == LI->getType() &&
+ (DT->dominates(DependingLoad, LI) ||
+ LI->getParent() == DependingLoad->getParent()))
+ continue;
+
+ SmallVector<BasicBlock *, 2> Preds;
+ for (auto *BB : predecessors(LI->getParent()))
+ Preds.push_back(BB);
+ IsLoadCoercionCandidate |=
+ findLiveOnEntryDependency(LI, DependingLoad, Preds, false);
+ }
+ }
+ return IsLoadCoercionCandidate;
+}
+
// Find load coercion opportunities between load (LI) and store instructions
// (DepSI).
bool NewGVN::performSymbolicLoadCoercionForNonConstantMemoryDef(
@@ -1702,29 +1837,30 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
// Check if we can apply load coercion.
bool IsLoadCoercionCandidate = false;
- if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
- if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
- Instruction *DefiningInst = MD->getMemoryInst();
- // If the defining instruction is not reachable, replace with poison.
- if (!ReachableBlocks.count(DefiningInst->getParent()))
- return createConstantExpression(PoisonValue::get(LI->getType()));
- // This will handle stores and memory insts. We only do if it the
- // defining access has a different type, or it is a pointer produced by
- // certain memory operations that cause the memory to have a fixed value
- // (IE things like calloc).
- if (const auto *CoercionResult =
- performSymbolicLoadCoercionForConstantMemoryDef(
- LI->getType(), LoadAddressLeader, LI, DefiningInst,
- DefiningAccess))
- return CoercionResult;
-
- if (EnableLoadCoercion) {
- if (auto *DepSI = dyn_cast<StoreInst>(DefiningInst)) {
- if (!isa<Constant>(lookupOperandLeader(DepSI->getValueOperand()))) {
- IsLoadCoercionCandidate =
- performSymbolicLoadCoercionForNonConstantMemoryDef(
- LI, DepSI, DefiningAccess);
- }
+ if (MSSA->isLiveOnEntryDef(DefiningAccess) && EnableLoadCoercion)
+ IsLoadCoercionCandidate =
+ performSymbolicLoadCoercionForLiveOnEntryDef(LI, DefiningAccess);
+ else if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
+ Instruction *DefiningInst = MD->getMemoryInst();
+ // If the defining instruction is not reachable, replace with poison.
+ if (!ReachableBlocks.count(DefiningInst->getParent()))
+ return createConstantExpression(PoisonValue::get(LI->getType()));
+ // This will handle stores and memory insts. We only do if it the
+ // defining access has a different type, or it is a pointer produced by
+ // certain memory operations that cause the memory to have a fixed value
+ // (IE things like calloc).
+ if (const auto *CoercionResult =
+ performSymbolicLoadCoercionForConstantMemoryDef(
+ LI->getType(), LoadAddressLeader, LI, DefiningInst,
+ DefiningAccess))
+ return CoercionResult;
+
+ if (EnableLoadCoercion) {
+ if (auto *DepSI = dyn_cast<StoreInst>(DefiningInst)) {
+ if (!isa<Constant>(lookupOperandLeader(DepSI->getValueOperand()))) {
+ IsLoadCoercionCandidate =
+ performSymbolicLoadCoercionForNonConstantMemoryDef(
+ LI, DepSI, DefiningAccess);
}
}
}
@@ -3172,6 +3308,7 @@ void NewGVN::cleanupTables() {
MemoryToUsers.clear();
RevisitOnReachabilityChange.clear();
IntrinsicInstPred.clear();
+ ICF->clear();
}
// Assign local DFS number mapping to instructions, and leave space for Value
@@ -3625,6 +3762,8 @@ bool NewGVN::runGVN() {
NumFuncArgs = F.arg_size();
MSSAWalker = MSSA->getWalker();
SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
+ ImplicitControlFlowTracking ImplicitCFT;
+ ICF = &ImplicitCFT;
// Count number of instructions for sizing of hash tables, and come
// up with a global dfs numbering for instructions.
diff --git a/llvm/test/Transforms/NewGVN/load_coercion_between_loads.ll b/llvm/test/Transforms/NewGVN/load_coercion_between_loads.ll
new file mode 100644
index 000000000000000..c4193acfcb22418
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/load_coercion_between_loads.ll
@@ -0,0 +1,424 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -p=gvn < %s | FileCheck %s -check-prefixes=GVN,OLDGVN
+; RUN: opt -S -p=newgvn < %s | FileCheck %s -check-prefixes=GVN,NEWGVN
+
+define i8 @test1(ptr %P) {
+; GVN-LABEL: @test1(
+; GVN-NEXT: [[V1:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V1]] to i8
+; GVN-NEXT: [[V4:%.*]] = add i8 [[TMP1]], [[TMP1]]
+; GVN-NEXT: ret i8 [[V4]]
+;
+ %V1 = load i32, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ %V3 = trunc i32 %V1 to i8
+ %V4 = add i8 %V2, %V3
+ ret i8 %V4
+}
+
+define i32 @test2(ptr %P) {
+; GVN-LABEL: @test2(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: [[V5:%.*]] = add i32 [[TMP1]], [[TMP1]]
+; GVN-NEXT: ret i32 [[V5]]
+;
+Entry:
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ %V3 = bitcast <2 x i32> %V1 to i64
+ %V4 = trunc i64 %V3 to i32
+ %V5 = add i32 %V2, %V4
+ ret i32 %V5
+}
+
+define <{<2 x i32>, i32}> @test3(ptr %P) {
+; GVN-LABEL: @test3(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <2 x i32>, i32 }> poison, <2 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <2 x i32>, i32 }> [[I1]], i32 [[TMP1]], 1
+; GVN-NEXT: ret <{ <2 x i32>, i32 }> [[I2]]
+;
+Entry:
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ %I1 = insertvalue <{<2 x i32>, i32}> poison, <2 x i32> %V1, 0
+ %I2 = insertvalue <{<2 x i32>, i32}> %I1, i32 %V2, 1
+ ret <{<2 x i32>, i32}> %I2
+}
+
+define <{<2 x i32>, i32}> @test4(ptr %P) {
+; GVN-LABEL: @test4(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V2:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P]], align 1
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <2 x i32>, i32 }> poison, <2 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <2 x i32>, i32 }> [[I1]], i32 [[V2]], 1
+; GVN-NEXT: ret <{ <2 x i32>, i32 }> [[I2]]
+;
+Entry:
+ %V2 = load i32, ptr %P, align 1
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %I1 = insertvalue <{<2 x i32>, i32}> poison, <2 x i32> %V1, 0
+ %I2 = insertvalue <{<2 x i32>, i32}> %I1, i32 %V2, 1
+ ret <{<2 x i32>, i32}> %I2
+}
+
+define <{<2 x i32>, i32, i32}> @test5(ptr %P) {
+; GVN-LABEL: @test5(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V0:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <2 x i32>, i32, i32 }> poison, <2 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <2 x i32>, i32, i32 }> [[I1]], i32 [[TMP1]], 1
+; GVN-NEXT: [[I3:%.*]] = insertvalue <{ <2 x i32>, i32, i32 }> [[I2]], i32 [[V0]], 2
+; GVN-NEXT: ret <{ <2 x i32>, i32, i32 }> [[I3]]
+;
+Entry:
+ %V0 = load i32, ptr %P, align 1
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ %I1 = insertvalue <{<2 x i32>, i32, i32}> poison, <2 x i32> %V1, 0
+ %I2 = insertvalue <{<2 x i32>, i32, i32}> %I1, i32 %V2, 1
+ %I3 = insertvalue <{<2 x i32>, i32, i32}> %I2, i32 %V0, 2
+ ret <{<2 x i32>, i32, i32}> %I3
+}
+
+define <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> @test6(ptr %P) {
+; GVN-LABEL: @test6(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[V1]] to i128
+; GVN-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; GVN-NEXT: [[TMP2:%.*]] = bitcast i64 [[TMP1]] to <2 x i32>
+; GVN-NEXT: [[TMP3:%.*]] = trunc i128 [[TMP0]] to i32
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> poison, <4 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I1]], <2 x i32> [[TMP2]], 1
+; GVN-NEXT: [[I3:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I2]], i32 [[TMP3]], 2
+; GVN-NEXT: [[I4:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I3]], <2 x i32> [[TMP2]], 3
+; GVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I4]]
+;
+Entry:
+ %V1 = load <4 x i32>, ptr %P, align 1
+ %V2 = load <2 x i32>, ptr %P, align 1
+ %V3 = load i32, ptr %P, align 1
+ %V4 = load <2 x i32>, ptr %P, align 1
+ %I1 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> poison, <4 x i32> %V1, 0
+ %I2 = insertvalue <{<4 x i32>,<2 x i32>, i32, <2 x i32>}> %I1, <2 x i32> %V2, 1
+ %I3 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I2, i32 %V3, 2
+ %I4 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I3, <2 x i32> %V4, 3
+ ret <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I4
+}
+
+define float @test7(ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; GVN-LABEL: @test7(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast i32 [[V1]] to float
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: ret float [[TMP0]]
+; GVN: F:
+; GVN-NEXT: ret float [[TMP0]]
+;
+Entry:
+ %V1 = load i32, ptr %P, align 1
+ %V2 = load float, ptr %P, align 1
+ br i1 %cond, label %T, label %F
+T:
+ ret float %V2
+
+F:
+ %V3 = bitcast i32 %V1 to float
+ ret float %V3
+}
+
+define <{<2 x i32>, i32, i32, i32, i32}> @test8(ptr %P) {
+; GVN-LABEL: @test8(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V0:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <2 x i32>, i32, i32, i32, i32 }> poison, <2 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <2 x i32>, i32, i32, i32, i32 }> [[I1]], i32 [[TMP1]], 1
+; GVN-NEXT: [[I3:%.*]] = insertvalue <{ <2 x i32>, i32, i32, i32, i32 }> [[I2]], i32 [[V0]], 2
+; GVN-NEXT: [[I4:%.*]] = insertvalue <{ <2 x i32>, i32, i32, i32, i32 }> [[I3]], i32 [[TMP1]], 3
+; GVN-NEXT: [[I5:%.*]] = insertvalue <{ <2 x i32>, i32, i32, i32, i32 }> [[I4]], i32 [[TMP1]], 4
+; GVN-NEXT: ret <{ <2 x i32>, i32, i32, i32, i32 }> [[I5]]
+;
+Entry:
+ %V0 = load i32, ptr %P, align 1
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ %I1 = insertvalue <{<2 x i32>, i32, i32, i32, i32}> poison, <2 x i32> %V1, 0
+ %I2 = insertvalue <{<2 x i32>, i32, i32, i32, i32}> %I1, i32 %V2, 1
+ %I3 = insertvalue <{<2 x i32>, i32, i32, i32, i32}> %I2, i32 %V0, 2
+ %V3 = load i32, ptr %P, align 1
+ %I4 = insertvalue <{<2 x i32>, i32, i32, i32, i32}> %I3, i32 %V3, 3
+ %V4 = load i32, ptr %P, align 1
+ %I5 = insertvalue <{<2 x i32>, i32, i32, i32, i32}> %I4, i32 %V4, 4
+ ret <{<2 x i32>, i32, i32, i32, i32}> %I5
+}
+
+define <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> @test9(ptr %P) {
+; GVN-LABEL: @test9(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[V1]] to i128
+; GVN-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; GVN-NEXT: [[TMP2:%.*]] = bitcast i64 [[TMP1]] to <2 x i32>
+; GVN-NEXT: [[TMP3:%.*]] = trunc i128 [[TMP0]] to i32
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> poison, <4 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I1]], <2 x i32> [[TMP2]], 1
+; GVN-NEXT: [[I3:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I2]], i32 [[TMP3]], 2
+; GVN-NEXT: [[I4:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I3]], <2 x i32> [[TMP2]], 3
+; GVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32, <2 x i32> }> [[I4]]
+;
+Entry:
+ %V1 = load <4 x i32>, ptr %P, align 1
+ %V2 = load <2 x i32>, ptr %P, align 1
+ %V3 = load i32, ptr %P, align 1
+ %V4 = load <2 x i32>, ptr %P, align 1
+ %I1 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> poison, <4 x i32> %V1, 0
+ %I2 = insertvalue <{<4 x i32>,<2 x i32>, i32, <2 x i32>}> %I1, <2 x i32> %V2, 1
+ %I3 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I2, i32 %V3, 2
+ %I4 = insertvalue <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I3, <2 x i32> %V4, 3
+ ret <{<4 x i32>, <2 x i32>, i32, <2 x i32>}> %I4
+}
+
+define <{<4 x i32>, <2 x i32>, i32}> @test10(ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; OLDGVN-LABEL: @test10(
+; OLDGVN-NEXT: Entry:
+; OLDGVN-NEXT: [[V1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
+; OLDGVN-NEXT: [[I1:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> poison, <4 x i32> [[V1]], 0
+; OLDGVN-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[V1]] to i128
+; OLDGVN-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i32
+; OLDGVN-NEXT: [[TMP2:%.*]] = trunc i128 [[TMP0]] to i64
+; OLDGVN-NEXT: [[TMP3:%.*]] = bitcast i64 [[TMP2]] to <2 x i32>
+; OLDGVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; OLDGVN: T:
+; OLDGVN-NEXT: [[I2:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> [[I1]], <2 x i32> [[TMP3]], 1
+; OLDGVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32 }> [[I2]]
+; OLDGVN: F:
+; OLDGVN-NEXT: [[I3:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> [[I1]], i32 [[TMP1]], 2
+; OLDGVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32 }> [[I3]]
+;
+; NEWGVN-LABEL: @test10(
+; NEWGVN-NEXT: Entry:
+; NEWGVN-NEXT: [[V1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
+; NEWGVN-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[V1]] to i128
+; NEWGVN-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i32
+; NEWGVN-NEXT: [[TMP2:%.*]] = trunc i128 [[TMP0]] to i64
+; NEWGVN-NEXT: [[TMP3:%.*]] = bitcast i64 [[TMP2]] to <2 x i32>
+; NEWGVN-NEXT: [[I1:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> poison, <4 x i32> [[V1]], 0
+; NEWGVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; NEWGVN: T:
+; NEWGVN-NEXT: [[I2:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> [[I1]], <2 x i32> [[TMP3]], 1
+; NEWGVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32 }> [[I2]]
+; NEWGVN: F:
+; NEWGVN-NEXT: [[I3:%.*]] = insertvalue <{ <4 x i32>, <2 x i32>, i32 }> [[I1]], i32 [[TMP1]], 2
+; NEWGVN-NEXT: ret <{ <4 x i32>, <2 x i32>, i32 }> [[I3]]
+;
+Entry:
+ %V1 = load <4 x i32>, ptr %P, align 1
+ %I1 = insertvalue <{<4 x i32>, <2 x i32>, i32}> poison, <4 x i32> %V1, 0
+ br i1 %cond, label %T, label %F
+T:
+ %V2 = load <2 x i32>, ptr %P, align 1
+ %I2 = insertvalue <{<4 x i32>,<2 x i32>, i32}> %I1, <2 x i32> %V2, 1
+ ret <{<4 x i32>, <2 x i32>, i32}> %I2
+
+F:
+ %V3 = load i32, ptr %P, align 1
+ %I3 = insertvalue <{<4 x i32>, <2 x i32>, i32}> %I1, i32 %V3, 2
+ ret <{<4 x i32>, <2 x i32>, i32}> %I3
+}
+
+define <{<2 x i32>, i32, float}> @test11(ptr %P) {
+; GVN-LABEL: @test11(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; GVN-NEXT: [[I1:%.*]] = insertvalue <{ <2 x i32>, i32, float }> poison, <2 x i32> [[V1]], 0
+; GVN-NEXT: [[I2:%.*]] = insertvalue <{ <2 x i32>, i32, float }> [[I1]], i32 [[TMP1]], 1
+; GVN-NEXT: [[I3:%.*]] = insertvalue <{ <2 x i32>, i32, float }> [[I2]], float [[TMP2]], 2
+; GVN-NEXT: ret <{ <2 x i32>, i32, float }> [[I3]]
+;
+Entry:
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = load i32, ptr %P, align 1
+ %V3 = load float, ptr %P, align 1
+ %I1 = insertvalue <{<2 x i32>, i32, float}> poison, <2 x i32> %V1, 0
+ %I2 = insertvalue <{<2 x i32>, i32, float}> %I1, i32 %V2, 1
+ %I3 = insertvalue <{<2 x i32>, i32, float}> %I2, float %V3, 2
+ ret <{<2 x i32>, i32, float}> %I3
+}
+
+define i8 @test12(ptr %P, i32 %V) {
+; GVN-LABEL: @test12(
+; GVN-NEXT: [[V1:%.*]] = load i32, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP1:%.*]] = trunc i32 [[V1]] to i8
+; GVN-NEXT: store i32 [[V:%.*]], ptr [[P]], align 1
+; GVN-NEXT: [[TMP2:%.*]] = trunc i32 [[V]] to i8
+; GVN-NEXT: [[V5:%.*]] = add i8 [[TMP1]], [[TMP1]]
+; GVN-NEXT: [[V6:%.*]] = add i8 [[TMP2]], [[V5]]
+; GVN-NEXT: ret i8 [[V6]]
+;
+ %V1 = load i32, ptr %P, align 1
+ %V2 = load i8, ptr %P, align 1
+ %V3 = trunc i32 %V1 to i8
+ store i32 %V, ptr %P, align 1
+ %V4 = load i8, ptr %P, align 1
+ %V5 = add i8 %V2, %V3
+ %V6 = add i8 %V4, %V5
+ ret i8 %V6
+}
+
+define float @test13(ptr %P, i1 %cond) {
+; Entry
+; / \
+; T F
+;
+; GVN-LABEL: @test13(
+; GVN-NEXT: Entry:
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: [[V2:%.*]] = load float, ptr [[P:%.*]], align 1
+; GVN-NEXT: ret float [[V2]]
+; GVN: F:
+; GVN-NEXT: [[V3:%.*]] = load float, ptr [[P]], align 1
+; GVN-NEXT: ret float [[V3]]
+;
+Entry:
+ %V1 = load i32, ptr %P, align 1
+ br i1 %cond, label %T, label %F
+T:
+ %V2 = load float, ptr %P, align 1
+ ret float %V2
+
+F:
+ %V3 = load float, ptr %P, align 1
+ ret float %V3
+}
+
+define i32 @test14(ptr %P, i1 %Cond) {
+; Entry
+; / \
+; T F
+; \ /
+; vv
+; Exit
+;
+; GVN-LABEL: @test14(
+; GVN-NEXT: Entry:
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[TMP0:%.*]] = bitcast <2 x i32> [[V1]] to i64
+; GVN-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; GVN-NEXT: br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; GVN: T:
+; GVN-NEXT: br label [[EXIT:%.*]]
+; GVN: F:
+; GVN-NEXT: br label [[EXIT]]
+; GVN: Exit:
+; GVN-NEXT: [[PHI:%.*]] = phi i32 [ 100, [[T]] ], [ 200, [[F]] ]
+; GVN-NEXT: [[V2:%.*]] = extractelement <2 x i32> [[V1]], i64 1
+; GVN-NEXT: [[V4:%.*]] = add i32 [[TMP1]], [[V2]]
+; GVN-NEXT: [[V5:%.*]] = add i32 [[V4]], [[PHI]]
+; GVN-NEXT: ret i32 [[V5]]
+;
+Entry:
+ %V1 = load <2 x i32>, ptr %P, align 1
+ br i1 %Cond, label %T, label %F
+
+T:
+ br label %Exit
+
+F:
+ br label %Exit
+
+Exit:
+ %Phi = phi i32 [100, %T], [200, %F]
+ %V2 = extractelement <2 x i32> %V1, i64 1
+ %V3 = load i32, ptr %P, align 1
+ %V4 = add i32 %V3, %V2
+ %V5 = add i32 %V4, %Phi
+ ret i32 %V5
+}
+
+define void @test15(i1 %Cond) {
+; OLDGVN-LABEL: @test15(
+; OLDGVN-NEXT: Entry:
+; OLDGVN-NEXT: br label [[LOOP_HEADER:%.*]]
+; OLDGVN: Loop.Header:
+; OLDGVN-NEXT: store i64 0, ptr null, align 8
+; OLDGVN-NEXT: br i1 [[COND:%.*]], label [[LOOP_HEADER]], label [[EXIT:%.*]]
+; OLDGVN: Exit:
+; OLDGVN-NEXT: ret void
+;
+; NEWGVN-LABEL: @test15(
+; NEWGVN-NEXT: Entry:
+; NEWGVN-NEXT: br label [[LOOP_HEADER:%.*]]
+; NEWGVN: Loop.Header:
+; NEWGVN-NEXT: br label [[LOOP_LATCH:%.*]]
+; NEWGVN: Loop.Latch:
+; NEWGVN-NEXT: store i64 0, ptr null, align 8
+; NEWGVN-NEXT: br i1 [[COND:%.*]], label [[LOOP_HEADER]], label [[EXIT:%.*]]
+; NEWGVN: Exit:
+; NEWGVN-NEXT: ret void
+;
+Entry:
+ br label %Loop.Header
+
+Loop.Header: ; preds = %Loop.Latch, %entry
+ %V1 = load ptr, ptr inttoptr (i64 16 to ptr), align 8
+ %V2 = load i64, ptr %V1, align 8
+ br label %Loop.Latch
+
+Loop.Latch: ; preds = %Loop.Header
+ %V3 = load ptr, ptr inttoptr (i64 16 to ptr), align 8
+ %V4 = load i64, ptr %V3, align 8
+ store i64 0, ptr null, align 8
+ br i1 %Cond, label %Loop.Header, label %Exit
+
+Exit:
+ ret void
+}
+
+declare void @foo(ptr, i32) #0
+
+define i32 @test16(ptr %P) {
+; GVN-LABEL: @test16(
+; GVN-NEXT: [[V1:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; GVN-NEXT: [[V2:%.*]] = extractelement <2 x i32> [[V1]], i64 1
+; GVN-NEXT: [[V3:%.*]] = call i32 @foo(ptr [[P]], i32 [[V2]]) #[[ATTR0:[0-9]+]]
+; GVN-NEXT: [[V4:%.*]] = load i32, ptr [[P]], align 1
+; GVN-NEXT: [[V5:%.*]] = add i32 [[V3]], [[V4]]
+; GVN-NEXT: ret i32 [[V5]]
+;
+ %V1 = load <2 x i32>, ptr %P, align 1
+ %V2 = extractelement <2 x i32> %V1, i64 1
+ %V3 = call i32 @foo(ptr %P, i32 %V2) #0
+ %V4 = load i32, ptr %P, align 1
+ %V5 = add i32 %V3, %V4
+ ret i32 %V5
+}
+
+attributes #0 = { willreturn }
More information about the llvm-commits
mailing list