[llvm-branch-commits] [llvm] [GVN] Replace SCEV address recovery with GEP peeling (PR #215234)
Madhur Amilkanthwar via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 10 03:11:37 PDT 2026
https://github.com/madhur13490 created https://github.com/llvm/llvm-project/pull/215234
Avoid introducing ScalarEvolution into GVN by teaching PHITransAddr to
recover affine-equivalent select-arm addresses from nested GEPs.
>From f0829eece7f53e22fee5ca358a535d99e6b13d2e Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 10 Aug 2026 02:00:14 -0700
Subject: [PATCH] [GVN] Replace SCEV address recovery with GEP peeling
Avoid introducing ScalarEvolution into GVN by teaching PHITransAddr to
recover affine-equivalent select-arm addresses from nested GEPs.
---
.../llvm/Analysis/MemoryDependenceAnalysis.h | 12 --
llvm/include/llvm/Analysis/PHITransAddr.h | 6 +
llvm/include/llvm/Transforms/Scalar/GVN.h | 3 -
.../lib/Analysis/MemoryDependenceAnalysis.cpp | 81 -------------
llvm/lib/Analysis/PHITransAddr.cpp | 106 +++++++++++++++---
llvm/lib/Transforms/Scalar/GVN.cpp | 6 -
...ll => phitrans-gep-select-load-address.ll} | 16 +--
7 files changed, 100 insertions(+), 130 deletions(-)
rename llvm/test/Transforms/GVN/{scev-select-load-address.ll => phitrans-gep-select-load-address.ll} (79%)
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index 98e7f84855b4a..459c8aeb5ab5b 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -32,7 +32,6 @@ namespace llvm {
class AssumptionCache;
class DominatorTree;
class PHITransAddr;
-class ScalarEvolution;
/// A memory dependence query can return one of three different answers.
class MemDepResult {
@@ -383,12 +382,6 @@ class MemoryDependenceResults {
PredIteratorCache PredCache;
EarliestEscapeAnalysis EEA;
- /// Optional, opt-in ScalarEvolution used only to recover select-dependent
- /// load addresses that are affine-equal (but not syntactically identical) to
- /// an existing pointer. Null unless a client (currently GVN) sets it, so all
- /// other MemDep users are unaffected.
- ScalarEvolution *SE = nullptr;
-
unsigned DefaultBlockScanLimit;
/// Offsets to dependant clobber loads.
@@ -402,11 +395,6 @@ class MemoryDependenceResults {
: AA(AA), AC(AC), TLI(TLI), DT(DT), EEA(DT),
DefaultBlockScanLimit(DefaultBlockScanLimit) {}
- /// Opt in to SCEV-based recovery of affine-equal select-dependent
- /// addresses. Passing null (the default) preserves the syntactic-only
- /// behavior.
- void setScalarEvolution(ScalarEvolution *S) { SE = S; }
-
/// Handle invalidation in the new PM.
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
FunctionAnalysisManager::Invalidator &Inv);
diff --git a/llvm/include/llvm/Analysis/PHITransAddr.h b/llvm/include/llvm/Analysis/PHITransAddr.h
index 90df01269057f..2a45539a70156 100644
--- a/llvm/include/llvm/Analysis/PHITransAddr.h
+++ b/llvm/include/llvm/Analysis/PHITransAddr.h
@@ -146,6 +146,12 @@ class PHITransAddr {
LLVM_ABI bool verify() const;
private:
+ /// Recover an available address when select-arm translation only fails
+ /// because a constant index delta is folded into a nested i8 GEP offset.
+ Value *findAvailableSelectArmAddr(Value *Cond, BasicBlock *CurBB,
+ BasicBlock *PredBB, const DominatorTree *DT,
+ bool CondVal) const;
+
Value *translateSubExpr(Value *V, BasicBlock *CurBB, BasicBlock *PredBB,
const DominatorTree *DT, Value *Cond = nullptr,
bool CondVal = false);
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 554d66deecf1c..0275c01b28020 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -49,7 +49,6 @@ class GetElementPtrInst;
class ImplicitControlFlowTracking;
class LoadInst;
class LoopInfo;
-class ScalarEvolution;
class MemDepResult;
class MemoryAccess;
class MemoryDependenceResults;
@@ -261,8 +260,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoopInfo *LI = nullptr;
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
- // Prototype: SCEV handed to MemDep for affine select-address recovery.
- ScalarEvolution *SE = nullptr;
ValueTable VN;
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 7cf1c7f35879b..6fda89af4867c 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -25,8 +25,6 @@
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/PHITransAddr.h"
-#include "llvm/Analysis/ScalarEvolution.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
@@ -1058,73 +1056,6 @@ MemoryDependenceResults::lookupNonLocalPointerDepVisited(BasicBlock *BB) const {
return NonLocalPointerDepVisited[BB->getNumber()].first;
}
-// When the syntactic PHI translation of a select-dependent load address fails
-// for one arm, try to recover an equivalent, already-existing address with
-// SCEV. The arm's address is the original address \p A with the recurrence
-// phi (whose \p PredBB incoming is a select on \p Cond) replaced by the value
-// chosen by \p CondVal. If SCEV proves that address equals the SCEV of an
-// existing pointer that is loaded from and dominates \p PredBB's terminator,
-// that pointer is returned so the load-PRE-through-select path can reuse it.
-// This closes the affine-address gap (different index with compensating
-// offsets) that the syntactic operand match cannot see. Returns null on
-// failure.
-static Value *recoverSelectArmAddr(Value *A, Value *Cond, BasicBlock *PredBB,
- bool CondVal, ScalarEvolution &SE,
- const DominatorTree &DT) {
- if (!A || !SE.isSCEVable(A->getType()))
- return nullptr;
-
- const SCEV *S = SE.getSCEV(A);
-
- // Locate the recurrence phi in the address whose PredBB incoming is a select
- // on Cond, and the arm chosen by CondVal.
- PHINode *RecPhi = nullptr;
- Value *Arm = nullptr;
- SCEVExprContains(S, [&](const SCEV *Sub) {
- if (RecPhi)
- return true;
- auto *U = dyn_cast<SCEVUnknown>(Sub);
- if (!U)
- return false;
- auto *PN = dyn_cast<PHINode>(U->getValue());
- if (!PN || PN->getBasicBlockIndex(PredBB) < 0)
- return false;
- auto *SI = dyn_cast<SelectInst>(PN->getIncomingValueForBlock(PredBB));
- if (!SI || SI->getCondition() != Cond)
- return false;
- RecPhi = PN;
- Arm = CondVal ? SI->getTrueValue() : SI->getFalseValue();
- return true;
- });
- if (!RecPhi || !Arm || !SE.isSCEVable(Arm->getType()))
- return nullptr;
-
- // The "keep" arm reproduces the original address exactly.
- if (Arm == RecPhi)
- return A;
-
- ValueToSCEVMapTy Map;
- Map[RecPhi] = SE.getSCEV(Arm);
- const SCEV *Target = SCEVParameterRewriter::rewrite(S, SE, Map);
- if (Target == S)
- return nullptr;
-
- // Return an existing loaded-from pointer with a matching address SCEV.
- for (BasicBlock *BB = PredBB; BB; BB = BB->getSinglePredecessor())
- for (Instruction &I : *BB) {
- auto *LD = dyn_cast<LoadInst>(&I);
- if (!LD)
- continue;
- Value *Ptr = LD->getPointerOperand();
- if (!SE.isSCEVable(Ptr->getType()) ||
- !DT.dominates(LD, PredBB->getTerminator()))
- continue;
- if (SE.getSCEV(Ptr) == Target)
- return Ptr;
- }
- return nullptr;
-}
-
/// Perform a dependency query based on pointer/pointeesize starting at the end
/// of StartBB.
///
@@ -1447,18 +1378,6 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB(
if (Value *Cond = PredPointer.getSelectCondition()) {
SelectAddr::SelectAddrs SelAddrs =
PHITransAddr(Pointer).translateValue(BB, Pred, &DT, Cond);
- // If a side failed the syntactic match, try to recover an existing
- // affine-equal address with SCEV (opt-in via setScalarEvolution).
- if (SE && (!SelAddrs.first || !SelAddrs.second)) {
- Value *A = Pointer.getAddr();
- if (!SelAddrs.first)
- SelAddrs.first = recoverSelectArmAddr(A, Cond, Pred,
- /*CondVal=*/true, *SE, DT);
- if (!SelAddrs.second)
- SelAddrs.second =
- recoverSelectArmAddr(A, Cond, Pred,
- /*CondVal=*/false, *SE, DT);
- }
if (SelAddrs.first && SelAddrs.second) {
Result.push_back(NonLocalDepResult(Pred, MemDepResult::getSelect(),
SelectAddr(Cond, SelAddrs)));
diff --git a/llvm/lib/Analysis/PHITransAddr.cpp b/llvm/lib/Analysis/PHITransAddr.cpp
index 03b931a2f0587..203a2d4050d00 100644
--- a/llvm/lib/Analysis/PHITransAddr.cpp
+++ b/llvm/lib/Analysis/PHITransAddr.cpp
@@ -15,12 +15,15 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
+using namespace llvm::PatternMatch;
static cl::opt<bool> EnableAddPhiTranslation(
"gvn-add-phi-translation", cl::init(false), cl::Hidden,
@@ -37,6 +40,31 @@ static bool canPHITrans(Instruction *Inst) {
return false;
}
+/// Return an existing GEP of \p SrcTy over \p Ops that dominates \p PredBB.
+static GetElementPtrInst *findAvailableGEP(Type *ResultTy, Type *SrcTy,
+ ArrayRef<Value *> Ops,
+ BasicBlock *CurBB,
+ BasicBlock *PredBB,
+ const DominatorTree *DT) {
+ assert(!Ops.empty() && "GEP needs a pointer operand");
+ Value *Ptr = Ops[0];
+ if (isa<ConstantData>(Ptr))
+ return nullptr;
+
+ for (User *U : Ptr->users()) {
+ auto *GEPI = dyn_cast<GetElementPtrInst>(U);
+ if (!GEPI || GEPI->getType() != ResultTy ||
+ GEPI->getSourceElementType() != SrcTy ||
+ GEPI->getNumOperands() != Ops.size() ||
+ GEPI->getParent()->getParent() != CurBB->getParent() ||
+ (DT && !DT->dominates(GEPI->getParent(), PredBB)))
+ continue;
+ if (std::equal(Ops.begin(), Ops.end(), GEPI->op_begin()))
+ return GEPI;
+ }
+ return nullptr;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void PHITransAddr::dump() const {
if (!Addr) {
@@ -236,22 +264,8 @@ Value *PHITransAddr::translateSubExpr(Value *V, BasicBlock *CurBB,
}
// Scan to see if we have this GEP available.
- Value *APHIOp = GEPOps[0];
- if (isa<ConstantData>(APHIOp))
- return nullptr;
-
- for (User *U : APHIOp->users()) {
- if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U))
- if (GEPI->getType() == GEP->getType() &&
- GEPI->getSourceElementType() == GEP->getSourceElementType() &&
- GEPI->getNumOperands() == GEPOps.size() &&
- GEPI->getParent()->getParent() == CurBB->getParent() &&
- (!DT || DT->dominates(GEPI->getParent(), PredBB))) {
- if (std::equal(GEPOps.begin(), GEPOps.end(), GEPI->op_begin()))
- return GEPI;
- }
- }
- return nullptr;
+ return findAvailableGEP(GEP->getType(), GEP->getSourceElementType(), GEPOps,
+ CurBB, PredBB, DT);
}
// Handle add with a constant RHS.
@@ -347,12 +361,70 @@ SelectAddr::SelectAddrs PHITransAddr::translateValue(BasicBlock *CurBB,
// Work on a copy so that the original address state is preserved and the
// other side can be translated independently.
PHITransAddr Tmp(*this);
- return Tmp.translateSubExpr(Tmp.Addr, CurBB, PredBB, DT, Cond, CondVal);
+ if (Value *V =
+ Tmp.translateSubExpr(Tmp.Addr, CurBB, PredBB, DT, Cond, CondVal))
+ return V;
+ // Syntactic match misses when a constant index delta is folded into an i8
+ // offset. Recover the affine-equal available pointer instead.
+ return findAvailableSelectArmAddr(Cond, CurBB, PredBB, DT, CondVal);
};
return {TranslateSide(/*CondVal=*/true), TranslateSide(/*CondVal=*/false)};
}
+Value *PHITransAddr::findAvailableSelectArmAddr(Value *Cond, BasicBlock *CurBB,
+ BasicBlock *PredBB,
+ const DominatorTree *DT,
+ bool CondVal) const {
+ // Addr must be: gep i8, (gep Ty, Base, Index), Off.
+ auto *Outer = dyn_cast<GetElementPtrInst>(Addr);
+ if (!Outer || Outer->getNumIndices() != 1 ||
+ !Outer->getSourceElementType()->isIntegerTy(8))
+ return nullptr;
+ auto *OffCI = dyn_cast<ConstantInt>(Outer->getOperand(1));
+ auto *Inner = dyn_cast<GetElementPtrInst>(Outer->getPointerOperand());
+ if (!OffCI || !Inner || Inner->getNumIndices() != 1)
+ return nullptr;
+
+ auto *RecPhi = dyn_cast<PHINode>(Inner->getOperand(1));
+ if (!RecPhi || RecPhi->getParent() != CurBB)
+ return nullptr;
+ auto *SI = dyn_cast<SelectInst>(RecPhi->getIncomingValueForBlock(PredBB));
+ if (!SI || SI->getCondition() != Cond)
+ return nullptr;
+
+ Value *Arm = CondVal ? SI->getTrueValue() : SI->getFalseValue();
+ if (Arm == RecPhi)
+ return Addr;
+
+ // Fold add(IV, C) into the outer byte offset and look for that GEP.
+ Value *IV = nullptr;
+ const APInt *C = nullptr;
+ if (!match(Arm, m_c_Add(m_Value(IV), m_APInt(C))))
+ return nullptr;
+
+ Type *Ty = Inner->getSourceElementType();
+ TypeSize ElemSize = DL.getTypeAllocSize(Ty);
+ if (ElemSize.isScalable())
+ return nullptr;
+
+ unsigned BitWidth = OffCI->getBitWidth();
+ APInt TargetOff =
+ OffCI->getValue() +
+ C->sextOrTrunc(BitWidth) * APInt(BitWidth, ElemSize.getFixedValue());
+
+ Value *InnerOps[] = {Inner->getPointerOperand(), IV};
+ auto *InnerAvail =
+ findAvailableGEP(Inner->getType(), Ty, InnerOps, CurBB, PredBB, DT);
+ if (!InnerAvail)
+ return nullptr;
+
+ Value *OuterOps[] = {InnerAvail,
+ ConstantInt::get(Outer->getContext(), TargetOff)};
+ return findAvailableGEP(Outer->getType(), Outer->getSourceElementType(),
+ OuterOps, CurBB, PredBB, DT);
+}
+
Value *PHITransAddr::getSelectCondition() const {
for (Instruction *I : InstInputs)
if (auto *SI = dyn_cast<SelectInst>(I))
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 2c5725a7817dd..517d33bfba103 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -41,7 +41,6 @@
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/PHITransAddr.h"
-#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
@@ -886,11 +885,6 @@ PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
MSSA = &AM.getResult<MemorySSAAnalysis>(F);
}
auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
- // Hand SCEV to MemDep so its select-address translation can recover
- // affine-equal addresses (reuses the load-PRE-through-select path).
- SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
- if (MemDep)
- MemDep->setScalarEvolution(SE);
bool Changed = runImpl(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
MSSA ? &MSSA->getMSSA() : nullptr);
if (!Changed)
diff --git a/llvm/test/Transforms/GVN/scev-select-load-address.ll b/llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
similarity index 79%
rename from llvm/test/Transforms/GVN/scev-select-load-address.ll
rename to llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
index 2786b6759fc10..854a232cbcbf5 100644
--- a/llvm/test/Transforms/GVN/scev-select-load-address.ll
+++ b/llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
@@ -4,17 +4,11 @@
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
target triple = "aarch64-unknown-linux-gnu"
-; Down-counting floating-point argmin after IndVarSimplify has widened the index
-; recurrence to i64, as it reaches GVN in the real pipeline. Each iteration
-; reloads a[minidx] to compare against the scanned element. minidx is the latch
-; select of the previous iteration, so the reload address is select-dependent.
-; The address is a typed [4 x i8] GEP composed with a byte-offset i8 GEP, so the
-; select arm is affine-equal to the scanned-element pointer but not syntactically
-; identical: upstream's syntactic PHITransAddr match fails on that arm. MemDep
-; rewrites the recurrence phi with the selected arm's SCEV, proves the reloaded
-; address equals a previously loaded pointer, and forwards the value. The
-; a[minidx] reload should be eliminated and replaced by a running-minimum phi
-; fed by the value select.
+; Down-counting FP argmin after IndVarSimplify widening. The reload of
+; a[minidx] uses gep([4 x i8], ...) + gep(i8, Off), so the select update arm
+; is affine-equal to the scanned pointer but not syntactically identical.
+; PHITransAddr folds the constant index delta into the byte offset and GVN
+; eliminates the reload.
define i32 @fp_argmin_decreasing(ptr %a, i32 %start, i64 %tc0) {
; CHECK-LABEL: @fp_argmin_decreasing(
; CHECK-NEXT: entry:
More information about the llvm-branch-commits
mailing list