[llvm] 7781381 - [SDAG] Use BatchAAResults for querying alias analysis (AA) results (#123934)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 23 01:16:13 PST 2025
Author: Benjamin Maxwell
Date: 2025-01-23T09:16:09Z
New Revision: 778138114e9e42e28fcb51c0a38224e667a3790c
URL: https://github.com/llvm/llvm-project/commit/778138114e9e42e28fcb51c0a38224e667a3790c
DIFF: https://github.com/llvm/llvm-project/commit/778138114e9e42e28fcb51c0a38224e667a3790c.diff
LOG: [SDAG] Use BatchAAResults for querying alias analysis (AA) results (#123934)
Once we get to SelectionDAG the IR should not be changing anymore, so we
can use BatchAAResults rather than AAResults to cache AA queries.
This should be a NFC change for targets that enable AA during codegen
(such as AArch64), but also give a nice compile-time improvement in some
cases. See:
https://github.com/llvm/llvm-project/pull/123787#issuecomment-2606797041
Note: This follows Nikita's suggestion on #123787.
Added:
Modified:
llvm/include/llvm/Analysis/AliasAnalysis.h
llvm/include/llvm/CodeGen/MachineInstr.h
llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
llvm/include/llvm/CodeGen/SelectionDAG.h
llvm/include/llvm/CodeGen/SelectionDAGISel.h
llvm/lib/CodeGen/MachineInstr.cpp
llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index acc580f92b40a3..b192a9f5e65e7f 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -643,6 +643,9 @@ class BatchAAResults {
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) {
return isNoModRef(AA.getModRefInfoMask(Loc, AAQI, OrLocal));
}
+ bool pointsToConstantMemory(const Value *P, bool OrLocal = false) {
+ return pointsToConstantMemory(MemoryLocation::getBeforeOrAfter(P), OrLocal);
+ }
ModRefInfo getModRefInfoMask(const MemoryLocation &Loc,
bool IgnoreLocals = false) {
return AA.getModRefInfoMask(Loc, AAQI, IgnoreLocals);
@@ -668,6 +671,9 @@ class BatchAAResults {
MemoryLocation(V2, LocationSize::precise(1))) ==
AliasResult::MustAlias;
}
+ bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
+ return alias(LocA, LocB) == AliasResult::NoAlias;
+ }
ModRefInfo callCapturesBefore(const Instruction *I,
const MemoryLocation &MemLoc,
DominatorTree *DT) {
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index efac83d9e1c92c..109aac44b86623 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -42,6 +42,7 @@ class DILabel;
class Instruction;
class MDNode;
class AAResults;
+class BatchAAResults;
template <typename T> class ArrayRef;
class DIExpression;
class DILocalVariable;
@@ -1753,6 +1754,8 @@ class MachineInstr
/// @param AA Optional alias analysis, used to compare memory operands.
/// @param Other MachineInstr to check aliasing against.
/// @param UseTBAA Whether to pass TBAA information to alias analysis.
+ bool mayAlias(BatchAAResults *AA, const MachineInstr &Other,
+ bool UseTBAA) const;
bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const;
/// Return true if this instruction may have an ordered
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 822b06f080fa64..aaa10e684687c6 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -19,6 +19,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseMultiSet.h"
#include "llvm/ADT/identity.h"
+#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/ScheduleDAG.h"
@@ -169,7 +170,7 @@ namespace llvm {
/// Tracks the last instructions in this region using each virtual register.
VReg2SUnitOperIdxMultiMap CurrentVRegUses;
- AAResults *AAForDep = nullptr;
+ mutable std::optional<BatchAAResults> AAForDep;
/// Remember a generic side-effecting instruction as we proceed.
/// No other SU ever gets scheduled around it (except in the special
@@ -201,6 +202,13 @@ namespace llvm {
/// a means of remembering which SUs depend on which memory locations.
class Value2SUsMap;
+ /// Returns a (possibly null) pointer to the current BatchAAResults.
+ BatchAAResults *getAAForDep() const {
+ if (AAForDep.has_value())
+ return &AAForDep.value();
+ return nullptr;
+ }
+
/// Reduces maps in FIFO order, by N SUs. This is better than turning
/// every Nth memory SU into BarrierChain in buildSchedGraph(), since
/// it avoids unnecessary edges between seen SUs above the new BarrierChain,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index ba0538f7084eec..461c0c1ead16d2 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -61,7 +61,7 @@ class Type;
template <class GraphType> struct GraphTraits;
template <typename T, unsigned int N> class SmallSetVector;
template <typename T, typename Enable> struct FoldingSetTrait;
-class AAResults;
+class BatchAAResults;
class BlockAddress;
class BlockFrequencyInfo;
class Constant;
@@ -602,7 +602,8 @@ class SelectionDAG {
/// certain types of nodes together, or eliminating superfluous nodes. The
/// Level argument controls whether Combine is allowed to produce nodes and
/// types that are illegal on the target.
- void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel);
+ void Combine(CombineLevel Level, BatchAAResults *BatchAA,
+ CodeGenOptLevel OptLevel);
/// This transforms the SelectionDAG into a SelectionDAG that
/// only uses types natively supported by the target.
@@ -1202,12 +1203,14 @@ class SelectionDAG {
/* \p CI if not null is the memset call being lowered.
* \p OverrideTailCall is an optional parameter that can be used to override
* the tail call optimization decision. */
- SDValue
- getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol, bool AlwaysInline,
- const CallInst *CI, std::optional<bool> OverrideTailCall,
- MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo = AAMDNodes(), AAResults *AA = nullptr);
+ SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment, bool isVol,
+ bool AlwaysInline, const CallInst *CI,
+ std::optional<bool> OverrideTailCall,
+ MachinePointerInfo DstPtrInfo,
+ MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ BatchAAResults *BatchAA = nullptr);
/* \p CI if not null is the memset call being lowered.
* \p OverrideTailCall is an optional parameter that can be used to override
@@ -1218,7 +1221,7 @@ class SelectionDAG {
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
- AAResults *AA = nullptr);
+ BatchAAResults *BatchAA = nullptr);
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 43ba8f4c44cf9c..e9452a6dc62339 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -14,6 +14,7 @@
#ifndef LLVM_CODEGEN_SELECTIONDAGISEL_H
#define LLVM_CODEGEN_SELECTIONDAGISEL_H
+#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/SelectionDAG.h"
@@ -52,7 +53,7 @@ class SelectionDAGISel {
MachineRegisterInfo *RegInfo;
SelectionDAG *CurDAG;
std::unique_ptr<SelectionDAGBuilder> SDB;
- AAResults *AA = nullptr;
+ mutable std::optional<BatchAAResults> BatchAA;
AssumptionCache *AC = nullptr;
GCFunctionInfo *GFI = nullptr;
SSPLayoutInfo *SP = nullptr;
@@ -81,6 +82,13 @@ class SelectionDAGISel {
CodeGenOptLevel OL = CodeGenOptLevel::Default);
virtual ~SelectionDAGISel();
+ /// Returns a (possibly null) pointer to the current BatchAAResults.
+ BatchAAResults *getBatchAA() const {
+ if (BatchAA.has_value())
+ return &BatchAA.value();
+ return nullptr;
+ }
+
const TargetLowering *getTargetLowering() const { return TLI; }
void initializeAnalysisResults(MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index ef36dfc4721975..8c2fab18a24ca0 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1350,8 +1350,9 @@ bool MachineInstr::wouldBeTriviallyDead() const {
return isPHI() || isSafeToMove(SawStore);
}
-static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
- bool UseTBAA, const MachineMemOperand *MMOa,
+static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI,
+ BatchAAResults *AA, bool UseTBAA,
+ const MachineMemOperand *MMOa,
const MachineMemOperand *MMOb) {
// The following interface to AA is fashioned after DAGCombiner::isAlias and
// operates with MachineMemOperand offset with some important assumptions:
@@ -1434,7 +1435,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
}
-bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
+bool MachineInstr::mayAlias(BatchAAResults *AA, const MachineInstr &Other,
bool UseTBAA) const {
const MachineFunction *MF = getMF();
const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
@@ -1478,6 +1479,15 @@ bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
return false;
}
+bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
+ bool UseTBAA) const {
+ if (AA) {
+ BatchAAResults BAA(*AA);
+ return mayAlias(&BAA, Other, UseTBAA);
+ }
+ return mayAlias(static_cast<BatchAAResults *>(nullptr), Other, UseTBAA);
+}
+
/// hasOrderedMemoryRef - Return true if this instruction may have an ordered
/// or volatile memory reference, or if the information describing the memory
/// reference is not available. Return false if it is known to have no ordered
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index 8e3e06bf57153e..cc98c52e90ea68 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -551,7 +551,7 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
void ScheduleDAGInstrs::addChainDependency (SUnit *SUa, SUnit *SUb,
unsigned Latency) {
- if (SUa->getInstr()->mayAlias(AAForDep, *SUb->getInstr(), UseTBAA)) {
+ if (SUa->getInstr()->mayAlias(getAAForDep(), *SUb->getInstr(), UseTBAA)) {
SDep Dep(SUa, SDep::MayAliasMem);
Dep.setLatency(Latency);
SUb->addPred(Dep);
@@ -740,7 +740,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AAResults *AA,
const TargetSubtargetInfo &ST = MF.getSubtarget();
bool UseAA = EnableAASchedMI.getNumOccurrences() > 0 ? EnableAASchedMI
: ST.useAA();
- AAForDep = UseAA ? AA : nullptr;
+ if (UseAA && AA)
+ AAForDep.emplace(*AA);
BarrierChain = nullptr;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 21d5e0a1b2953d..a0c703d2df8a2f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -191,8 +191,8 @@ namespace {
/// candidate again.
DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
- // AA - Used for DAG load/store alias analysis.
- AliasAnalysis *AA;
+ // BatchAA - Used for DAG load/store alias analysis.
+ BatchAAResults *BatchAA;
/// This caches all chains that have already been processed in
/// DAGCombiner::getStoreMergeCandidates() and found to have no mergeable
@@ -247,9 +247,10 @@ namespace {
SDValue visit(SDNode *N);
public:
- DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOptLevel OL)
+ DAGCombiner(SelectionDAG &D, BatchAAResults *BatchAA, CodeGenOptLevel OL)
: DAG(D), TLI(D.getTargetLoweringInfo()),
- STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
+ STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL),
+ BatchAA(BatchAA) {
ForCodeSize = DAG.shouldOptForSize();
DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel);
@@ -28918,7 +28919,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
UseAA = false;
#endif
- if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
+ if (UseAA && BatchAA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
Size0.hasValue() && Size1.hasValue() &&
// Can't represent a scalable size + fixed offset in LocationSize
(!Size0.isScalable() || SrcValOffset0 == 0) &&
@@ -28933,7 +28934,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
LocationSize Loc1 =
Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
- if (AA->isNoAlias(
+ if (BatchAA->isNoAlias(
MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
MemoryLocation(MUC1.MMO->getValue(), Loc1,
@@ -29239,8 +29240,8 @@ bool DAGCombiner::findBetterNeighborChains(StoreSDNode *St) {
}
/// This is the entry point for the file.
-void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis *AA,
+void SelectionDAG::Combine(CombineLevel Level, BatchAAResults *BatchAA,
CodeGenOptLevel OptLevel) {
/// This is the main entry point to this class.
- DAGCombiner(*this, AA, OptLevel).Run(Level);
+ DAGCombiner(*this, BatchAA, OptLevel).Run(Level);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index 26eba4b257fb9c..fd4641ec6f124e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -118,7 +118,7 @@ void ScheduleDAGFast::Schedule() {
LiveRegCycles.resize(TRI->getNumRegs(), 0);
// Build the scheduling graph.
- BuildSchedGraph(nullptr);
+ BuildSchedGraph();
LLVM_DEBUG(dump());
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 51ee3cc681f05b..436c42f7e18fa9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -370,7 +370,7 @@ void ScheduleDAGRRList::Schedule() {
assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences");
// Build the scheduling graph.
- BuildSchedGraph(nullptr);
+ BuildSchedGraph();
LLVM_DEBUG(dump());
Topo.MarkDirty();
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index ac6c44ec635451..d04bd6e98097ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -536,7 +536,7 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
/// are input. This SUnit graph is similar to the SelectionDAG, but
/// excludes nodes that aren't interesting to scheduling, and represents
/// glued together nodes with a single SUnit.
-void ScheduleDAGSDNodes::BuildSchedGraph(AAResults *AA) {
+void ScheduleDAGSDNodes::BuildSchedGraph() {
// Cluster certain nodes which should be scheduled together.
ClusterNodes();
// Populate the SUnits array.
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index b7d25c6ccc9b06..ff5615b7658f37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -94,7 +94,7 @@ class InstrItineraryData;
/// are input. This SUnit graph is similar to the SelectionDAG, but
/// excludes nodes that aren't interesting to scheduling, and represents
/// flagged together nodes with a single SUnit.
- void BuildSchedGraph(AAResults *AA);
+ void BuildSchedGraph();
/// InitNumRegDefsLeft - Determine the # of regs defined by this node.
///
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
index ae42a870ea2fe9..def0f9589f3f37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
@@ -59,14 +59,10 @@ class ScheduleDAGVLIW : public ScheduleDAGSDNodes {
/// HazardRec - The hazard recognizer to use.
ScheduleHazardRecognizer *HazardRec;
- /// AA - AAResults for making memory reference queries.
- AAResults *AA;
-
public:
- ScheduleDAGVLIW(MachineFunction &mf, AAResults *aa,
- SchedulingPriorityQueue *availqueue)
- : ScheduleDAGSDNodes(mf), AvailableQueue(availqueue), AA(aa) {
- const TargetSubtargetInfo &STI = mf.getSubtarget();
+ ScheduleDAGVLIW(MachineFunction &MF, SchedulingPriorityQueue *AvailableQueue)
+ : ScheduleDAGSDNodes(MF), AvailableQueue(AvailableQueue) {
+ const TargetSubtargetInfo &STI = MF.getSubtarget();
HazardRec = STI.getInstrInfo()->CreateTargetHazardRecognizer(&STI, this);
}
@@ -91,7 +87,7 @@ void ScheduleDAGVLIW::Schedule() {
<< " '" << BB->getName() << "' **********\n");
// Build the scheduling graph.
- BuildSchedGraph(AA);
+ BuildSchedGraph();
AvailableQueue->initNodes(SUnits);
@@ -267,5 +263,5 @@ void ScheduleDAGVLIW::listScheduleTopDown() {
/// createVLIWDAGScheduler - This creates a top-down list scheduler.
ScheduleDAGSDNodes *llvm::createVLIWDAGScheduler(SelectionDAGISel *IS,
CodeGenOptLevel) {
- return new ScheduleDAGVLIW(*IS->MF, IS->AA, new ResourcePriorityQueue(IS));
+ return new ScheduleDAGVLIW(*IS->MF, new ResourcePriorityQueue(IS));
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 743ae4895a1b1c..0f9790a10a1397 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8126,13 +8126,11 @@ static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
}
}
-static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
- SDValue Chain, SDValue Dst, SDValue Src,
- uint64_t Size, Align Alignment,
- bool isVol, bool AlwaysInline,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo, AAResults *AA) {
+static SDValue getMemcpyLoadsAndStores(
+ SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
+ uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
+ MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
// Turn a memcpy of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -8198,8 +8196,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
bool isConstant =
- AA && SrcVal &&
- AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
+ BatchAA && SrcVal &&
+ BatchAA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
MachineMemOperand::Flags MMOFlags =
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
@@ -8584,7 +8582,8 @@ SDValue SelectionDAG::getMemcpy(
SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) {
+ MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
+ BatchAAResults *BatchAA) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -8595,7 +8594,7 @@ SDValue SelectionDAG::getMemcpy(
SDValue Result = getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
- isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
+ isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
if (Result.getNode())
return Result;
}
@@ -8616,7 +8615,7 @@ SDValue SelectionDAG::getMemcpy(
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
- isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
+ isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
}
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
@@ -8711,7 +8710,8 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo, AAResults *AA) {
+ const AAMDNodes &AAInfo,
+ BatchAAResults *BatchAA) {
// Check to see if we should lower the memmove to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9f1aadcb279a99..700eb26ca02a83 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1082,10 +1082,10 @@ RegsForValue::getRegsAndSizes() const {
return OutVec;
}
-void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
+void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
AssumptionCache *ac,
const TargetLibraryInfo *li) {
- AA = aa;
+ BatchAA = aa;
AC = ac;
GFI = gfi;
LibInfo = li;
@@ -4585,8 +4585,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
Root = getRoot();
else if (NumValues > MaxParallelChains)
Root = getMemoryRoot();
- else if (AA &&
- AA->pointsToConstantMemory(MemoryLocation(
+ else if (BatchAA &&
+ BatchAA->pointsToConstantMemory(MemoryLocation(
SV,
LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
AAInfo))) {
@@ -4688,8 +4688,8 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
const Value *SV = I.getOperand(0);
Type *Ty = I.getType();
assert(
- (!AA ||
- !AA->pointsToConstantMemory(MemoryLocation(
+ (!BatchAA ||
+ !BatchAA->pointsToConstantMemory(MemoryLocation(
SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
I.getAAMetadata()))) &&
"load_from_swift_error should not be constant memory");
@@ -4998,7 +4998,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
// Do not serialize masked loads of constant memory with anything.
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
- bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
+ bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
@@ -6534,7 +6534,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
/* AlwaysInline */ false, &I, std::nullopt,
MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata(), AA);
+ I.getAAMetadata(), BatchAA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6555,7 +6555,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
/* AlwaysInline */ true, &I, std::nullopt,
MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata(), AA);
+ I.getAAMetadata(), BatchAA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6608,7 +6608,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
/* OverrideTailCall */ std::nullopt,
MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata(), AA);
+ I.getAAMetadata(), BatchAA);
updateDAGForMaybeTailCall(MM);
return;
}
@@ -8435,7 +8435,7 @@ void SelectionDAGBuilder::visitVPLoad(
if (!Alignment)
Alignment = DAG.getEVTAlign(VT);
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
- bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
+ bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
@@ -8564,7 +8564,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = getRangeMetadata(VPIntrin);
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
- bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
+ bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
@@ -9021,7 +9021,7 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
bool ConstantMemory = false;
// Do not serialize (non-volatile) loads of constant memory with anything.
- if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
+ if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
Root = Builder.DAG.getEntryNode();
ConstantMemory = true;
} else {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 3a8dc25e98700e..ed85deef64fa79 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -225,7 +225,7 @@ class SelectionDAGBuilder {
static const unsigned LowestSDNodeOrder = 1;
SelectionDAG &DAG;
- AAResults *AA = nullptr;
+ BatchAAResults *BatchAA = nullptr;
AssumptionCache *AC = nullptr;
const TargetLibraryInfo *LibInfo = nullptr;
@@ -280,7 +280,7 @@ class SelectionDAGBuilder {
SL(std::make_unique<SDAGSwitchLowering>(this, funcinfo)),
FuncInfo(funcinfo), SwiftError(swifterror) {}
- void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC,
+ void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC,
const TargetLibraryInfo *li);
/// Clear out the current SelectionDAG and the associated state and prepare
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index b416e98fe61a8b..3b1abf7f3d994f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -502,9 +502,9 @@ void SelectionDAGISel::initializeAnalysisResults(
FuncInfo->BPI = nullptr;
if (OptLevel != CodeGenOptLevel::None)
- AA = &FAM.getResult<AAManager>(Fn);
+ BatchAA.emplace(FAM.getResult<AAManager>(Fn));
else
- AA = nullptr;
+ BatchAA = std::nullopt;
SP = &FAM.getResult<SSPLayoutAnalysis>(Fn);
@@ -560,9 +560,9 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
FuncInfo->BPI = nullptr;
if (OptLevel != CodeGenOptLevel::None)
- AA = &MFP.getAnalysis<AAResultsWrapperPass>().getAAResults();
+ BatchAA.emplace(MFP.getAnalysis<AAResultsWrapperPass>().getAAResults());
else
- AA = nullptr;
+ BatchAA = std::nullopt;
SP = &MFP.getAnalysis<StackProtector>().getLayoutInfo();
@@ -581,7 +581,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
ISEL_DUMP(dbgs() << "\n\n\n=== " << FuncName << '\n');
- SDB->init(GFI, AA, AC, LibInfo);
+ SDB->init(GFI, getBatchAA(), AC, LibInfo);
MF->setHasInlineAsm(false);
@@ -955,7 +955,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
{
NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
GroupDescription, TimePassesIsEnabled);
- CurDAG->Combine(BeforeLegalizeTypes, AA, OptLevel);
+ CurDAG->Combine(BeforeLegalizeTypes, getBatchAA(), OptLevel);
}
ISEL_DUMP(dbgs() << "\nOptimized lowered selection DAG: "
@@ -1001,7 +1001,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
{
NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
GroupName, GroupDescription, TimePassesIsEnabled);
- CurDAG->Combine(AfterLegalizeTypes, AA, OptLevel);
+ CurDAG->Combine(AfterLegalizeTypes, getBatchAA(), OptLevel);
}
ISEL_DUMP(dbgs() << "\nOptimized type-legalized selection DAG: "
@@ -1055,7 +1055,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
{
NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
GroupName, GroupDescription, TimePassesIsEnabled);
- CurDAG->Combine(AfterLegalizeVectorOps, AA, OptLevel);
+ CurDAG->Combine(AfterLegalizeVectorOps, getBatchAA(), OptLevel);
}
ISEL_DUMP(dbgs() << "\nOptimized vector-legalized selection DAG: "
@@ -1095,7 +1095,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
{
NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
GroupDescription, TimePassesIsEnabled);
- CurDAG->Combine(AfterLegalizeDAG, AA, OptLevel);
+ CurDAG->Combine(AfterLegalizeDAG, getBatchAA(), OptLevel);
}
ISEL_DUMP(dbgs() << "\nOptimized legalized selection DAG: "
diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
index 3d90e3f6f67817..caf01ccd1ef7c0 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
@@ -1498,8 +1498,8 @@ bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
if (V1 == V2 && End1 == End2)
return false;
- return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
- MemoryLocation(V2, End2, Store->getAAInfo()));
+ return BatchAA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
+ MemoryLocation(V2, End2, Store->getAAInfo()));
}
bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
More information about the llvm-commits
mailing list