[llvm] [DependenceAnalysis] Extending SIV to handle separate loops (PR #128782)
Alireza Torabian via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 10 14:01:09 PDT 2025
https://github.com/1997alireza updated https://github.com/llvm/llvm-project/pull/128782
>From 3cfd2e0fd475ada4c916c39ba965bf331a920c92 Mon Sep 17 00:00:00 2001
From: Alireza Torabian <alireza.torabian at huawei.com>
Date: Mon, 24 Feb 2025 11:53:53 -0500
Subject: [PATCH] [DependenceAnalysis] Extending SIV to handle separate loops
When there is a dependency between two memory instructions in separate
loops, SIV will be able to test them and compute the direction and
the distance of the dependency.
---
.../llvm/Analysis/DependenceAnalysis.h | 159 ++++--
llvm/lib/Analysis/DependenceAnalysis.cpp | 477 ++++++++++++------
.../PreliminaryNoValidityCheckFixedSize.ll | 2 +-
.../DependenceAnalysis/SIVSeparateLoops.ll | 209 ++++++++
4 files changed, 655 insertions(+), 192 deletions(-)
create mode 100644 llvm/test/Analysis/DependenceAnalysis/SIVSeparateLoops.ll
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index 426ac757b4b0d..8c06a58c5aa03 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -152,13 +152,24 @@ namespace llvm {
/// source and destination of the dependence.
virtual unsigned getLevels() const { return 0; }
+ /// getSeparateLevels - Returns the number of separate loops surrounding
+ /// the source and destination of the dependence.
+ virtual unsigned getSeparateLevels() const { return 0; }
+
/// getDirection - Returns the direction associated with a particular
- /// level.
- virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; }
+ /// level. If Separate is set to true, information about a separate
+ /// level is provided.
+ virtual unsigned getDirection(unsigned Level, bool Separate = false) const {
+ return DVEntry::ALL;
+ }
/// getDistance - Returns the distance (or NULL) associated with a
- /// particular level.
- virtual const SCEV *getDistance(unsigned Level) const { return nullptr; }
+ /// particular level. If Separate is set to true, information about
+ /// a separate level is provided.
+ virtual const SCEV *getDistance(unsigned Level,
+ bool Separate = false) const {
+ return nullptr;
+ }
/// Check if the direction vector is negative. A negative direction
/// vector means Src and Dst are reversed in the actual program.
@@ -171,21 +182,35 @@ namespace llvm {
virtual bool normalize(ScalarEvolution *SE) { return false; }
/// isPeelFirst - Returns true if peeling the first iteration from
- /// this loop will break this dependence.
- virtual bool isPeelFirst(unsigned Level) const { return false; }
+ /// this loop will break this dependence. If Separate is set to true,
+ /// information about a separate level is provided.
+ virtual bool isPeelFirst(unsigned Level, bool Separate = false) const {
+ return false;
+ }
/// isPeelLast - Returns true if peeling the last iteration from
- /// this loop will break this dependence.
- virtual bool isPeelLast(unsigned Level) const { return false; }
+ /// this loop will break this dependence. If Separate is set to true,
+ /// information about a separate level is provided.
+ virtual bool isPeelLast(unsigned Level, bool Separate = false) const {
+ return false;
+ }
/// isSplitable - Returns true if splitting this loop will break
- /// the dependence.
- virtual bool isSplitable(unsigned Level) const { return false; }
+ /// the dependence. If Separate is set to true, information about a
+ /// separate level is provided.
+ virtual bool isSplitable(unsigned Level, bool Separate = false) const {
+ return false;
+ }
+
+ /// inSeparateLoops - Returns true if this level is performed across
+ /// two separate loop nests.
+ virtual bool inSeparateLoops(unsigned Level) const { return false; }
/// isScalar - Returns true if a particular level is scalar; that is,
/// if no subscript in the source or destination mention the induction
- /// variable associated with the loop at this level.
- virtual bool isScalar(unsigned Level) const;
+ /// variable associated with the loop at this level. If Separate is
+ /// set to true, information about a separate level is provided.
+ virtual bool isScalar(unsigned Level, bool Separate = false) const;
/// getNextPredecessor - Returns the value of the NextPredecessor
/// field.
@@ -245,13 +270,20 @@ namespace llvm {
/// source and destination of the dependence.
unsigned getLevels() const override { return Levels; }
+ /// getSeparateLevels - Returns the number of separate loops surrounding
+ /// the source and destination of the dependence.
+ unsigned getSeparateLevels() const override { return SeparateLevels; }
+
/// getDirection - Returns the direction associated with a particular
- /// level.
- unsigned getDirection(unsigned Level) const override;
+ /// level. If Separate is set to true, information about a separate
+ /// level is provided.
+ unsigned getDirection(unsigned Level, bool Separate = false) const override;
/// getDistance - Returns the distance (or NULL) associated with a
- /// particular level.
- const SCEV *getDistance(unsigned Level) const override;
+ /// particular level. If Separate is set to true, information about
+ /// a separate level is provided.
+ const SCEV *getDistance(unsigned Level,
+ bool Separate = false) const override;
/// Check if the direction vector is negative. A negative direction
/// vector means Src and Dst are reversed in the actual program.
@@ -264,27 +296,37 @@ namespace llvm {
bool normalize(ScalarEvolution *SE) override;
/// isPeelFirst - Returns true if peeling the first iteration from
- /// this loop will break this dependence.
- bool isPeelFirst(unsigned Level) const override;
+ /// this loop will break this dependence. If Separate is set to true,
+ /// information about a separate level is provided.
+ bool isPeelFirst(unsigned Level, bool Separate = false) const override;
/// isPeelLast - Returns true if peeling the last iteration from
- /// this loop will break this dependence.
- bool isPeelLast(unsigned Level) const override;
+ /// this loop will break this dependence. If Separate is set to true,
+ /// information about a separate level is provided.
+ bool isPeelLast(unsigned Level, bool Separate = false) const override;
/// isSplitable - Returns true if splitting the loop will break
- /// the dependence.
- bool isSplitable(unsigned Level) const override;
+ /// the dependence. If Separate is set to true, information about a
+ /// separate level is provided.
+ bool isSplitable(unsigned Level, bool Separate = false) const override;
+
+ /// inSeparateLoops - Returns true if this level is performed across
+ /// two separate loop nests.
+ bool inSeparateLoops(unsigned Level) const override;
/// isScalar - Returns true if a particular level is scalar; that is,
/// if no subscript in the source or destination mention the induction
- /// variable associated with the loop at this level.
- bool isScalar(unsigned Level) const override;
+ /// variable associated with the loop at this level. If Separate is
+ /// set to true, information about a separate level is provided.
+ bool isScalar(unsigned Level, bool Separate = false) const override;
private:
unsigned short Levels;
+ unsigned short SeparateLevels;
bool LoopIndependent;
bool Consistent; // Init to true, then refine.
std::unique_ptr<DVEntry[]> DV;
+ std::unique_ptr<DVEntry[]> DVSeparate;
friend class DependenceInfo;
};
@@ -405,7 +447,8 @@ namespace llvm {
const SCEV *A;
const SCEV *B;
const SCEV *C;
- const Loop *AssociatedLoop;
+ const Loop *AssociatedSrcLoop;
+ const Loop *AssociatedDstLoop;
public:
/// isEmpty - Return true if the constraint is of kind Empty.
@@ -449,18 +492,25 @@ namespace llvm {
/// Otherwise assert.
const SCEV *getD() const;
- /// getAssociatedLoop - Returns the loop associated with this constraint.
- const Loop *getAssociatedLoop() const;
+ /// getAssociatedSrcLoop - Returns the source loop associated with this
+ /// constraint.
+ const Loop *getAssociatedSrcLoop() const;
+
+ /// getAssociatedDstLoop - Returns the destination loop associated with
+ /// this constraint.
+ const Loop *getAssociatedDstLoop() const;
/// setPoint - Change a constraint to Point.
- void setPoint(const SCEV *X, const SCEV *Y, const Loop *CurrentLoop);
+ void setPoint(const SCEV *X, const SCEV *Y, const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop);
/// setLine - Change a constraint to Line.
- void setLine(const SCEV *A, const SCEV *B,
- const SCEV *C, const Loop *CurrentLoop);
+ void setLine(const SCEV *A, const SCEV *B, const SCEV *C,
+ const Loop *CurrentSrcLoop, const Loop *CurrentDstLoop);
/// setDistance - Change a constraint to Distance.
- void setDistance(const SCEV *D, const Loop *CurrentLoop);
+ void setDistance(const SCEV *D, const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop);
/// setEmpty - Change a constraint to Empty.
void setEmpty();
@@ -473,6 +523,10 @@ namespace llvm {
void dump(raw_ostream &OS) const;
};
+ /// Returns true if two loops are the same or they have the same tripcount
+ /// and depth
+ bool areLoopsSimilar(const Loop *SrcLoop, const Loop *DstLoop) const;
+
/// establishNestingLevels - Examines the loop nesting of the Src and Dst
/// instructions and establishes their shared loops. Sets the variables
/// CommonLevels, SrcLevels, and MaxLevels.
@@ -523,10 +577,30 @@ namespace llvm {
/// e - 5
/// f - 6
/// g - 7 = MaxLevels
- void establishNestingLevels(const Instruction *Src,
- const Instruction *Dst);
-
- unsigned CommonLevels, SrcLevels, MaxLevels;
+ /// If ConsiderSeparateLoops is true then we also want to consider similar
+ /// seperate loops. Assume that loop nests at level c and e are similar,
+ /// meaning that they have the same tripcount and depth. Then we consider
+ /// them as a separate common level.
+ /// a - 1
+ /// b - 2
+ /// <c, e> - 3 = CommonLevels
+ /// d - 4 = SrcLevels
+ /// f - 5
+ /// g - 6 = MaxLevels
+ /// Initially similar separate levels are included in common levels. After
+ /// the separate level extraction at the end of the depends api we have
+ /// a - 1
+ /// b - 2 = CommonLevels
+ /// <c, e> - 3 : A SeparateLevel
+ /// d - 4 = SrcLevels
+ /// f - 6
+ /// g - 7 = MaxLevels
+ /// SeparateLevels is the number of levels after CommonLevels that are
+ /// similar, which is 1 in this case.
+ void establishNestingLevels(const Instruction *Src, const Instruction *Dst,
+ bool ConsiderSeparateLoops = false);
+
+ unsigned CommonLevels, SrcLevels, MaxLevels, SeparateLevels;
/// mapSrcLoop - Given one of the loops containing the source, return
/// its level index in our numbering scheme.
@@ -668,7 +742,8 @@ namespace llvm {
bool strongSIVtest(const SCEV *Coeff,
const SCEV *SrcConst,
const SCEV *DstConst,
- const Loop *CurrentLoop,
+ const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop,
unsigned Level,
FullDependence &Result,
Constraint &NewConstraint) const;
@@ -686,7 +761,8 @@ namespace llvm {
bool weakCrossingSIVtest(const SCEV *SrcCoeff,
const SCEV *SrcConst,
const SCEV *DstConst,
- const Loop *CurrentLoop,
+ const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop,
unsigned Level,
FullDependence &Result,
Constraint &NewConstraint,
@@ -705,7 +781,8 @@ namespace llvm {
const SCEV *DstCoeff,
const SCEV *SrcConst,
const SCEV *DstConst,
- const Loop *CurrentLoop,
+ const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop,
unsigned Level,
FullDependence &Result,
Constraint &NewConstraint) const;
@@ -723,7 +800,8 @@ namespace llvm {
bool weakZeroSrcSIVtest(const SCEV *DstCoeff,
const SCEV *SrcConst,
const SCEV *DstConst,
- const Loop *CurrentLoop,
+ const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop,
unsigned Level,
FullDependence &Result,
Constraint &NewConstraint) const;
@@ -741,7 +819,8 @@ namespace llvm {
bool weakZeroDstSIVtest(const SCEV *SrcCoeff,
const SCEV *SrcConst,
const SCEV *DstConst,
- const Loop *CurrentLoop,
+ const Loop *CurrentSrcLoop,
+ const Loop *CurrentDstLoop,
unsigned Level,
FullDependence &Result,
Constraint &NewConstraint) const;
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index dc0ed22dbcc0b..9f493574b168b 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -104,6 +104,7 @@ STATISTIC(GCDindependence, "GCD independence");
STATISTIC(BanerjeeApplications, "Banerjee applications");
STATISTIC(BanerjeeIndependence, "Banerjee independence");
STATISTIC(BanerjeeSuccesses, "Banerjee successes");
+STATISTIC(SeparateLoopsConsidered, "Separate loops considered");
static cl::opt<bool>
Delinearize("da-delinearize", cl::init(true), cl::Hidden,
@@ -255,7 +256,7 @@ bool Dependence::isAnti() const {
// if no subscript in the source or destination mention the induction
// variable associated with the loop at this level.
// Leave this out of line, so it will serve as a virtual method anchor
-bool Dependence::isScalar(unsigned level) const {
+bool Dependence::isScalar(unsigned level, bool Separate) const {
return false;
}
@@ -332,50 +333,92 @@ bool FullDependence::normalize(ScalarEvolution *SE) {
// The rest are simple getters that hide the implementation.
// getDirection - Returns the direction associated with a particular level.
-unsigned FullDependence::getDirection(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].Direction;
+// If Separate is set to true, information about a separate level is provided.
+unsigned FullDependence::getDirection(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].Direction;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].Direction;
+ }
}
-
-// Returns the distance (or NULL) associated with a particular level.
-const SCEV *FullDependence::getDistance(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].Distance;
+// Returns the distance (or NULL) associated with a particular level. If
+// Separate is set to true, information about a separate level is provided.
+const SCEV *FullDependence::getDistance(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].Distance;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].Distance;
+ }
}
-
// Returns true if a particular level is scalar; that is,
// if no subscript in the source or destination mention the induction
-// variable associated with the loop at this level.
-bool FullDependence::isScalar(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].Scalar;
+// variable associated with the loop at this level. If Separate is set
+// to true, information about a separate level is provided.
+bool FullDependence::isScalar(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].Scalar;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].Scalar;
+ }
}
-
// Returns true if peeling the first iteration from this loop
-// will break this dependence.
-bool FullDependence::isPeelFirst(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].PeelFirst;
+// will break this dependence. If Separate is set to true, information
+// about a separate level is provided.
+bool FullDependence::isPeelFirst(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].PeelFirst;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].PeelFirst;
+ }
}
-
// Returns true if peeling the last iteration from this loop
-// will break this dependence.
-bool FullDependence::isPeelLast(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].PeelLast;
+// will break this dependence. If Separate is set to true, information
+// about a separate level is provided.
+bool FullDependence::isPeelLast(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].PeelLast;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].PeelLast;
+ }
}
-
-// Returns true if splitting this loop will break the dependence.
-bool FullDependence::isSplitable(unsigned Level) const {
- assert(0 < Level && Level <= Levels && "Level out of range");
- return DV[Level - 1].Splitable;
+// Returns true if splitting this loop will break the dependence. If
+// Separate is set to true, information about a separate level is provided.
+bool FullDependence::isSplitable(unsigned Level, bool Separate) const {
+ if (!Separate) {
+ assert(0 < Level && Level <= Levels && "Level out of range");
+ return DV[Level - 1].Splitable;
+ } else {
+ assert(Levels < Level && Level <= Levels + SeparateLevels &&
+ "Separate level out of range");
+ return DVSeparate[Level - Levels - 1].Splitable;
+ }
}
+// Returns true if this level is performed across two separate loop nests.
+bool FullDependence::inSeparateLoops(unsigned Level) const {
+ assert(0 < Level && Level <= Levels + SeparateLevels && "Level out of range");
+ return Level > Levels;
+}
//===----------------------------------------------------------------------===//
// DependenceInfo::Constraint methods
@@ -430,38 +473,50 @@ const SCEV *DependenceInfo::Constraint::getD() const {
return SE->getNegativeSCEV(C);
}
+// Returns the source loop associated with this constraint.
+const Loop *DependenceInfo::Constraint::getAssociatedSrcLoop() const {
+ assert((Kind == Distance || Kind == Line || Kind == Point) &&
+ "Kind should be Distance, Line, or Point");
+ return AssociatedSrcLoop;
+}
-// Returns the loop associated with this constraint.
-const Loop *DependenceInfo::Constraint::getAssociatedLoop() const {
+// Returns the destination loop associated with this constraint.
+const Loop *DependenceInfo::Constraint::getAssociatedDstLoop() const {
assert((Kind == Distance || Kind == Line || Kind == Point) &&
"Kind should be Distance, Line, or Point");
- return AssociatedLoop;
+ return AssociatedDstLoop;
}
void DependenceInfo::Constraint::setPoint(const SCEV *X, const SCEV *Y,
- const Loop *CurLoop) {
+ const Loop *CurSrcLoop,
+ const Loop *CurDstLoop) {
Kind = Point;
A = X;
B = Y;
- AssociatedLoop = CurLoop;
+ AssociatedSrcLoop = CurSrcLoop;
+ AssociatedDstLoop = CurDstLoop;
}
void DependenceInfo::Constraint::setLine(const SCEV *AA, const SCEV *BB,
- const SCEV *CC, const Loop *CurLoop) {
+ const SCEV *CC, const Loop *CurSrcLoop,
+ const Loop *CurDstLoop) {
Kind = Line;
A = AA;
B = BB;
C = CC;
- AssociatedLoop = CurLoop;
+ AssociatedSrcLoop = CurSrcLoop;
+ AssociatedDstLoop = CurDstLoop;
}
void DependenceInfo::Constraint::setDistance(const SCEV *D,
- const Loop *CurLoop) {
+ const Loop *CurSrcLoop,
+ const Loop *CurDstLoop) {
Kind = Distance;
A = SE->getOne(D->getType());
B = SE->getNegativeSCEV(A);
C = SE->getNegativeSCEV(D);
- AssociatedLoop = CurLoop;
+ AssociatedSrcLoop = CurSrcLoop;
+ AssociatedDstLoop = CurDstLoop;
}
void DependenceInfo::Constraint::setEmpty() { Kind = Empty; }
@@ -608,8 +663,8 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
++DeltaSuccesses;
return true;
}
- if (const SCEVConstant *CUB =
- collectConstantUpperBound(X->getAssociatedLoop(), Prod1->getType())) {
+ if (const SCEVConstant *CUB = collectConstantUpperBound(
+ X->getAssociatedSrcLoop(), Prod1->getType())) {
const APInt &UpperBound = CUB->getAPInt();
LLVM_DEBUG(dbgs() << "\t\tupper bound = " << UpperBound << "\n");
if (Xq.sgt(UpperBound) || Yq.sgt(UpperBound)) {
@@ -618,9 +673,8 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
return true;
}
}
- X->setPoint(SE->getConstant(Xq),
- SE->getConstant(Yq),
- X->getAssociatedLoop());
+ X->setPoint(SE->getConstant(Xq), SE->getConstant(Yq),
+ X->getAssociatedSrcLoop(), X->getAssociatedDstLoop());
++DeltaSuccesses;
return true;
}
@@ -656,6 +710,7 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
// For debugging purposes. Dumps a dependence to OS.
void Dependence::dump(raw_ostream &OS) const {
bool Splitable = false;
+ bool SeparatesStarted = false;
if (isConfused())
OS << "confused";
else {
@@ -670,19 +725,24 @@ void Dependence::dump(raw_ostream &OS) const {
else if (isInput())
OS << "input";
unsigned Levels = getLevels();
+ unsigned SeparateLevels = getSeparateLevels();
OS << " [";
- for (unsigned II = 1; II <= Levels; ++II) {
- if (isSplitable(II))
+ for (unsigned II = 1; II <= Levels + SeparateLevels; ++II) {
+ if (!SeparatesStarted && inSeparateLoops(II)) {
+ SeparatesStarted = true;
+ OS << "/ ";
+ }
+ if (isSplitable(II, SeparatesStarted))
Splitable = true;
- if (isPeelFirst(II))
+ if (isPeelFirst(II, SeparatesStarted))
OS << 'p';
- const SCEV *Distance = getDistance(II);
+ const SCEV *Distance = getDistance(II, SeparatesStarted);
if (Distance)
OS << *Distance;
- else if (isScalar(II))
+ else if (isScalar(II, SeparatesStarted))
OS << "S";
else {
- unsigned Direction = getDirection(II);
+ unsigned Direction = getDirection(II, SeparatesStarted);
if (Direction == DVEntry::ALL)
OS << "*";
else {
@@ -694,9 +754,9 @@ void Dependence::dump(raw_ostream &OS) const {
OS << ">";
}
}
- if (isPeelLast(II))
+ if (isPeelLast(II, SeparatesStarted))
OS << 'p';
- if (II < Levels)
+ if (II < Levels + SeparateLevels)
OS << " ";
}
if (isLoopIndependent())
@@ -758,6 +818,34 @@ bool isLoadOrStore(const Instruction *I) {
return false;
}
+// Returns true if two loops are the same or they have the same tripcount and
+// depth
+bool DependenceInfo::areLoopsSimilar(const Loop *SrcLoop,
+ const Loop *DstLoop) const {
+ if (SrcLoop == DstLoop)
+ return true;
+
+ if (SrcLoop->getLoopDepth() != DstLoop->getLoopDepth())
+ return false;
+
+ if (!SrcLoop || !SrcLoop->getLoopLatch() || !DstLoop ||
+ !DstLoop->getLoopLatch())
+ return false;
+
+ const SCEV *SrcUB, *DstUP;
+ if (SE->hasLoopInvariantBackedgeTakenCount(SrcLoop))
+ SrcUB = SE->getBackedgeTakenCount(SrcLoop);
+ if (SE->hasLoopInvariantBackedgeTakenCount(DstLoop))
+ DstUP = SE->getBackedgeTakenCount(DstLoop);
+
+ if (SrcUB == nullptr || DstUP == nullptr)
+ return false;
+
+ if (SE->isKnownPredicate(ICmpInst::ICMP_EQ, SrcUB, DstUP))
+ return true;
+
+ return false;
+}
// Examines the loop nesting of the Src and Dst
// instructions and establishes their shared loops. Sets the variables
@@ -809,8 +897,29 @@ bool isLoadOrStore(const Instruction *I) {
// e - 5
// f - 6
// g - 7 = MaxLevels
+// If ConsiderSeparateLoops is true then we also want to consider similar
+// seperate loops. Assume that loop nests at level c and e are similar,
+// meaning that they have the same tripcount and depth. Then we consider
+// them as a separate common level.
+// a - 1
+// b - 2
+// <c, e> - 3 = CommonLevels
+// d - 4 = SrcLevels
+// f - 5
+// g - 6 = MaxLevels
+// Initially similar separate levels are included in common levels. After
+// the separate level extraction at the end of the depends api we have
+// a - 1
+// b - 2 = CommonLevels
+// <c, e> - 3 : A SeparateLevel
+// d - 4 = SrcLevels
+// f - 6
+// g - 7 = MaxLevels
+// SeparateLevels is the number of levels after CommonLevels that are
+// similar, which is 1 in this case.
void DependenceInfo::establishNestingLevels(const Instruction *Src,
- const Instruction *Dst) {
+ const Instruction *Dst,
+ bool ConsiderSeparateLoops) {
const BasicBlock *SrcBlock = Src->getParent();
const BasicBlock *DstBlock = Dst->getParent();
unsigned SrcLevel = LI->getLoopDepth(SrcBlock);
@@ -819,6 +928,7 @@ void DependenceInfo::establishNestingLevels(const Instruction *Src,
const Loop *DstLoop = LI->getLoopFor(DstBlock);
SrcLevels = SrcLevel;
MaxLevels = SrcLevel + DstLevel;
+ SeparateLevels = 0;
while (SrcLevel > DstLevel) {
SrcLoop = SrcLoop->getParentLoop();
SrcLevel--;
@@ -827,16 +937,27 @@ void DependenceInfo::establishNestingLevels(const Instruction *Src,
DstLoop = DstLoop->getParentLoop();
DstLevel--;
}
- while (SrcLoop != DstLoop) {
- SrcLoop = SrcLoop->getParentLoop();
- DstLoop = DstLoop->getParentLoop();
- SrcLevel--;
- }
+ if (ConsiderSeparateLoops) {
+ while (!areLoopsSimilar(SrcLoop, DstLoop)) {
+ SrcLoop = SrcLoop->getParentLoop();
+ DstLoop = DstLoop->getParentLoop();
+ SrcLevel--;
+ }
+ while (SrcLoop != DstLoop) {
+ SrcLoop = SrcLoop->getParentLoop();
+ DstLoop = DstLoop->getParentLoop();
+ SeparateLevels++;
+ }
+ } else
+ while (SrcLoop != DstLoop) {
+ SrcLoop = SrcLoop->getParentLoop();
+ DstLoop = DstLoop->getParentLoop();
+ SrcLevel--;
+ }
CommonLevels = SrcLevel;
MaxLevels -= CommonLevels;
}
-
// Given one of the loops containing the source, return
// its level index in our numbering scheme.
unsigned DependenceInfo::mapSrcLoop(const Loop *SrcLoop) const {
@@ -1223,8 +1344,9 @@ bool DependenceInfo::testZIV(const SCEV *Src, const SCEV *Dst,
//
// Return true if dependence disproved.
bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
- const SCEV *DstConst, const Loop *CurLoop,
- unsigned Level, FullDependence &Result,
+ const SCEV *DstConst, const Loop *CurSrcLoop,
+ const Loop *CurDstLoop, unsigned Level,
+ FullDependence &Result,
Constraint &NewConstraint) const {
LLVM_DEBUG(dbgs() << "\tStrong SIV test\n");
LLVM_DEBUG(dbgs() << "\t Coeff = " << *Coeff);
@@ -1242,7 +1364,8 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
LLVM_DEBUG(dbgs() << ", " << *Delta->getType() << "\n");
// check that |Delta| < iteration count
- if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+ if (const SCEV *UpperBound =
+ collectUpperBound(CurSrcLoop, Delta->getType())) {
LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound);
LLVM_DEBUG(dbgs() << ", " << *UpperBound->getType() << "\n");
const SCEV *AbsDelta =
@@ -1275,7 +1398,8 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
return true;
}
Result.DV[Level].Distance = SE->getConstant(Distance);
- NewConstraint.setDistance(SE->getConstant(Distance), CurLoop);
+ NewConstraint.setDistance(SE->getConstant(Distance), CurSrcLoop,
+ CurDstLoop);
if (Distance.sgt(0))
Result.DV[Level].Direction &= Dependence::DVEntry::LT;
else if (Distance.slt(0))
@@ -1287,7 +1411,7 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
else if (Delta->isZero()) {
// since 0/X == 0
Result.DV[Level].Distance = Delta;
- NewConstraint.setDistance(Delta, CurLoop);
+ NewConstraint.setDistance(Delta, CurSrcLoop, CurDstLoop);
Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
++StrongSIVsuccesses;
}
@@ -1295,13 +1419,12 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
if (Coeff->isOne()) {
LLVM_DEBUG(dbgs() << "\t Distance = " << *Delta << "\n");
Result.DV[Level].Distance = Delta; // since X/1 == X
- NewConstraint.setDistance(Delta, CurLoop);
+ NewConstraint.setDistance(Delta, CurSrcLoop, CurDstLoop);
}
else {
Result.Consistent = false;
- NewConstraint.setLine(Coeff,
- SE->getNegativeSCEV(Coeff),
- SE->getNegativeSCEV(Delta), CurLoop);
+ NewConstraint.setLine(Coeff, SE->getNegativeSCEV(Coeff),
+ SE->getNegativeSCEV(Delta), CurSrcLoop, CurDstLoop);
}
// maybe we can get a useful direction
@@ -1329,7 +1452,6 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
return false;
}
-
// weakCrossingSIVtest -
// From the paper, Practical Dependence Testing, Section 4.2.2
//
@@ -1360,8 +1482,9 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
// Return true if dependence disproved.
bool DependenceInfo::weakCrossingSIVtest(
const SCEV *Coeff, const SCEV *SrcConst, const SCEV *DstConst,
- const Loop *CurLoop, unsigned Level, FullDependence &Result,
- Constraint &NewConstraint, const SCEV *&SplitIter) const {
+ const Loop *CurSrcLoop, const Loop *CurDstLoop, unsigned Level,
+ FullDependence &Result, Constraint &NewConstraint,
+ const SCEV *&SplitIter) const {
LLVM_DEBUG(dbgs() << "\tWeak-Crossing SIV test\n");
LLVM_DEBUG(dbgs() << "\t Coeff = " << *Coeff << "\n");
LLVM_DEBUG(dbgs() << "\t SrcConst = " << *SrcConst << "\n");
@@ -1372,7 +1495,7 @@ bool DependenceInfo::weakCrossingSIVtest(
Result.Consistent = false;
const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
LLVM_DEBUG(dbgs() << "\t Delta = " << *Delta << "\n");
- NewConstraint.setLine(Coeff, Coeff, Delta, CurLoop);
+ NewConstraint.setLine(Coeff, Coeff, Delta, CurSrcLoop, CurDstLoop);
if (Delta->isZero()) {
Result.DV[Level].Direction &= ~Dependence::DVEntry::LT;
Result.DV[Level].Direction &= ~Dependence::DVEntry::GT;
@@ -1420,7 +1543,8 @@ bool DependenceInfo::weakCrossingSIVtest(
// We're certain that Delta > 0 and ConstCoeff > 0.
// Check Delta/(2*ConstCoeff) against upper loop bound
- if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+ if (const SCEV *UpperBound =
+ collectUpperBound(CurSrcLoop, Delta->getType())) {
LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound << "\n");
const SCEV *ConstantTwo = SE->getConstant(UpperBound->getType(), 2);
const SCEV *ML = SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound),
@@ -1474,7 +1598,6 @@ bool DependenceInfo::weakCrossingSIVtest(
return false;
}
-
// Kirch's algorithm, from
//
// Optimizing Supercompilers for Supercomputers
@@ -1560,7 +1683,8 @@ static APInt ceilingOfQuotient(const APInt &A, const APInt &B) {
// returns all the dependencies that exist between Dst and Src.
bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
const SCEV *SrcConst, const SCEV *DstConst,
- const Loop *CurLoop, unsigned Level,
+ const Loop *CurSrcLoop,
+ const Loop *CurDstLoop, unsigned Level,
FullDependence &Result,
Constraint &NewConstraint) const {
LLVM_DEBUG(dbgs() << "\tExact SIV test\n");
@@ -1575,7 +1699,7 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
LLVM_DEBUG(dbgs() << "\t Delta = " << *Delta << "\n");
NewConstraint.setLine(SrcCoeff, SE->getNegativeSCEV(DstCoeff), Delta,
- CurLoop);
+ CurSrcLoop, CurDstLoop);
const SCEVConstant *ConstDelta = dyn_cast<SCEVConstant>(Delta);
const SCEVConstant *ConstSrcCoeff = dyn_cast<SCEVConstant>(SrcCoeff);
const SCEVConstant *ConstDstCoeff = dyn_cast<SCEVConstant>(DstCoeff);
@@ -1602,7 +1726,7 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
bool UMValid = false;
// UM is perhaps unavailable, let's check
if (const SCEVConstant *CUB =
- collectConstantUpperBound(CurLoop, Delta->getType())) {
+ collectConstantUpperBound(CurSrcLoop, Delta->getType())) {
UM = CUB->getAPInt();
LLVM_DEBUG(dbgs() << "\t UM = " << UM << "\n");
UMValid = true;
@@ -1709,7 +1833,6 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
return Result.DV[Level].Direction == Dependence::DVEntry::NONE;
}
-
// Return true if the divisor evenly divides the dividend.
static
bool isRemainderZero(const SCEVConstant *Dividend,
@@ -1751,12 +1874,10 @@ bool isRemainderZero(const SCEVConstant *Dividend,
// (see also weakZeroDstSIVtest)
//
// Return true if dependence disproved.
-bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
- const SCEV *SrcConst,
- const SCEV *DstConst,
- const Loop *CurLoop, unsigned Level,
- FullDependence &Result,
- Constraint &NewConstraint) const {
+bool DependenceInfo::weakZeroSrcSIVtest(
+ const SCEV *DstCoeff, const SCEV *SrcConst, const SCEV *DstConst,
+ const Loop *CurSrcLoop, const Loop *CurDstLoop, unsigned Level,
+ FullDependence &Result, Constraint &NewConstraint) const {
// For the WeakSIV test, it's possible the loop isn't common to
// the Src and Dst loops. If it isn't, then there's no need to
// record a direction.
@@ -1770,7 +1891,7 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
Result.Consistent = false;
const SCEV *Delta = SE->getMinusSCEV(SrcConst, DstConst);
NewConstraint.setLine(SE->getZero(Delta->getType()), DstCoeff, Delta,
- CurLoop);
+ CurSrcLoop, CurDstLoop);
LLVM_DEBUG(dbgs() << "\t Delta = " << *Delta << "\n");
if (isKnownPredicate(CmpInst::ICMP_EQ, SrcConst, DstConst)) {
if (Level < CommonLevels) {
@@ -1791,7 +1912,8 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
// check that Delta/SrcCoeff < iteration count
// really check NewDelta < count*AbsCoeff
- if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+ if (const SCEV *UpperBound =
+ collectUpperBound(CurSrcLoop, Delta->getType())) {
LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound << "\n");
const SCEV *Product = SE->getMulExpr(AbsCoeff, UpperBound);
if (isKnownPredicate(CmpInst::ICMP_SGT, NewDelta, Product)) {
@@ -1829,7 +1951,6 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
return false;
}
-
// weakZeroDstSIVtest -
// From the paper, Practical Dependence Testing, Section 4.2.2
//
@@ -1861,12 +1982,10 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
// (see also weakZeroSrcSIVtest)
//
// Return true if dependence disproved.
-bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
- const SCEV *SrcConst,
- const SCEV *DstConst,
- const Loop *CurLoop, unsigned Level,
- FullDependence &Result,
- Constraint &NewConstraint) const {
+bool DependenceInfo::weakZeroDstSIVtest(
+ const SCEV *SrcCoeff, const SCEV *SrcConst, const SCEV *DstConst,
+ const Loop *CurSrcLoop, const Loop *CurDstLoop, unsigned Level,
+ FullDependence &Result, Constraint &NewConstraint) const {
// For the WeakSIV test, it's possible the loop isn't common to the
// Src and Dst loops. If it isn't, then there's no need to record a direction.
LLVM_DEBUG(dbgs() << "\tWeak-Zero (dst) SIV test\n");
@@ -1879,7 +1998,7 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
Result.Consistent = false;
const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
NewConstraint.setLine(SrcCoeff, SE->getZero(Delta->getType()), Delta,
- CurLoop);
+ CurSrcLoop, CurDstLoop);
LLVM_DEBUG(dbgs() << "\t Delta = " << *Delta << "\n");
if (isKnownPredicate(CmpInst::ICMP_EQ, DstConst, SrcConst)) {
if (Level < CommonLevels) {
@@ -1900,7 +2019,8 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
// check that Delta/SrcCoeff < iteration count
// really check NewDelta < count*AbsCoeff
- if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+ if (const SCEV *UpperBound =
+ collectUpperBound(CurSrcLoop, Delta->getType())) {
LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound << "\n");
const SCEV *Product = SE->getMulExpr(AbsCoeff, UpperBound);
if (isKnownPredicate(CmpInst::ICMP_SGT, NewDelta, Product)) {
@@ -1938,7 +2058,6 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
return false;
}
-
// exactRDIVtest - Tests the RDIV subscript pair for dependence.
// Things of the form [c1 + a*i] and [c2 + b*j],
// where i and j are induction variable, c1 and c2 are loop invariant,
@@ -2230,43 +2349,47 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level,
const SCEV *DstConst = DstAddRec->getStart();
const SCEV *SrcCoeff = SrcAddRec->getStepRecurrence(*SE);
const SCEV *DstCoeff = DstAddRec->getStepRecurrence(*SE);
- const Loop *CurLoop = SrcAddRec->getLoop();
- assert(CurLoop == DstAddRec->getLoop() &&
- "both loops in SIV should be same");
- Level = mapSrcLoop(CurLoop);
+ const Loop *CurSrcLoop = SrcAddRec->getLoop();
+ const Loop *CurDstLoop = DstAddRec->getLoop();
+ assert(areLoopsSimilar(CurSrcLoop, CurDstLoop) &&
+ "both loops in SIV should be the same or have the same tripcount "
+ "and depth");
+ Level = mapSrcLoop(CurSrcLoop);
bool disproven;
if (SrcCoeff == DstCoeff)
- disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
- Level, Result, NewConstraint);
+ disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurSrcLoop,
+ CurDstLoop, Level, Result, NewConstraint);
else if (SrcCoeff == SE->getNegativeSCEV(DstCoeff))
- disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
- Level, Result, NewConstraint, SplitIter);
+ disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurSrcLoop,
+ CurDstLoop, Level, Result, NewConstraint,
+ SplitIter);
else
- disproven = exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop,
- Level, Result, NewConstraint);
- return disproven ||
- gcdMIVtest(Src, Dst, Result) ||
- symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, CurLoop);
+ disproven =
+ exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurSrcLoop,
+ CurDstLoop, Level, Result, NewConstraint);
+ return disproven || gcdMIVtest(Src, Dst, Result) ||
+ symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurSrcLoop,
+ CurDstLoop);
}
if (SrcAddRec) {
const SCEV *SrcConst = SrcAddRec->getStart();
const SCEV *SrcCoeff = SrcAddRec->getStepRecurrence(*SE);
const SCEV *DstConst = Dst;
- const Loop *CurLoop = SrcAddRec->getLoop();
- Level = mapSrcLoop(CurLoop);
- return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
- Level, Result, NewConstraint) ||
- gcdMIVtest(Src, Dst, Result);
+ const Loop *CurSrcLoop = SrcAddRec->getLoop();
+ Level = mapSrcLoop(CurSrcLoop);
+ return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurSrcLoop,
+ CurSrcLoop, Level, Result, NewConstraint) ||
+ gcdMIVtest(Src, Dst, Result);
}
if (DstAddRec) {
const SCEV *DstConst = DstAddRec->getStart();
const SCEV *DstCoeff = DstAddRec->getStepRecurrence(*SE);
const SCEV *SrcConst = Src;
- const Loop *CurLoop = DstAddRec->getLoop();
- Level = mapDstLoop(CurLoop);
- return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst,
- CurLoop, Level, Result, NewConstraint) ||
- gcdMIVtest(Src, Dst, Result);
+ const Loop *CurDstLoop = DstAddRec->getLoop();
+ Level = mapDstLoop(CurDstLoop);
+ return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, CurDstLoop,
+ CurDstLoop, Level, Result, NewConstraint) ||
+ gcdMIVtest(Src, Dst, Result);
}
llvm_unreachable("SIV test expected at least one AddRec");
return false;
@@ -3171,19 +3294,20 @@ bool DependenceInfo::propagate(const SCEV *&Src, const SCEV *&Dst,
bool DependenceInfo::propagateDistance(const SCEV *&Src, const SCEV *&Dst,
Constraint &CurConstraint,
bool &Consistent) {
- const Loop *CurLoop = CurConstraint.getAssociatedLoop();
+ const Loop *CurSrcLoop = CurConstraint.getAssociatedSrcLoop();
+ const Loop *CurDstLoop = CurConstraint.getAssociatedDstLoop();
LLVM_DEBUG(dbgs() << "\t\tSrc is " << *Src << "\n");
- const SCEV *A_K = findCoefficient(Src, CurLoop);
+ const SCEV *A_K = findCoefficient(Src, CurSrcLoop);
if (A_K->isZero())
return false;
const SCEV *DA_K = SE->getMulExpr(A_K, CurConstraint.getD());
Src = SE->getMinusSCEV(Src, DA_K);
- Src = zeroCoefficient(Src, CurLoop);
+ Src = zeroCoefficient(Src, CurSrcLoop);
LLVM_DEBUG(dbgs() << "\t\tnew Src is " << *Src << "\n");
LLVM_DEBUG(dbgs() << "\t\tDst is " << *Dst << "\n");
- Dst = addToCoefficient(Dst, CurLoop, SE->getNegativeSCEV(A_K));
+ Dst = addToCoefficient(Dst, CurDstLoop, SE->getNegativeSCEV(A_K));
LLVM_DEBUG(dbgs() << "\t\tnew Dst is " << *Dst << "\n");
- if (!findCoefficient(Dst, CurLoop)->isZero())
+ if (!findCoefficient(Dst, CurDstLoop)->isZero())
Consistent = false;
return true;
}
@@ -3197,7 +3321,8 @@ bool DependenceInfo::propagateDistance(const SCEV *&Src, const SCEV *&Dst,
bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
Constraint &CurConstraint,
bool &Consistent) {
- const Loop *CurLoop = CurConstraint.getAssociatedLoop();
+ const Loop *CurSrcLoop = CurConstraint.getAssociatedSrcLoop();
+ const Loop *CurDstLoop = CurConstraint.getAssociatedDstLoop();
const SCEV *A = CurConstraint.getA();
const SCEV *B = CurConstraint.getB();
const SCEV *C = CurConstraint.getC();
@@ -3213,11 +3338,11 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
APInt Charlie = Cconst->getAPInt();
APInt CdivB = Charlie.sdiv(Beta);
assert(Charlie.srem(Beta) == 0 && "C should be evenly divisible by B");
- const SCEV *AP_K = findCoefficient(Dst, CurLoop);
+ const SCEV *AP_K = findCoefficient(Dst, CurDstLoop);
// Src = SE->getAddExpr(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
Src = SE->getMinusSCEV(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
- Dst = zeroCoefficient(Dst, CurLoop);
- if (!findCoefficient(Src, CurLoop)->isZero())
+ Dst = zeroCoefficient(Dst, CurDstLoop);
+ if (!findCoefficient(Src, CurSrcLoop)->isZero())
Consistent = false;
}
else if (B->isZero()) {
@@ -3228,10 +3353,10 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
APInt Charlie = Cconst->getAPInt();
APInt CdivA = Charlie.sdiv(Alpha);
assert(Charlie.srem(Alpha) == 0 && "C should be evenly divisible by A");
- const SCEV *A_K = findCoefficient(Src, CurLoop);
+ const SCEV *A_K = findCoefficient(Src, CurSrcLoop);
Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, SE->getConstant(CdivA)));
- Src = zeroCoefficient(Src, CurLoop);
- if (!findCoefficient(Dst, CurLoop)->isZero())
+ Src = zeroCoefficient(Src, CurSrcLoop);
+ if (!findCoefficient(Dst, CurDstLoop)->isZero())
Consistent = false;
}
else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) {
@@ -3242,22 +3367,22 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
APInt Charlie = Cconst->getAPInt();
APInt CdivA = Charlie.sdiv(Alpha);
assert(Charlie.srem(Alpha) == 0 && "C should be evenly divisible by A");
- const SCEV *A_K = findCoefficient(Src, CurLoop);
+ const SCEV *A_K = findCoefficient(Src, CurSrcLoop);
Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, SE->getConstant(CdivA)));
- Src = zeroCoefficient(Src, CurLoop);
- Dst = addToCoefficient(Dst, CurLoop, A_K);
- if (!findCoefficient(Dst, CurLoop)->isZero())
+ Src = zeroCoefficient(Src, CurSrcLoop);
+ Dst = addToCoefficient(Dst, CurDstLoop, A_K);
+ if (!findCoefficient(Dst, CurDstLoop)->isZero())
Consistent = false;
}
else {
// paper is incorrect here, or perhaps just misleading
- const SCEV *A_K = findCoefficient(Src, CurLoop);
+ const SCEV *A_K = findCoefficient(Src, CurSrcLoop);
Src = SE->getMulExpr(Src, A);
Dst = SE->getMulExpr(Dst, A);
Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, C));
- Src = zeroCoefficient(Src, CurLoop);
- Dst = addToCoefficient(Dst, CurLoop, SE->getMulExpr(A_K, B));
- if (!findCoefficient(Dst, CurLoop)->isZero())
+ Src = zeroCoefficient(Src, CurSrcLoop);
+ Dst = addToCoefficient(Dst, CurDstLoop, SE->getMulExpr(A_K, B));
+ if (!findCoefficient(Dst, CurDstLoop)->isZero())
Consistent = false;
}
LLVM_DEBUG(dbgs() << "\t\tnew Src = " << *Src << "\n");
@@ -3271,17 +3396,18 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
// Return true if some simplification occurs.
bool DependenceInfo::propagatePoint(const SCEV *&Src, const SCEV *&Dst,
Constraint &CurConstraint) {
- const Loop *CurLoop = CurConstraint.getAssociatedLoop();
- const SCEV *A_K = findCoefficient(Src, CurLoop);
- const SCEV *AP_K = findCoefficient(Dst, CurLoop);
+ const Loop *CurSrcLoop = CurConstraint.getAssociatedSrcLoop();
+ const Loop *CurDstLoop = CurConstraint.getAssociatedDstLoop();
+ const SCEV *A_K = findCoefficient(Src, CurSrcLoop);
+ const SCEV *AP_K = findCoefficient(Dst, CurDstLoop);
const SCEV *XA_K = SE->getMulExpr(A_K, CurConstraint.getX());
const SCEV *YAP_K = SE->getMulExpr(AP_K, CurConstraint.getY());
LLVM_DEBUG(dbgs() << "\t\tSrc is " << *Src << "\n");
Src = SE->getAddExpr(Src, SE->getMinusSCEV(XA_K, YAP_K));
- Src = zeroCoefficient(Src, CurLoop);
+ Src = zeroCoefficient(Src, CurSrcLoop);
LLVM_DEBUG(dbgs() << "\t\tnew Src is " << *Src << "\n");
LLVM_DEBUG(dbgs() << "\t\tDst is " << *Dst << "\n");
- Dst = zeroCoefficient(Dst, CurLoop);
+ Dst = zeroCoefficient(Dst, CurDstLoop);
LLVM_DEBUG(dbgs() << "\t\tnew Dst is " << *Dst << "\n");
return true;
}
@@ -3617,14 +3743,6 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
break; // The underlying objects alias; test accesses for dependence.
}
- // establish loop nesting levels
- establishNestingLevels(Src, Dst);
- LLVM_DEBUG(dbgs() << " common nesting levels = " << CommonLevels << "\n");
- LLVM_DEBUG(dbgs() << " maximum nesting levels = " << MaxLevels << "\n");
-
- FullDependence Result(Src, Dst, PossiblyLoopIndependent, CommonLevels);
- ++TotalArrayPairs;
-
unsigned Pairs = 1;
SmallVector<Subscript, 2> Pair(Pairs);
const SCEV *SrcSCEV = SE->getSCEV(SrcPtr);
@@ -3651,6 +3769,38 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
}
}
+ // establish loop nesting levels considering separate but similar loops as
+ // common
+ establishNestingLevels(Src, Dst, true);
+ if (SeparateLevels > 0) {
+ // MIV is not handled yet on separate loops; check if there is any MIV test
+ for (unsigned P = 0; P < Pairs; ++P) {
+ Pair[P].Loops.resize(MaxLevels + 1);
+ auto classification = classifyPair(
+ Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst,
+ LI->getLoopFor(Dst->getParent()), Pair[P].Loops);
+ if (classification == Subscript::MIV) {
+ // reestablish loop nesting levels without considering separate loops as
+ // common
+ establishNestingLevels(Src, Dst, false);
+ break;
+ }
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << " common nesting levels = " << CommonLevels << "\n");
+ LLVM_DEBUG(dbgs() << " maximum nesting levels = " << MaxLevels << "\n");
+ LLVM_DEBUG(dbgs() << " separate nesting levels = " << SeparateLevels
+ << "\n");
+
+ FullDependence Result(Src, Dst, PossiblyLoopIndependent, CommonLevels);
+ ++TotalArrayPairs;
+
+ if (SeparateLevels > 0) {
+ Result.Consistent = false;
+ SeparateLoopsConsidered++;
+ }
+
for (unsigned P = 0; P < Pairs; ++P) {
Pair[P].Loops.resize(MaxLevels + 1);
Pair[P].GroupLoops.resize(MaxLevels + 1);
@@ -3940,6 +4090,31 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
if (CompleteLoops[II])
Result.DV[II - 1].Scalar = false;
+ if (SeparateLevels > 0) {
+ // Extracting separate levels from the common levels
+ assert(CommonLevels >= SeparateLevels);
+ CommonLevels -= SeparateLevels;
+ MaxLevels += SeparateLevels;
+ LLVM_DEBUG(dbgs() << "Updated levels after extracting separate levels from "
+ "common levels\n");
+ LLVM_DEBUG(dbgs() << " common nesting levels = " << CommonLevels
+ << "\n");
+ LLVM_DEBUG(dbgs() << " maximum nesting levels = " << MaxLevels << "\n");
+ LLVM_DEBUG(dbgs() << " separate nesting levels = " << SeparateLevels
+ << "\n");
+ auto DV = std::make_unique<FullDependence::DVEntry[]>(CommonLevels);
+ auto DVSeparate =
+ std::make_unique<FullDependence::DVEntry[]>(SeparateLevels);
+ for (unsigned level = 0; level < CommonLevels; ++level)
+ DV[level] = Result.DV[level];
+ for (unsigned level = 0; level < SeparateLevels; ++level)
+ DVSeparate[level] = Result.DV[CommonLevels + level];
+ Result.DV = std::move(DV);
+ Result.DVSeparate = std::move(DVSeparate);
+ Result.Levels = CommonLevels;
+ Result.SeparateLevels = SeparateLevels;
+ }
+
if (PossiblyLoopIndependent) {
// Make sure the LoopIndependent flag is set correctly.
// All directions must include equal, otherwise no
diff --git a/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll b/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
index 404018707c0a5..0811a56b495c4 100644
--- a/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/PreliminaryNoValidityCheckFixedSize.ll
@@ -20,7 +20,7 @@ define void @p2(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
; CHECK-NEXT: Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %i.011, ptr %arrayidx8, align 8
; CHECK-NEXT: da analyze - none!
; CHECK-NEXT: Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
-; CHECK-NEXT: da analyze - flow [-3 -2]!
+; CHECK-NEXT: da analyze - flow [-3 -2 / -1]!
; CHECK-NEXT: Src: store i64 %i.011, ptr %arrayidx8, align 8 --> Dst: store i64 %0, ptr %B.addr.24, align 8
; CHECK-NEXT: da analyze - confused!
; CHECK-NEXT: Src: %0 = load i64, ptr %arrayidx17, align 8 --> Dst: %0 = load i64, ptr %arrayidx17, align 8
diff --git a/llvm/test/Analysis/DependenceAnalysis/SIVSeparateLoops.ll b/llvm/test/Analysis/DependenceAnalysis/SIVSeparateLoops.ll
new file mode 100644
index 0000000000000..febc2ed4353ac
--- /dev/null
+++ b/llvm/test/Analysis/DependenceAnalysis/SIVSeparateLoops.ll
@@ -0,0 +1,209 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: -da-disable-delinearization-checks | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.6.0"
+
+;; for (long int i = 0; i < n; i++) {
+;; for (long int j = 0; j < n; j++) {
+;; for (long int k = 0; k < n; k++) {
+;; A[i][j][k] = i;
+;; }
+;; for (long int k = 0; k < n; k++) {
+;; *B++ = A[i + 3][j + 2][k + 1];
+
+define void @SIVSeparate0(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+entry:
+ %cmp10 = icmp sgt i64 %n, 0
+ br i1 %cmp10, label %for.cond1.preheader.preheader, label %for.end26
+
+; CHECK-LABEL: SIVSeparate0
+; CHECK: da analyze - none!
+; CHECK: da analyze - flow [-3 -2 / -1]!
+; CHECK: da analyze - confused!
+; CHECK: da analyze - none!
+; CHECK: da analyze - confused!
+; CHECK: da analyze - output [* * *]!
+
+for.cond1.preheader.preheader: ; preds = %entry
+ br label %for.cond1.preheader
+
+for.cond1.preheader: ; preds = %for.cond1.preheader.preheader, %for.inc24
+ %B.addr.012 = phi ptr [ %B.addr.1.lcssa, %for.inc24 ], [ %B, %for.cond1.preheader.preheader ]
+ %i.011 = phi i64 [ %inc25, %for.inc24 ], [ 0, %for.cond1.preheader.preheader ]
+ %cmp26 = icmp sgt i64 %n, 0
+ br i1 %cmp26, label %for.cond4.preheader.preheader, label %for.inc24
+
+for.cond4.preheader.preheader: ; preds = %for.cond1.preheader
+ br label %for.cond4.preheader
+
+for.cond4.preheader: ; preds = %for.cond4.preheader.preheader, %for.inc21
+ %B.addr.18 = phi ptr [ %B.addr.2.lcssa, %for.inc21 ], [ %B.addr.012, %for.cond4.preheader.preheader ]
+ %j.07 = phi i64 [ %inc22, %for.inc21 ], [ 0, %for.cond4.preheader.preheader ]
+ %cmp51 = icmp sgt i64 %n, 0
+ br i1 %cmp51, label %for.body6.preheader, label %for.cond10.loopexit
+
+for.body6.preheader: ; preds = %for.cond4.preheader
+ br label %for.body6
+
+for.body6: ; preds = %for.body6.preheader, %for.body6
+ %k.02 = phi i64 [ %inc, %for.body6 ], [ 0, %for.body6.preheader ]
+ %arrayidx8 = getelementptr inbounds [100 x [100 x i64]], ptr %A, i64 %i.011, i64 %j.07, i64 %k.02
+ store i64 %i.011, ptr %arrayidx8, align 8
+ %inc = add nsw i64 %k.02, 1
+ %exitcond13 = icmp ne i64 %inc, %n
+ br i1 %exitcond13, label %for.body6, label %for.cond10.loopexit.loopexit
+
+for.cond10.loopexit.loopexit: ; preds = %for.body6
+ br label %for.cond10.loopexit
+
+for.cond10.loopexit: ; preds = %for.cond10.loopexit.loopexit, %for.cond4.preheader
+ %cmp113 = icmp sgt i64 %n, 0
+ br i1 %cmp113, label %for.body12.preheader, label %for.inc21
+
+for.body12.preheader: ; preds = %for.cond10.loopexit
+ br label %for.body12
+
+for.body12: ; preds = %for.body12.preheader, %for.body12
+ %k9.05 = phi i64 [ %inc19, %for.body12 ], [ 0, %for.body12.preheader ]
+ %B.addr.24 = phi ptr [ %incdec.ptr, %for.body12 ], [ %B.addr.18, %for.body12.preheader ]
+ %add = add nsw i64 %k9.05, 1
+ %add13 = add nsw i64 %j.07, 2
+ %add14 = add nsw i64 %i.011, 3
+ %arrayidx17 = getelementptr inbounds [100 x [100 x i64]], ptr %A, i64 %add14, i64 %add13, i64 %add
+ %0 = load i64, ptr %arrayidx17, align 8
+ %incdec.ptr = getelementptr inbounds i64, ptr %B.addr.24, i64 1
+ store i64 %0, ptr %B.addr.24, align 8
+ %inc19 = add nsw i64 %k9.05, 1
+ %exitcond = icmp ne i64 %inc19, %n
+ br i1 %exitcond, label %for.body12, label %for.inc21.loopexit
+
+for.inc21.loopexit: ; preds = %for.body12
+ %scevgep = getelementptr i64, ptr %B.addr.18, i64 %n
+ br label %for.inc21
+
+for.inc21: ; preds = %for.inc21.loopexit, %for.cond10.loopexit
+ %B.addr.2.lcssa = phi ptr [ %B.addr.18, %for.cond10.loopexit ], [ %scevgep, %for.inc21.loopexit ]
+ %inc22 = add nsw i64 %j.07, 1
+ %exitcond14 = icmp ne i64 %inc22, %n
+ br i1 %exitcond14, label %for.cond4.preheader, label %for.inc24.loopexit
+
+for.inc24.loopexit: ; preds = %for.inc21
+ %B.addr.2.lcssa.lcssa = phi ptr [ %B.addr.2.lcssa, %for.inc21 ]
+ br label %for.inc24
+
+for.inc24: ; preds = %for.inc24.loopexit, %for.cond1.preheader
+ %B.addr.1.lcssa = phi ptr [ %B.addr.012, %for.cond1.preheader ], [ %B.addr.2.lcssa.lcssa, %for.inc24.loopexit ]
+ %inc25 = add nsw i64 %i.011, 1
+ %exitcond15 = icmp ne i64 %inc25, %n
+ br i1 %exitcond15, label %for.cond1.preheader, label %for.end26.loopexit
+
+for.end26.loopexit: ; preds = %for.inc24
+ br label %for.end26
+
+for.end26: ; preds = %for.end26.loopexit, %entry
+ ret void
+}
+
+
+;; for (long int i = 0; i < n; i++) {
+;; for (long int j = 0; j < n; j++) {
+;; for (long int k = 0; k < n; k++) {
+;; A[i][j][k] = i;
+;; }
+;; for (long int k = 1; k < n+1; k++) {
+;; *B++ = A[i + 3][j + 2][k + 1];
+
+define void @SIVSeparate1(i64 %n, ptr %A, ptr %B) nounwind uwtable ssp {
+entry:
+ %cmp10 = icmp sgt i64 %n, 0
+ br i1 %cmp10, label %for.cond1.preheader.preheader, label %for.end26
+
+; CHECK-LABEL: SIVSeparate1
+; CHECK: da analyze - none!
+; CHECK: da analyze - flow [-3 -2 / -2]!
+; CHECK: da analyze - confused!
+; CHECK: da analyze - none!
+; CHECK: da analyze - confused!
+; CHECK: da analyze - output [* * *]!
+
+for.cond1.preheader.preheader: ; preds = %entry
+ br label %for.cond1.preheader
+
+for.cond1.preheader: ; preds = %for.cond1.preheader.preheader, %for.inc24
+ %B.addr.012 = phi ptr [ %B.addr.1.lcssa, %for.inc24 ], [ %B, %for.cond1.preheader.preheader ]
+ %i.011 = phi i64 [ %inc25, %for.inc24 ], [ 0, %for.cond1.preheader.preheader ]
+ %cmp26 = icmp sgt i64 %n, 0
+ br i1 %cmp26, label %for.cond4.preheader.preheader, label %for.inc24
+
+for.cond4.preheader.preheader: ; preds = %for.cond1.preheader
+ br label %for.cond4.preheader
+
+for.cond4.preheader: ; preds = %for.cond4.preheader.preheader, %for.inc21
+ %B.addr.18 = phi ptr [ %B.addr.2.lcssa, %for.inc21 ], [ %B.addr.012, %for.cond4.preheader.preheader ]
+ %j.07 = phi i64 [ %inc22, %for.inc21 ], [ 0, %for.cond4.preheader.preheader ]
+ %cmp51 = icmp sgt i64 %n, 0
+ br i1 %cmp51, label %for.body6.preheader, label %for.cond10.loopexit
+
+for.body6.preheader: ; preds = %for.cond4.preheader
+ br label %for.body6
+
+for.body6: ; preds = %for.body6.preheader, %for.body6
+ %k.02 = phi i64 [ %inc, %for.body6 ], [ 0, %for.body6.preheader ]
+ %arrayidx8 = getelementptr inbounds [100 x [100 x i64]], ptr %A, i64 %i.011, i64 %j.07, i64 %k.02
+ store i64 %i.011, ptr %arrayidx8, align 8
+ %inc = add nsw i64 %k.02, 1
+ %exitcond13 = icmp ne i64 %inc, %n
+ br i1 %exitcond13, label %for.body6, label %for.cond10.loopexit.loopexit
+
+for.cond10.loopexit.loopexit: ; preds = %for.body6
+ br label %for.cond10.loopexit
+
+for.cond10.loopexit: ; preds = %for.cond10.loopexit.loopexit, %for.cond4.preheader
+ %cmp113 = icmp sgt i64 %n, 0
+ br i1 %cmp113, label %for.body12.preheader, label %for.inc21
+
+for.body12.preheader: ; preds = %for.cond10.loopexit
+ br label %for.body12
+
+for.body12: ; preds = %for.body12.preheader, %for.body12
+ %k9.05 = phi i64 [ %inc19, %for.body12 ], [ 1, %for.body12.preheader ]
+ %B.addr.24 = phi ptr [ %incdec.ptr, %for.body12 ], [ %B.addr.18, %for.body12.preheader ]
+ %add = add nsw i64 %k9.05, 1
+ %add13 = add nsw i64 %j.07, 2
+ %add14 = add nsw i64 %i.011, 3
+ %arrayidx17 = getelementptr inbounds [100 x [100 x i64]], ptr %A, i64 %add14, i64 %add13, i64 %add
+ %0 = load i64, ptr %arrayidx17, align 8
+ %incdec.ptr = getelementptr inbounds i64, ptr %B.addr.24, i64 1
+ store i64 %0, ptr %B.addr.24, align 8
+ %inc19 = add nsw i64 %k9.05, 1
+ %inc20 = add nsw i64 %n, 1
+ %exitcond = icmp ne i64 %inc19, %inc20
+ br i1 %exitcond, label %for.body12, label %for.inc21.loopexit
+
+for.inc21.loopexit: ; preds = %for.body12
+ %scevgep = getelementptr i64, ptr %B.addr.18, i64 %n
+ br label %for.inc21
+
+for.inc21: ; preds = %for.inc21.loopexit, %for.cond10.loopexit
+ %B.addr.2.lcssa = phi ptr [ %B.addr.18, %for.cond10.loopexit ], [ %scevgep, %for.inc21.loopexit ]
+ %inc22 = add nsw i64 %j.07, 1
+ %exitcond14 = icmp ne i64 %inc22, %n
+ br i1 %exitcond14, label %for.cond4.preheader, label %for.inc24.loopexit
+
+for.inc24.loopexit: ; preds = %for.inc21
+ %B.addr.2.lcssa.lcssa = phi ptr [ %B.addr.2.lcssa, %for.inc21 ]
+ br label %for.inc24
+
+for.inc24: ; preds = %for.inc24.loopexit, %for.cond1.preheader
+ %B.addr.1.lcssa = phi ptr [ %B.addr.012, %for.cond1.preheader ], [ %B.addr.2.lcssa.lcssa, %for.inc24.loopexit ]
+ %inc25 = add nsw i64 %i.011, 1
+ %exitcond15 = icmp ne i64 %inc25, %n
+ br i1 %exitcond15, label %for.cond1.preheader, label %for.end26.loopexit
+
+for.end26.loopexit: ; preds = %for.inc24
+ br label %for.end26
+
+for.end26: ; preds = %for.end26.loopexit, %entry
+ ret void
+}
More information about the llvm-commits
mailing list