[llvm] [LoopInterchange] Prevent interchange if one index is constant and the other has loop carried dependence (PR #79123)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 24 00:09:47 PST 2024
https://github.com/ShivaChen updated https://github.com/llvm/llvm-project/pull/79123
>From 835047124f99fbcfa4243be2b86f4e2da39686ea Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Tue, 23 Jan 2024 07:44:54 +0000
Subject: [PATCH 1/4] Add pr54176.ll
---
.../Transforms/LoopInterchange/pr54176.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/Transforms/LoopInterchange/pr54176.ll
diff --git a/llvm/test/Transforms/LoopInterchange/pr54176.ll b/llvm/test/Transforms/LoopInterchange/pr54176.ll
new file mode 100644
index 00000000000000..6047ece119f1b0
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/pr54176.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -passes=loop-interchange -cache-line-size=64 -verify-dom-info -verify-loop-info \
+; RUN: -S -debug 2>&1 | FileCheck %s
+
+ at bb = global [1024 x [128 x float]] zeroinitializer, align 4
+ at aa = global [1024 x [128 x float]] zeroinitializer, align 4
+ at cc = global [1024 x [128 x float]] zeroinitializer, align 4
+
+
+;; for (int j = 1; j < M; j++)
+;; for (int i = 1; i < N; i++) {
+;; aa[1][j-1] += bb[i][j];
+;; cc[i][j] = aa[1][j];
+;; }
+
+; CHECK: Loops interchanged.
+
+define void @pr54176() {
+entry:
+ br label %for.cond1.preheader
+
+; Loop:
+for.cond1.preheader: ; preds = %entry, %for.cond.cleanup3
+ %indvars.iv28 = phi i64 [ 1, %entry ], [ %indvars.iv.next29, %for.cond.cleanup3 ]
+ %0 = add nsw i64 %indvars.iv28, -1
+ %arrayidx8 = getelementptr inbounds [1024 x [128 x float]], ptr @aa, i64 0, i64 1, i64 %0
+ %arrayidx10 = getelementptr inbounds [1024 x [128 x float]], ptr @aa, i64 0, i64 1, i64 %indvars.iv28
+ br label %for.body4
+
+for.cond.cleanup3: ; preds = %for.body4
+ %indvars.iv.next29 = add nuw nsw i64 %indvars.iv28, 1
+ %exitcond31 = icmp ne i64 %indvars.iv.next29, 128
+ br i1 %exitcond31, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.body4: ; preds = %for.cond1.preheader, %for.body4
+ %indvars.iv = phi i64 [ 1, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+ %arrayidx6 = getelementptr inbounds [1024 x [128 x float]], ptr @bb, i64 0, i64 %indvars.iv, i64 %indvars.iv28
+ %1 = load float, ptr %arrayidx6, align 4
+ %2 = load float, ptr %arrayidx8, align 4
+ %add = fadd float %1, %2
+ store float %add, ptr %arrayidx8, align 4
+ %3 = load float, ptr %arrayidx10, align 4
+ %arrayidx14 = getelementptr inbounds [1024 x [128 x float]], ptr @cc, i64 0, i64 %indvars.iv, i64 %indvars.iv28
+ store float %3, ptr %arrayidx14, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, 1024
+ br i1 %exitcond, label %for.body4, label %for.cond.cleanup3
+
+; Exit blocks
+for.cond.cleanup: ; preds = %for.cond.cleanup3
+ ret void
+}
>From 6113bc94c65d367564a1e944ad49a791e27efe6f Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Tue, 23 Jan 2024 07:34:17 +0000
Subject: [PATCH 2/4] [LoopInterchange] Prevent interchange if one index is
constant and the other has loop carried dependence
Consider the following loop from
https://github.com/llvm/llvm-project/issues/54176
for (int j = 1; j < M; j++)
for (int i = 1; i < N; i++) {
aa[1][j-1] += bb[i][j];
cc[i][j] = aa[1][j];
}
Loops should not be interchanged in this case as it will change the
cc array results.
---
llvm/include/llvm/Analysis/DependenceAnalysis.h | 6 +++---
llvm/lib/Analysis/DependenceAnalysis.cpp | 5 ++++-
llvm/lib/Transforms/Scalar/LoopInterchange.cpp | 10 +++++++++-
llvm/test/Transforms/LoopInterchange/pr54176.ll | 5 ++++-
4 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index f0a09644e0f4b6..63aa04abc6df79 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -306,9 +306,9 @@ namespace llvm {
/// The flag PossiblyLoopIndependent should be set by the caller
/// if it appears that control flow can reach from Src to Dst
/// without traversing a loop back edge.
- std::unique_ptr<Dependence> depends(Instruction *Src,
- Instruction *Dst,
- bool PossiblyLoopIndependent);
+ std::unique_ptr<Dependence> depends(Instruction *Src, Instruction *Dst,
+ bool PossiblyLoopIndependent,
+ bool *HasConstantIndex = nullptr);
/// getSplitIteration - Give a dependence that's splittable at some
/// particular level, return the iteration that should be used to split
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 1bce9aae09bb26..392d5329a3a356 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -3588,7 +3588,7 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
// up to date with respect to this routine.
std::unique_ptr<Dependence>
DependenceInfo::depends(Instruction *Src, Instruction *Dst,
- bool PossiblyLoopIndependent) {
+ bool PossiblyLoopIndependent, bool *HasConstantIndex) {
if (Src == Dst)
PossiblyLoopIndependent = false;
@@ -3674,6 +3674,9 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
LLVM_DEBUG(dbgs() << "\tclass = " << Pair[P].Classification << "\n");
LLVM_DEBUG(dbgs() << "\tloops = ");
LLVM_DEBUG(dumpSmallBitVector(Pair[P].Loops));
+ if (HasConstantIndex)
+ if (isa<SCEVConstant>(Pair[P].Src) || isa<SCEVConstant>(Pair[P].Dst))
+ *HasConstantIndex = true;
}
SmallBitVector Separable(Pairs);
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 277f530ee25fc1..d5825052bcdf2f 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -117,11 +117,12 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
std::vector<char> Dep;
Instruction *Src = cast<Instruction>(*I);
Instruction *Dst = cast<Instruction>(*J);
+ bool HasConstantIndex = false;
// Ignore Input dependencies.
if (isa<LoadInst>(Src) && isa<LoadInst>(Dst))
continue;
// Track Output, Flow, and Anti dependencies.
- if (auto D = DI->depends(Src, Dst, true)) {
+ if (auto D = DI->depends(Src, Dst, true, &HasConstantIndex)) {
assert(D->isOrdered() && "Expected an output, flow or anti dep.");
// If the direction vector is negative, normalize it to
// make it non-negative.
@@ -150,6 +151,13 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
Direction = '=';
else
Direction = '*';
+
+ // Bail out if there is constant index and the other has loop
+ // carried dependence.
+ if (HasConstantIndex && (Direction == '>' || Direction == '<')) {
+ dbgs() << "Has Constant Index and loop carried dependence\n";
+ return false;
+ }
Dep.push_back(Direction);
}
}
diff --git a/llvm/test/Transforms/LoopInterchange/pr54176.ll b/llvm/test/Transforms/LoopInterchange/pr54176.ll
index 6047ece119f1b0..79bf712b2b6968 100644
--- a/llvm/test/Transforms/LoopInterchange/pr54176.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr54176.ll
@@ -6,13 +6,16 @@
@cc = global [1024 x [128 x float]] zeroinitializer, align 4
+;; Loops should not be interchanged in this case as it will change the
+;; cc array results.
+;;
;; for (int j = 1; j < M; j++)
;; for (int i = 1; i < N; i++) {
;; aa[1][j-1] += bb[i][j];
;; cc[i][j] = aa[1][j];
;; }
-; CHECK: Loops interchanged.
+; CHECK-NOT: Loops interchanged.
define void @pr54176() {
entry:
>From b2ff0950c0a2b99c5f33127b2484f4e428701aca Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Wed, 24 Jan 2024 06:47:44 +0000
Subject: [PATCH 3/4] Revert "[LoopInterchange] Prevent interchange if one
index is constant and the other has loop carried dependence"
This reverts commit 6113bc94c65d367564a1e944ad49a791e27efe6f.
---
llvm/include/llvm/Analysis/DependenceAnalysis.h | 6 +++---
llvm/lib/Analysis/DependenceAnalysis.cpp | 5 +----
llvm/lib/Transforms/Scalar/LoopInterchange.cpp | 10 +---------
llvm/test/Transforms/LoopInterchange/pr54176.ll | 5 +----
4 files changed, 6 insertions(+), 20 deletions(-)
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index 63aa04abc6df79..f0a09644e0f4b6 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -306,9 +306,9 @@ namespace llvm {
/// The flag PossiblyLoopIndependent should be set by the caller
/// if it appears that control flow can reach from Src to Dst
/// without traversing a loop back edge.
- std::unique_ptr<Dependence> depends(Instruction *Src, Instruction *Dst,
- bool PossiblyLoopIndependent,
- bool *HasConstantIndex = nullptr);
+ std::unique_ptr<Dependence> depends(Instruction *Src,
+ Instruction *Dst,
+ bool PossiblyLoopIndependent);
/// getSplitIteration - Give a dependence that's splittable at some
/// particular level, return the iteration that should be used to split
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 392d5329a3a356..1bce9aae09bb26 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -3588,7 +3588,7 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
// up to date with respect to this routine.
std::unique_ptr<Dependence>
DependenceInfo::depends(Instruction *Src, Instruction *Dst,
- bool PossiblyLoopIndependent, bool *HasConstantIndex) {
+ bool PossiblyLoopIndependent) {
if (Src == Dst)
PossiblyLoopIndependent = false;
@@ -3674,9 +3674,6 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
LLVM_DEBUG(dbgs() << "\tclass = " << Pair[P].Classification << "\n");
LLVM_DEBUG(dbgs() << "\tloops = ");
LLVM_DEBUG(dumpSmallBitVector(Pair[P].Loops));
- if (HasConstantIndex)
- if (isa<SCEVConstant>(Pair[P].Src) || isa<SCEVConstant>(Pair[P].Dst))
- *HasConstantIndex = true;
}
SmallBitVector Separable(Pairs);
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index d5825052bcdf2f..277f530ee25fc1 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -117,12 +117,11 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
std::vector<char> Dep;
Instruction *Src = cast<Instruction>(*I);
Instruction *Dst = cast<Instruction>(*J);
- bool HasConstantIndex = false;
// Ignore Input dependencies.
if (isa<LoadInst>(Src) && isa<LoadInst>(Dst))
continue;
// Track Output, Flow, and Anti dependencies.
- if (auto D = DI->depends(Src, Dst, true, &HasConstantIndex)) {
+ if (auto D = DI->depends(Src, Dst, true)) {
assert(D->isOrdered() && "Expected an output, flow or anti dep.");
// If the direction vector is negative, normalize it to
// make it non-negative.
@@ -151,13 +150,6 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
Direction = '=';
else
Direction = '*';
-
- // Bail out if there is constant index and the other has loop
- // carried dependence.
- if (HasConstantIndex && (Direction == '>' || Direction == '<')) {
- dbgs() << "Has Constant Index and loop carried dependence\n";
- return false;
- }
Dep.push_back(Direction);
}
}
diff --git a/llvm/test/Transforms/LoopInterchange/pr54176.ll b/llvm/test/Transforms/LoopInterchange/pr54176.ll
index 79bf712b2b6968..6047ece119f1b0 100644
--- a/llvm/test/Transforms/LoopInterchange/pr54176.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr54176.ll
@@ -6,16 +6,13 @@
@cc = global [1024 x [128 x float]] zeroinitializer, align 4
-;; Loops should not be interchanged in this case as it will change the
-;; cc array results.
-;;
;; for (int j = 1; j < M; j++)
;; for (int i = 1; i < N; i++) {
;; aa[1][j-1] += bb[i][j];
;; cc[i][j] = aa[1][j];
;; }
-; CHECK-NOT: Loops interchanged.
+; CHECK: Loops interchanged.
define void @pr54176() {
entry:
>From 8a784e7b409697eac2ec83a252ff98e74ce99b16 Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Wed, 24 Jan 2024 08:02:40 +0000
Subject: [PATCH 4/4] Use HasZIVDep to detect scalar index dependence
---
llvm/lib/Analysis/DependenceAnalysis.cpp | 13 +++++++++++++
llvm/test/Transforms/LoopInterchange/pr54176.ll | 6 ++++--
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 1bce9aae09bb26..92a62fac44c3f0 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -3786,6 +3786,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
Constraint NewConstraint;
NewConstraint.setAny(SE);
+ bool HasZIVDep = false;
// test separable subscripts
for (unsigned SI : Separable.set_bits()) {
LLVM_DEBUG(dbgs() << "testing subscript " << SI);
@@ -3794,6 +3795,8 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
LLVM_DEBUG(dbgs() << ", ZIV\n");
if (testZIV(Pair[SI].Src, Pair[SI].Dst, Result))
return nullptr;
+ else
+ HasZIVDep = true;
break;
case Subscript::SIV: {
LLVM_DEBUG(dbgs() << ", SIV\n");
@@ -3971,6 +3974,16 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
return nullptr;
}
+ if (HasZIVDep)
+ for (unsigned II = 1; II <= CommonLevels; ++II) {
+ unsigned Direction = Result.DV[II - 1].Direction;
+ if (Direction == Dependence::DVEntry::GT ||
+ Direction == Dependence::DVEntry::GE ||
+ Direction == Dependence::DVEntry::LT ||
+ Direction == Dependence::DVEntry::LE)
+ Result.DV[II - 1].Direction = Dependence::DVEntry::ALL;
+ }
+
return std::make_unique<FullDependence>(std::move(Result));
}
diff --git a/llvm/test/Transforms/LoopInterchange/pr54176.ll b/llvm/test/Transforms/LoopInterchange/pr54176.ll
index 6047ece119f1b0..48a03d0a9df220 100644
--- a/llvm/test/Transforms/LoopInterchange/pr54176.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr54176.ll
@@ -5,14 +5,16 @@
@aa = global [1024 x [128 x float]] zeroinitializer, align 4
@cc = global [1024 x [128 x float]] zeroinitializer, align 4
-
+;; Loops should not be interchanged in this case as it will change the
+;; cc array results.
+;;
;; for (int j = 1; j < M; j++)
;; for (int i = 1; i < N; i++) {
;; aa[1][j-1] += bb[i][j];
;; cc[i][j] = aa[1][j];
;; }
-; CHECK: Loops interchanged.
+; CHECK-NOT: Loops interchanged.
define void @pr54176() {
entry:
More information about the llvm-commits
mailing list