[llvm] [LAA] Determine Dst and Src overlapping by SCEV of Src and Dist (PR #79947)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 29 21:48:47 PST 2024
https://github.com/ShivaChen created https://github.com/llvm/llvm-project/pull/79947
Consider the following case:
for (int j = 0; j < 256; j++) // Loop j
for (int i = j+1; i < 256; i++)// Loop i
a[i] -= aa[j][i] * a[j];
Given that SCEV of `&a[j]` is `{@a,+,4}<Loop j>`, a[j] will be treated as scalar when vectorizing Loop i. If the `accessing size of a[j]` <= `Dist(a[j], a[i])`, there is no overlapped and can be vectorized.
In this case, accessing size of a[j] is 4 byte(float) and Dist(a[j], a[i]) is {4,+,4} which bring the minimum distance as 4
>From b06f1a8df5c7b5a876499cff6902399474b2bc75 Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Tue, 30 Jan 2024 03:52:25 +0000
Subject: [PATCH 1/2] Add vectorize-s115.ll
---
.../LoopVectorize/vectorize-s115.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll b/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
new file mode 100644
index 0000000000000..327b8b8fd7c7f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -S | FileCheck %s
+
+ at aa = global [256 x [256 x float]] zeroinitializer, align 4
+ at a = global [32000 x float] zeroinitializer, align 4
+
+;; for (int j = 0; j < 256; j++)
+;; for (int i = j+1; i < 256; i++)
+;; a[i] -= aa[j][i] * a[j];
+
+; CHECK-NOT: vector.body:
+
+define signext i32 @s115() {
+entry:
+ br label %for.body
+
+for.cond.loopexit.loopexit: ; preds = %for.body4
+ br label %for.cond.loopexit
+
+for.cond.loopexit: ; preds = %for.cond.loopexit.loopexit, %for.body
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond28.not = icmp eq i64 %indvars.iv.next27, 256
+ br i1 %exitcond28.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.loopexit
+ ret i32 0
+
+for.body: ; preds = %entry, %for.cond.loopexit
+ %indvars.iv26 = phi i64 [ 0, %entry ], [ %indvars.iv.next27, %for.cond.loopexit ]
+ %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.cond.loopexit ]
+ %indvars.iv.next27 = add nuw nsw i64 %indvars.iv26, 1
+ %cmp221 = icmp ult i64 %indvars.iv26, 255
+ br i1 %cmp221, label %for.body4.lr.ph, label %for.cond.loopexit
+
+for.body4.lr.ph: ; preds = %for.body
+ %arrayidx8 = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 %indvars.iv26
+ br label %for.body4
+
+for.body4: ; preds = %for.body4.lr.ph, %for.body4
+ %indvars.iv24 = phi i64 [ %indvars.iv, %for.body4.lr.ph ], [ %indvars.iv.next25, %for.body4 ]
+ %arrayidx6 = getelementptr inbounds [256 x [256 x float]], ptr @aa, i64 0, i64 %indvars.iv26, i64 %indvars.iv24
+ %0 = load float, ptr %arrayidx6, align 4
+ %1 = load float, ptr %arrayidx8, align 4
+ %arrayidx10 = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 %indvars.iv24
+ %2 = load float, ptr %arrayidx10, align 4
+ %neg = fneg float %0
+ %3 = tail call float @llvm.fmuladd.f32(float %neg, float %1, float %2)
+ store float %3, ptr %arrayidx10, align 4
+ %indvars.iv.next25 = add nuw nsw i64 %indvars.iv24, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next25, 256
+ br i1 %exitcond.not, label %for.cond.loopexit.loopexit, label %for.body4
+}
>From 34ca142ef264ead710799e28ff64bdc182f7af09 Mon Sep 17 00:00:00 2001
From: Shiva Chen <shiva.chen at imgtec.com>
Date: Mon, 29 Jan 2024 08:12:04 +0000
Subject: [PATCH 2/2] [LAA] Determine Dst and Src overlapping by SCEV of Src
and Dist
Consider the following case:
for (int j = 0; j < 256; j++) // Loop j
for (int i = j+1; i < 256; i++)// Loop i
a[i] -= aa[j][i] * a[j];
Given that SCEV of &a[j] is {@a,+,4}<Loop j>, a[j] will be treated as scalar
when vectorizing Loop i. If the accessing size of a[j] <= Dist(a[j], a[i]),
there is no overlapped and can be vectorized.
In this case, accessing size of a[j] is 4 byte(float) and Dist(a[j], a[i])
is {4,+,4} which bring the minimum distance as 4
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 71 +++++++++++++++++++
.../LoopVectorize/vectorize-s115.ll | 13 +++-
2 files changed, 81 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index dd6b88fee415a..67beec09949f4 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1917,6 +1917,74 @@ isLoopVariantIndirectAddress(ArrayRef<const Value *> UnderlyingObjects,
});
}
+static bool isAffectedByLoop(const SCEV *Expr, const Loop *L,
+ ScalarEvolution &SE) {
+ const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
+ if (!AddRec)
+ return false;
+
+ if (AddRec->getLoop() == L)
+ return true;
+
+ const SCEV *Start = AddRec->getStart();
+ const SCEV *Step = AddRec->getStepRecurrence(SE);
+ return isAffectedByLoop(Start, L, SE) || isAffectedByLoop(Step, L, SE);
+}
+
+// Consider the following case:
+//
+// for (int j = 0; j < 256; j++) // Loop j
+// for (int i = j+1; i < 256; i++)// Loop i
+// a[i] -= aa[j][i] * a[j];
+//
+// Given that SCEV of &a[j] is {@a,+,4}<Loop j>, a[j] will be treated as scalar
+// when vectorizing Loop i. If the accessing size of a[j] <= Dist(a[j], a[i]),
+// there is no overlapped and can be vectorized.
+//
+// In this case, accessing size of a[j] is 4 byte(float) and Dist(a[j], a[i])
+// is {4,+,4} which bring the minimum distance as 4.
+//
+// Return true if Dist is equal or greater than the accessing size of Src.
+static bool isSrcNoOverlap(const SCEV *Src, Instruction *AInst,
+ const SCEV *Dist, const Loop *InnermostLoop,
+ ScalarEvolution &SE) {
+ // If the Src is not affected by InnermostLoop, when vectorizing
+ // InnermostLoop, Src will be treated as scalar instead of widening to vector.
+ if (isAffectedByLoop(Src, InnermostLoop, SE))
+ return false;
+
+ if (!isa<SCEVAddRecExpr>(Dist))
+ return false;
+
+ auto *Diff = cast<SCEVAddRecExpr>(Dist);
+
+ if (Diff->getLoop() != InnermostLoop)
+ return false;
+
+ if (!isa<SCEVConstant>(Diff->getStart()))
+ return false;
+
+ if (!isa<SCEVConstant>(Diff->getStepRecurrence(SE)))
+ return false;
+
+ const SCEVConstant *DiffInc = cast<SCEVConstant>(Diff->getStepRecurrence(SE));
+ if (DiffInc->getAPInt().isNegative())
+ return false;
+
+ // If the step of Diff is positve and the Start of diff is constant,
+ // we can get the minimum diff between Src and Dst.
+ const SCEVConstant *MinDiff = cast<SCEVConstant>(Diff->getStart());
+
+ // If we get here, Src won't be vectorized, so we only need to consider the
+ // scalar load/store size. If the minimum diff between Src and Dst is equal
+ // or greater than the load/store size, there is no overlapped.
+ if (MinDiff->getAPInt().getSExtValue() >=
+ getLoadStoreType(AInst)->getScalarSizeInBits() / 8)
+ return true;
+
+ return false;
+}
+
// Get the dependence distance, stride, type size in whether i is a write for
// the dependence between A and B. Returns a DepType, if we can prove there's
// no dependence or the analysis fails. Outlined to lambda to limit he scope
@@ -1979,6 +2047,9 @@ getDependenceDistanceStrideAndSize(
InnermostLoop))
return MemoryDepChecker::Dependence::IndirectUnsafe;
+ if (isSrcNoOverlap(Src, AInst, Dist, InnermostLoop, SE))
+ return MemoryDepChecker::Dependence::NoDep;
+
// Need accesses with constant stride. We don't want to vectorize
// "A[B[i]] += ..." and similar code or pointer arithmetic that could wrap
// in the address space.
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll b/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
index 327b8b8fd7c7f..e17236a8f23a1 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-s115.ll
@@ -3,11 +3,18 @@
@aa = global [256 x [256 x float]] zeroinitializer, align 4
@a = global [32000 x float] zeroinitializer, align 4
-;; for (int j = 0; j < 256; j++)
-;; for (int i = j+1; i < 256; i++)
+;; Given that SCEV of &a[j] is {@a,+,4}<Loop j>, a[j] will be treated as scalar
+;; when vectorizing Loop i. If the accessing size of a[j] <= Dist(a[j], a[i]),
+;; there is no overlapped and can be vectorized.
+;;
+;; In this case, accessing size of a[j] is 4 byte(float) and Dist(a[j], a[i])
+;; is {4,+,4} which bring the minimum distance as 4.
+;;
+;; for (int j = 0; j < 256; j++) // Loop j
+;; for (int i = j+1; i < 256; i++)// Loop i
;; a[i] -= aa[j][i] * a[j];
-; CHECK-NOT: vector.body:
+; CHECK: vector.body:
define signext i32 @s115() {
entry:
More information about the llvm-commits
mailing list