[llvm] [LAA] Be more precise on different store sizes (PR #122318)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 22 06:04:14 PDT 2025
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/122318
>From 2b0769c4d0d7cee1f087965e19483d4141c095cb Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Thu, 9 Jan 2025 16:50:04 +0000
Subject: [PATCH 1/2] [LAA] Be more precise on different store sizes
The HasSameSize checks, which are triggered on different store sizes, in
MemDepChecker::isDependent are ad-hoc and imprecise, leading to spurious
dependencies and runtime-checks. Identify that the exact scenario in
which to bail out is unequal store sizes when dependence distance is
possibly zero, and check precisely this condition in
MemDepChecker::getDependenceDistanceAndSize, eliminating all the ad-hoc
checks in isDependent and making LoopAccessAnalysis more precise.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 13 ++---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 53 ++++++++-----------
.../accesses-completely-before-or-after.ll | 17 ++----
.../forward-loop-carried.ll | 4 --
...endence-distance-different-access-sizes.ll | 17 ++----
...interleave-allocsize-not-equal-typesize.ll | 40 +++++++++++---
6 files changed, 68 insertions(+), 76 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 73bfe1aabb4e0..3c8d0cf9e8a86 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -396,8 +396,7 @@ class MemoryDepChecker {
uint64_t MaxStride;
std::optional<uint64_t> CommonStride;
- /// TypeByteSize is either the common store size of both accesses, or 0 when
- /// store sizes mismatch.
+ /// TypeByteSize is the size of the sink.
uint64_t TypeByteSize;
bool AIsWrite;
@@ -412,13 +411,9 @@ class MemoryDepChecker {
};
/// Get the dependence distance, strides, type size and whether it is a write
- /// for the dependence between A and B. Returns a DepType, if we can prove
- /// there's no dependence or the analysis fails. Outlined to lambda to limit
- /// he scope of various temporary variables, like A/BPtr, StrideA/BPtr and
- /// others. Returns either the dependence result, if it could already be
- /// determined, or a DepDistanceStrideAndSizeInfo struct, noting that
- /// TypeByteSize could be 0 when store sizes mismatch, and this should be
- /// checked in the caller.
+ /// for the dependence between A and B. Returns either a DepType, the
+ /// dependence result, if it could already be determined, or a
+ /// DepDistanceStrideAndSizeInfo struct.
std::variant<Dependence::DepType, DepDistanceStrideAndSizeInfo>
getDependenceDistanceStrideAndSize(const MemAccessInfo &A, Instruction *AInst,
const MemAccessInfo &B,
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index f3a32d3055edb..9dca197d34953 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2061,14 +2061,12 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
return MemoryDepChecker::Dependence::Unknown;
}
- TypeSize AStoreSz = DL.getTypeStoreSize(ATy);
- TypeSize BStoreSz = DL.getTypeStoreSize(BTy);
-
- // If store sizes are not the same, set TypeByteSize to zero, so we can check
- // it in the caller isDependent.
uint64_t ASz = DL.getTypeAllocSize(ATy);
uint64_t BSz = DL.getTypeAllocSize(BTy);
- uint64_t TypeByteSize = (AStoreSz == BStoreSz) ? BSz : 0;
+
+ // The TypeByteSize is used to scale Distance and VF. In these contexts, the
+ // only size that matters is the size of the Sink.
+ uint64_t TypeByteSize = BSz;
uint64_t StrideAScaled = std::abs(StrideAPtrInt) * ASz;
uint64_t StrideBScaled = std::abs(StrideBPtrInt) * BSz;
@@ -2091,6 +2089,17 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
return Dependence::Unknown;
}
+ // When the distance is possibly zero, we're reading/writing the same memory
+ // location: if the store sizes are not equal, fail with an unknown
+ // dependence.
+ TypeSize AStoreSz = DL.getTypeStoreSize(ATy);
+ TypeSize BStoreSz = DL.getTypeStoreSize(BTy);
+ if (AStoreSz != BStoreSz && !SE.isKnownNonZero(Dist)) {
+ LLVM_DEBUG(dbgs() << "LAA: possibly zero dependence distance with "
+ "different type sizes\n");
+ return Dependence::Unknown;
+ }
+
return DepDistanceStrideAndSizeInfo(Dist, MaxStride, CommonStride,
TypeByteSize, AIsWrite, BIsWrite);
}
@@ -2126,8 +2135,6 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
auto &[Dist, MaxStride, CommonStride, TypeByteSize, AIsWrite, BIsWrite] =
std::get<DepDistanceStrideAndSizeInfo>(Res);
- bool HasSameSize = TypeByteSize > 0;
-
ScalarEvolution &SE = *PSE.getSE();
auto &DL = InnermostLoop->getHeader()->getDataLayout();
@@ -2136,8 +2143,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// upper bound of the number of iterations), the accesses are independet, i.e.
// they are far enough appart that accesses won't access the same location
// across all loop ierations.
- if (HasSameSize &&
- isSafeDependenceDistance(
+ if (isSafeDependenceDistance(
DL, SE, *(PSE.getSymbolicMaxBackedgeTakenCount()), *Dist, MaxStride))
return Dependence::NoDep;
@@ -2151,7 +2157,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
if (APDist) {
// If the distance between accesses and their strides are known constants,
// check whether the accesses interlace each other.
- if (ConstDist > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
+ if (ConstDist > 0 && CommonStride && CommonStride > 1 &&
areStridedAccessesIndependent(ConstDist, *CommonStride, TypeByteSize)) {
LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
return Dependence::NoDep;
@@ -2165,15 +2171,9 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// Negative distances are not plausible dependencies.
if (SE.isKnownNonPositive(Dist)) {
- if (SE.isKnownNonNegative(Dist)) {
- if (HasSameSize) {
- // Write to the same location with the same size.
- return Dependence::Forward;
- }
- LLVM_DEBUG(dbgs() << "LAA: possibly zero dependence difference but "
- "different type sizes\n");
- return Dependence::Unknown;
- }
+ if (SE.isKnownNonNegative(Dist))
+ // Write to the same location with the same size.
+ return Dependence::Forward;
bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
// Check if the first access writes to a location that is read in a later
@@ -2185,12 +2185,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// forward dependency will allow vectorization using any width.
if (IsTrueDataDependence && EnableForwardingConflictDetection) {
- if (!ConstDist) {
+ if (!ConstDist)
return CheckCompletelyBeforeOrAfter() ? Dependence::NoDep
: Dependence::Unknown;
- }
- if (!HasSameSize ||
- couldPreventStoreLoadForward(ConstDist, TypeByteSize)) {
+ if (couldPreventStoreLoadForward(ConstDist, TypeByteSize)) {
LLVM_DEBUG(
dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
return Dependence::ForwardButPreventsForwarding;
@@ -2208,13 +2206,6 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
: Dependence::Unknown;
}
- if (!HasSameSize) {
- if (CheckCompletelyBeforeOrAfter())
- return Dependence::NoDep;
- LLVM_DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with "
- "different type sizes\n");
- return Dependence::Unknown;
- }
// Bail out early if passed-in parameters make vectorization not feasible.
unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
VectorizerParams::VectorizationFactor : 1);
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll b/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
index 9a329b70338bd..f7e00c5f2bd54 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
@@ -109,21 +109,14 @@ exit:
define void @may_overlap_stores_with_different_sizes(ptr %dst) {
; CHECK-LABEL: 'may_overlap_stores_with_different_sizes'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 256 bits
; CHECK-NEXT: Dependences:
+; CHECK-NEXT: BackwardVectorizable:
+; CHECK-NEXT: store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT: store i8 0, ptr %gep.dst.128.iv, align 1
+; CHECK-EMPTY:
; CHECK-NEXT: Run-time memory checks:
-; CHECK-NEXT: Check 0:
-; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %gep.iv = getelementptr i16, ptr %dst, i64 %iv
-; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.dst.128.iv = getelementptr i8, ptr %gep.dst.128, i64 %iv
; CHECK-NEXT: Grouped accesses:
-; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: %dst High: (130 + %dst))
-; CHECK-NEXT: Member: {%dst,+,2}<nw><%loop>
-; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: (128 + %dst)<nuw> High: (193 + %dst))
-; CHECK-NEXT: Member: {(128 + %dst)<nuw>,+,1}<nw><%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/forward-loop-carried.ll b/llvm/test/Analysis/LoopAccessAnalysis/forward-loop-carried.ll
index adfd19923e921..7837c20f003e2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/forward-loop-carried.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/forward-loop-carried.ll
@@ -70,10 +70,6 @@ define void @forward_different_access_sizes(ptr readnone %end, ptr %start) {
; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 ->
; CHECK-NEXT: %l = load i24, ptr %gep.1, align 1
; CHECK-EMPTY:
-; CHECK-NEXT: Forward:
-; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 ->
-; CHECK-NEXT: store i24 %l, ptr %ptr.iv, align 1
-; CHECK-EMPTY:
; CHECK-NEXT: Run-time memory checks:
; CHECK-NEXT: Grouped accesses:
; CHECK-EMPTY:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
index 1a6e25859f085..c966bc4e7d028 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
@@ -8,21 +8,14 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
define void @test_distance_positive_independent_via_trip_count(ptr %A) {
; CHECK-LABEL: 'test_distance_positive_independent_via_trip_count'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 3200 bits
; CHECK-NEXT: Dependences:
+; CHECK-NEXT: BackwardVectorizable:
+; CHECK-NEXT: %l = load i8, ptr %gep.A, align 1 ->
+; CHECK-NEXT: store i32 %ext, ptr %gep.A.400, align 4
+; CHECK-EMPTY:
; CHECK-NEXT: Run-time memory checks:
-; CHECK-NEXT: Check 0:
-; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: %gep.A.400 = getelementptr inbounds i32, ptr %A.400, i64 %iv
-; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
; CHECK-NEXT: Grouped accesses:
-; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: (400 + %A)<nuw> High: (804 + %A))
-; CHECK-NEXT: Member: {(400 + %A)<nuw>,+,4}<nuw><%loop>
-; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: %A High: (101 + %A))
-; CHECK-NEXT: Member: {%A,+,1}<nuw><%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
index 79d7ab84b3a0f..c48ae69093e99 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
@@ -35,10 +35,10 @@ define void @pr58722_load_interleave_group(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = load i24, ptr [[TMP9]], align 4, !alias.scope !0
-; CHECK-NEXT: [[TMP14:%.*]] = load i24, ptr [[TMP10]], align 4, !alias.scope !0
-; CHECK-NEXT: [[TMP15:%.*]] = load i24, ptr [[TMP11]], align 4, !alias.scope !0
-; CHECK-NEXT: [[TMP16:%.*]] = load i24, ptr [[TMP12]], align 4, !alias.scope !0
+; CHECK-NEXT: [[TMP13:%.*]] = load i24, ptr [[TMP9]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i24, ptr [[TMP10]], align 4, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i24, ptr [[TMP11]], align 4, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i24, ptr [[TMP12]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i24> poison, i24 [[TMP13]], i32 0
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i24> [[TMP17]], i24 [[TMP14]], i32 1
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i24> [[TMP18]], i24 [[TMP15]], i32 2
@@ -47,7 +47,7 @@ define void @pr58722_load_interleave_group(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP22:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0
-; CHECK-NEXT: store <4 x i32> [[TMP22]], ptr [[TMP24]], align 4, !alias.scope !3, !noalias !0
+; CHECK-NEXT: store <4 x i32> [[TMP22]], ptr [[TMP24]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
; CHECK-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -96,17 +96,41 @@ exit:
define void @pr58722_store_interleave_group(ptr %src, ptr %dst) {
; CHECK-LABEL: @pr58722_store_interleave_group(
; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i32 [[TMP1]]
+; CHECK-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP2]], align 4
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[OFFSET_IDX]] to i24
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP1]] to i24
+; CHECK-NEXT: store i24 [[TMP6]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store i24 [[TMP7]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 5000
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 10000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i32 [[IV]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i32 [[IV]]
; CHECK-NEXT: store i32 [[IV]], ptr [[GEP_IV]], align 4
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[GEP_IV]], i64 1
; CHECK-NEXT: [[TRUNC_IV:%.*]] = trunc i32 [[IV]] to i24
; CHECK-NEXT: store i24 [[TRUNC_IV]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[IV]], 10000
-; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT: br i1 [[CMP]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
>From 32cac7d4c39b4756c2221754881bb94f33797945 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Mon, 21 Jul 2025 18:28:16 +0100
Subject: [PATCH 2/2] [LAA] Employ more careful thinking, fixing bugs
Co-authored-by: Igor Kirillov <igor.kirillov at arm.com>
---
.../llvm/Analysis/LoopAccessAnalysis.h | 8 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 55 +++--
.../accesses-completely-before-or-after.ll | 2 +-
.../LoopAccessAnalysis/depend_diff_types.ll | 210 ++++++++++++++++++
...endence-distance-different-access-sizes.ll | 2 +-
5 files changed, 254 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 3c8d0cf9e8a86..a9b4362a1f8e8 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -396,16 +396,16 @@ class MemoryDepChecker {
uint64_t MaxStride;
std::optional<uint64_t> CommonStride;
- /// TypeByteSize is the size of the sink.
- uint64_t TypeByteSize;
+ /// TypeByteSize is a pair of SourceSize and SinkSize.
+ std::pair<uint64_t, uint64_t> TypeByteSize;
bool AIsWrite;
bool BIsWrite;
DepDistanceStrideAndSizeInfo(const SCEV *Dist, uint64_t MaxStride,
std::optional<uint64_t> CommonStride,
- uint64_t TypeByteSize, bool AIsWrite,
- bool BIsWrite)
+ std::pair<uint64_t, uint64_t> TypeByteSize,
+ bool AIsWrite, bool BIsWrite)
: Dist(Dist), MaxStride(MaxStride), CommonStride(CommonStride),
TypeByteSize(TypeByteSize), AIsWrite(AIsWrite), BIsWrite(BIsWrite) {}
};
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9dca197d34953..149245f51e9fd 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1922,17 +1922,25 @@ static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE,
/// Check the dependence for two accesses with the same stride \p Stride.
/// \p Distance is the positive distance in bytes, and \p TypeByteSize is type
-/// size in bytes.
+/// size of source and sink in bytes.
///
/// \returns true if they are independent.
-static bool areStridedAccessesIndependent(uint64_t Distance, uint64_t Stride,
- uint64_t TypeByteSize) {
+static bool
+areStridedAccessesIndependent(uint64_t Distance, uint64_t Stride,
+ std::pair<uint64_t, uint64_t> TypeByteSize) {
assert(Stride > 1 && "The stride must be greater than 1");
- assert(TypeByteSize > 0 && "The type size in byte must be non-zero");
+ assert(TypeByteSize.first > 0 && TypeByteSize.second > 0 &&
+ "The type size in byte must be non-zero");
assert(Distance > 0 && "The distance must be non-zero");
- // Skip if the distance is not multiple of type byte size.
- if (Distance % TypeByteSize)
+ // Consider two 8-byte regions accessed at x and y:
+ //
+ // [o o x x y o o o] 2 and 1 byte acccesses
+ // [o o x y y o o o] 1 and 2 byte access
+ //
+ // Skip if the distance is not multiple of type byte size of either source or
+ // sink.
+ if (Distance % TypeByteSize.first || Distance % TypeByteSize.second)
return false;
// No dependence if the distance is not multiple of the stride.
@@ -2064,9 +2072,9 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
uint64_t ASz = DL.getTypeAllocSize(ATy);
uint64_t BSz = DL.getTypeAllocSize(BTy);
- // The TypeByteSize is used to scale Distance and VF. In these contexts, the
- // only size that matters is the size of the Sink.
- uint64_t TypeByteSize = BSz;
+ // Both the source and sink sizes are neeeded in dependence checks, depending
+ // on the use.
+ std::pair<uint64_t, uint64_t> TypeByteSize(ASz, BSz);
uint64_t StrideAScaled = std::abs(StrideAPtrInt) * ASz;
uint64_t StrideBScaled = std::abs(StrideBPtrInt) * BSz;
@@ -2171,9 +2179,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// Negative distances are not plausible dependencies.
if (SE.isKnownNonPositive(Dist)) {
- if (SE.isKnownNonNegative(Dist))
+ if (SE.isKnownNonNegative(Dist)) {
// Write to the same location with the same size.
return Dependence::Forward;
+ }
bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
// Check if the first access writes to a location that is read in a later
@@ -2185,10 +2194,14 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// forward dependency will allow vectorization using any width.
if (IsTrueDataDependence && EnableForwardingConflictDetection) {
- if (!ConstDist)
+ if (!ConstDist) {
return CheckCompletelyBeforeOrAfter() ? Dependence::NoDep
: Dependence::Unknown;
- if (couldPreventStoreLoadForward(ConstDist, TypeByteSize)) {
+ }
+ // couldPreventStoreLoadForward checks that the distance between loads
+ // cannot prevent forwarding: for its purposes, the source size is
+ // sufficient.
+ if (couldPreventStoreLoadForward(ConstDist, TypeByteSize.first)) {
LLVM_DEBUG(
dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
return Dependence::ForwardButPreventsForwarding;
@@ -2216,8 +2229,9 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// It's not vectorizable if the distance is smaller than the minimum distance
// needed for a vectroized/unrolled version. Vectorizing one iteration in
- // front needs MaxStride. Vectorizing the last iteration needs TypeByteSize.
- // (No need to plus the last gap distance).
+ // front needs MaxStride. Vectorizing the last iteration needs TypeByteSize of
+ // the source, for the purposes of determining Backward dependence. (No need
+ // to plus the last gap distance).
//
// E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
// foo(int *A) {
@@ -2247,7 +2261,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// We know that Dist is positive, but it may not be constant. Use the signed
// minimum for computations below, as this ensures we compute the closest
// possible dependence distance.
- uint64_t MinDistanceNeeded = MaxStride * (MinNumIter - 1) + TypeByteSize;
+ uint64_t MinDistanceNeeded =
+ MaxStride * (MinNumIter - 1) + TypeByteSize.first;
if (MinDistanceNeeded > static_cast<uint64_t>(MinDistance)) {
if (!ConstDist) {
// For non-constant distances, we checked the lower bound of the
@@ -2275,14 +2290,20 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
if (IsTrueDataDependence && EnableForwardingConflictDetection && ConstDist &&
- couldPreventStoreLoadForward(MinDistance, TypeByteSize, *CommonStride))
+ // couldPreventStoreLoadForward checks that the distance between loads
+ // cannot prevent forwarding: for its purposes, the source size is
+ // sufficient.
+ couldPreventStoreLoadForward(MinDistance, TypeByteSize.first,
+ *CommonStride))
return Dependence::BackwardVectorizableButPreventsForwarding;
uint64_t MaxVF = MinDepDistBytes / MaxStride;
LLVM_DEBUG(dbgs() << "LAA: Positive min distance " << MinDistance
<< " with max VF = " << MaxVF << '\n');
- uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
+ // The VF should be computed for the purposes of Backward dependencies, with
+ // the TypeByteSize of the source.
+ uint64_t MaxVFInBits = MaxVF * TypeByteSize.first * 8;
if (!ConstDist && MaxVFInBits < MaxTargetVectorWidthInBits) {
// For non-constant distances, we checked the lower bound of the dependence
// distance and the distance may be larger at runtime (and safe for
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll b/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
index f7e00c5f2bd54..16a23ddd989e8 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/accesses-completely-before-or-after.ll
@@ -109,7 +109,7 @@ exit:
define void @may_overlap_stores_with_different_sizes(ptr %dst) {
; CHECK-LABEL: 'may_overlap_stores_with_different_sizes'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 256 bits
+; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 512 bits
; CHECK-NEXT: Dependences:
; CHECK-NEXT: BackwardVectorizable:
; CHECK-NEXT: store i16 0, ptr %gep.iv, align 2 ->
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
index 1bf80489b1ea0..c1ea4b9b7cbe1 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
@@ -262,3 +262,213 @@ loop:
exit:
ret void
}
+
+define void @different_type_sizes_strided_accesses_independent(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_strided_accesses_independent'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+ store i16 0, ptr %gep.iv
+ %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+ store i8 1, ptr %gep.10.iv
+ %iv.next = add i64 %iv, 8
+ %ec = icmp eq i64 %iv.next, 64
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @different_type_sizes_strided_accesses_dependent(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_strided_accesses_dependent'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT: Backward loop carried data dependence.
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Backward:
+; CHECK-NEXT: store i64 0, ptr %gep.iv, align 4 ->
+; CHECK-NEXT: store i8 1, ptr %gep.10.iv, align 1
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+ store i64 0, ptr %gep.iv
+ %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+ store i8 1, ptr %gep.10.iv
+ %iv.next = add i64 %iv, 8
+ %ec = icmp eq i64 %iv.next, 64
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @different_type_sizes_source_size_backwardvectorizible(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_source_size_backwardvectorizible'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 16 bits
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: BackwardVectorizable:
+; CHECK-NEXT: store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT: store i32 1, ptr %gep.10.iv, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+ store i16 0, ptr %gep.iv
+ %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+ store i32 1, ptr %gep.10.iv
+ %iv.next = add i64 %iv, 8
+ %ec = icmp eq i64 %iv.next, 64
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @different_type_sizes_forward(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_forward'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Forward:
+; CHECK-NEXT: store i32 0, ptr %gep.10.iv, align 4 ->
+; CHECK-NEXT: store i16 1, ptr %gep.iv, align 2
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+ store i32 0, ptr %gep.10.iv
+ %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+ store i16 1, ptr %gep.iv
+ %iv.next = add i64 %iv, 8
+ %ec = icmp eq i64 %iv.next, 64
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @different_type_sizes_store_size_cannot_prevent_forwarding(ptr %A, ptr noalias %B) {
+; CHECK-LABEL: 'different_type_sizes_store_size_cannot_prevent_forwarding'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Forward:
+; CHECK-NEXT: store i32 0, ptr %gep.A, align 4 ->
+; CHECK-NEXT: %l = load i16, ptr %gep.A.1, align 2
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %A.1 = getelementptr i32, ptr %A, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1022, %entry ], [ %iv.next, %loop ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i32 0, ptr %gep.A
+ %gep.A.1 = getelementptr i32, ptr %A.1, i64 %iv
+ %l = load i16, ptr %gep.A.1
+ store i16 %l, ptr %B
+ %iv.next = add nsw i64 %iv, -1
+ %cmp = icmp eq i64 %iv, 0
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @different_type_sizes_load_size_prevents_forwarding(ptr %A, ptr noalias %B) {
+; CHECK-LABEL: 'different_type_sizes_load_size_prevents_forwarding'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding.
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: ForwardButPreventsForwarding:
+; CHECK-NEXT: store i16 0, ptr %gep.A, align 2 ->
+; CHECK-NEXT: %l = load i32, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %A.1 = getelementptr i32, ptr %A, i64 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1022, %entry ], [ %iv.next, %loop ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i16 0, ptr %gep.A
+ %gep.A.1 = getelementptr i32, ptr %A.1, i64 %iv
+ %l = load i32, ptr %gep.A.1
+ store i32 %l, ptr %B
+ %iv.next = add nsw i64 %iv, -1
+ %cmp = icmp eq i64 %iv, 0
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
index c966bc4e7d028..22889a0c77e6e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
define void @test_distance_positive_independent_via_trip_count(ptr %A) {
; CHECK-LABEL: 'test_distance_positive_independent_via_trip_count'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 3200 bits
+; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 800 bits
; CHECK-NEXT: Dependences:
; CHECK-NEXT: BackwardVectorizable:
; CHECK-NEXT: %l = load i8, ptr %gep.A, align 1 ->
More information about the llvm-commits
mailing list