[PATCH] D156165: [LAA] MaxSafeVectorWidthBits depends on changes to MinDepDist

[Michael Maitland via Phabricator via llvm-commits]

michaelmaitland created this revision.
michaelmaitland added reviewers: fhahn, Ayal, reames.
Herald added subscribers: StephenFan, hiraditya.
Herald added a project: All.
michaelmaitland requested review of this revision.
Herald added subscribers: llvm-commits, wangpc.
Herald added a project: LLVM.

If MinDepDist changes, then it is possible that MaxSafeVectorWidthInBits
changes since it depends on MinDepDist. Whenever we may change
MinDepDist, we must change MaxSafeVectorWidthInBits.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D156165

Files:
  llvm/include/llvm/Analysis/LoopAccessAnalysis.h
  llvm/lib/Analysis/LoopAccessAnalysis.cpp


Index: llvm/lib/Analysis/LoopAccessAnalysis.cpp
===================================================================
--- llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1662,7 +1662,8 @@
 }
 
 bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
-                                                    uint64_t TypeByteSize) {
+                                                    uint64_t TypeByteSize,
+                                                    uint64_t Stride) {
   // If loads occur at a distance that is not a multiple of a feasible vector
   // factor store-load forwarding does not take place.
   // Positive dependences might cause troubles because vectorizing them might
@@ -1700,8 +1701,14 @@
 
   if (MaxVFWithoutSLForwardIssues < MinDepDistBytes &&
       MaxVFWithoutSLForwardIssues !=
-          VectorizerParams::MaxVectorWidth * TypeByteSize)
+          VectorizerParams::MaxVectorWidth * TypeByteSize) {
     MinDepDistBytes = MaxVFWithoutSLForwardIssues;
+    // An update to MaxSafeDepDistBytes requires an update to
+    // MaxSafeVectorWidthInBits.
+    uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * Stride);
+    uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
+    MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits);
+  }
   return false;
 }
 
@@ -1900,7 +1907,8 @@
   if (Val.isNegative()) {
     bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
     if (IsTrueDataDependence && EnableForwardingConflictDetection &&
-        (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) ||
+        (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize,
+                                      Stride) ||
          !HasSameSize)) {
       LLVM_DEBUG(dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
       return Dependence::ForwardButPreventsForwarding;
@@ -1996,16 +2004,19 @@
   MinDepDistBytes =
       std::min(static_cast<uint64_t>(Distance), MinDepDistBytes);
 
-  bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
-  if (IsTrueDataDependence && EnableForwardingConflictDetection &&
-      couldPreventStoreLoadForward(Distance, TypeByteSize))
-    return Dependence::BackwardVectorizableButPreventsForwarding;
-
+  // An update to MaxSafeDepDistBytes requires an update to
+  // MaxSafeVectorWidthInBits.
   uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * Stride);
   LLVM_DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue()
                     << " with max VF = " << MaxVF << '\n');
   uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
   MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits);
+
+  bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
+  if (IsTrueDataDependence && EnableForwardingConflictDetection &&
+      couldPreventStoreLoadForward(Distance, TypeByteSize, Stride))
+    return Dependence::BackwardVectorizableButPreventsForwarding;
+
   return Dependence::BackwardVectorizable;
 }
 
Index: llvm/include/llvm/Analysis/LoopAccessAnalysis.h
===================================================================
--- llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -325,8 +325,10 @@
   /// forwarding.
   ///
   /// \return false if we shouldn't vectorize at all or avoid larger
-  /// vectorization factors by limiting MinDepDistBytes.
-  bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize);
+  /// vectorization factors by limiting MinDepDistBytes and
+  /// MaxSafeVectorWidthInBits.
+  bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize,
+                                    uint64_t Stride);
 
   /// Updates the current safety status with \p S. We can go from Safe to
   /// either PossiblySafeWithRtChecks or Unsafe and from


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D156165.543684.patch
Type: text/x-patch
Size: 3860 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20230724/67e76cd6/attachment.bin>