[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)

[via llvm-commits]

================
@@ -27384,6 +27395,115 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
   return Changed;
 }
 
+bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
+                                                       unsigned VF) {
+  if (Chain.empty() || !LAIs)
+    return false;
+
+  auto *FirstStore = cast<StoreInst>(Chain[0]);
+  Loop *L = LI->getLoopFor(FirstStore->getParent());
+  if (!L) {
+    // STLF stalls only matter inside loops where the store buffer stays hot
+    // across iterations.
+    LLVM_DEBUG(dbgs() << "SLP: STLF check: stores not in a loop, skipping\n");
+    return false;
+  }
+
+  Type *ValueTy = FirstStore->getValueOperand()->getType();
+  TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+  // SLP only handles fixed-width vectors; bail out on scalable types.
+  if (StoreSize.isScalable())
+    return false;
+  uint64_t ElementSize = StoreSize.getFixedValue();
+  if (ElementSize == 0)
+    return false;
+  uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+  LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+                    << " ElementSize=" << ElementSize
+                    << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+  // Build a quick lookup of the chain's stores so we can pick out only the
+  // dependences that actually involve this chain. LAA computes a single
+  // per-loop store-load forwarding cap that is the minimum across *every*
+  // dep in the loop, which over-restricts when the loop contains multiple
+  // independent SLP chains. Iterating per-dep keeps us chain-granular.
+  SmallPtrSet<const Instruction *, 8> ChainStores;
+  for (Value *V : Chain)
+    if (auto *I = dyn_cast<Instruction>(V))
+      ChainStores.insert(I);
+  if (ChainStores.empty())
+    return false;
+
+  // LoopAccessInfo is computed lazily and cached per loop, so subsequent
+  // queries for chains in the same loop are O(1) (same machinery the loop
+  // vectorizer consumes).
+  const LoopAccessInfo &LAI = LAIs->getInfo(*L);
+  const MemoryDepChecker &Dep = LAI.getDepChecker();
+  const auto *Deps = Dep.getDependences();
+  if (!Deps) {
+    // LAA bailed (volatile load, indirect access, exceeded MaxDependences,
+    // ...). Be conservative and don't reject vectorization on the SLP side.
+    LLVM_DEBUG(dbgs() << "SLP: STLF: no LAA dependence list available\n");
+    return false;
+  }
+
+  // Avoid repeating the predicate work for loads that LAA reports against
+  // multiple chain stores in turn.
+  SmallPtrSet<const LoadInst *, 8> SeenLoads;
+  for (const MemoryDepChecker::Dependence &D : *Deps) {
+    Instruction *Src = D.getSource(Dep);
+    Instruction *Dst = D.getDestination(Dep);
+
+    // Only inspect deps where one end is a store from this chain and the
+    // other end is a (simple) load.
+    bool SrcInChain = ChainStores.contains(Src);
+    bool DstInChain = ChainStores.contains(Dst);
+    if (!SrcInChain && !DstInChain)
+      continue;
+    auto *LoadI = dyn_cast<LoadInst>(SrcInChain ? Dst : Src);
+    if (!LoadI || !LoadI->isSimple() || !SeenLoads.insert(LoadI).second)
+      continue;
+
+    // Compute distance from the chain's leading edge (Chain[0]) rather than
+    // the specific store LAA paired with the load: LAA may pick any chain
+    // store as the dependence representative, but the predicate cares about
+    // the chain window's start.
+    std::optional<int64_t> Diff =
+        getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
+                        LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
+                        /*StrictCheck=*/false, /*CheckType=*/false);
----------------
mbhade-amd wrote:

Thanks @alexey-bataev. Working on the fix.

https://github.com/llvm/llvm-project/pull/199606