[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)

[Alexey Bataev via llvm-commits]

================
@@ -28257,6 +28285,109 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
   return Changed;
 }
 
+bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
+                                              unsigned VF) {
+  if (!BaseStore)
+    return false;
+
+  StoreInst *FirstStore = BaseStore;
+
+  // Memoize per (store, VF); the entry is re-costed repeatedly.
+  auto Key = std::make_pair(FirstStore, VF);
+  auto CacheIt = StlfConflictCache.find(Key);
+  if (CacheIt != StlfConflictCache.end())
+    return CacheIt->second;
+
+  auto CacheAndReturn = [&](bool Result) -> bool {
+    StlfConflictCache[Key] = Result;
+    // No conflict at VF implies none at any smaller power-of-2 VF, so seed
+    // those entries too. Conflicts do not propagate downward.
+    if (!Result && isPowerOf2_32(VF))
+      for (unsigned V = VF / 2; V >= 2; V /= 2)
+        StlfConflictCache.try_emplace(std::make_pair(FirstStore, V), false);
+    return Result;
+  };
+
+  Loop *L = LI->getLoopFor(FirstStore->getParent());
+  if (!L)
+    return CacheAndReturn(false);
+
+  Type *ValueTy = FirstStore->getValueOperand()->getType();
+  TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+  if (StoreSize.isScalable())
+    return CacheAndReturn(false);
+  uint64_t ElementSize = StoreSize.getFixedValue();
+  if (ElementSize == 0)
+    return CacheAndReturn(false);
+  uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+  LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+                    << " ElementSize=" << ElementSize
+                    << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+  // Enumerate candidate loads from the tree, not the whole loop. A conflicting
+  // load feeds the vectorized nodes, so it is reachable by walking operands
+  // from the tree scalars (including through gather leaves such as splats).
+  // Stay inside the loop to bound the walk by the tree's cone.
+  Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+  SmallPtrSet<LoadInst *, 8> CandidateLoads;
+  SmallPtrSet<const Value *, 32> Visited;
+  SmallVector<Value *, 32> Worklist;
+  for (const std::unique_ptr<TreeEntry> &TEPtr : VectorizableTree) {
+    const TreeEntry *TE = TEPtr.get();
+    if (DeletedNodes.contains(TE))
+      continue;
+    Worklist.append(TE->Scalars.begin(), TE->Scalars.end());
+  }
----------------
alexey-bataev wrote:

You don't need to walk over each element in the tree, just find vector TEs with Opcode=Load or gather nodes, that's it

https://github.com/llvm/llvm-project/pull/199606