[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)

Alexey Bataev via llvm-commits llvm-commits at lists.llvm.org
Wed Jul 22 05:22:28 PDT 2026


================
@@ -27384,6 +27418,142 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
   return Changed;
 }
 
+bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
+                                              unsigned VF) {
+  if (!BaseStore || !LAIs)
+    return false;
+
+  StoreInst *FirstStore = BaseStore;
+
+  // Cache lookup: avoid re-walking the LAA dependence list when the same
+  // chain is retried at multiple vector factors.
+  auto Key = std::make_pair(FirstStore, VF);
+  auto CacheIt = StlfConflictCache.find(Key);
+  if (CacheIt != StlfConflictCache.end())
+    return CacheIt->second;
+
+  auto CacheAndReturn = [&](bool Result) -> bool {
+    StlfConflictCache[Key] = Result;
+    return Result;
+  };
+
+  Loop *L = LI->getLoopFor(FirstStore->getParent());
+  if (!L)
+    return CacheAndReturn(false);
+
+  Type *ValueTy = FirstStore->getValueOperand()->getType();
+  TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+  if (StoreSize.isScalable())
+    return CacheAndReturn(false);
+  uint64_t ElementSize = StoreSize.getFixedValue();
+  if (ElementSize == 0)
+    return CacheAndReturn(false);
+  uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+  LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+                    << " ElementSize=" << ElementSize
+                    << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+  // Cheap early-gate: if the loop has no loads from the same underlying
+  // object as the store chain, STLF conflicts are impossible. This avoids
+  // paying for LAA on loops that obviously cannot conflict.
+  Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+  auto HasSameBaseLoad = [&]() {
+    for (BasicBlock *BB : L->blocks())
+      for (Instruction &I : *BB)
+        if (auto *LdI = dyn_cast<LoadInst>(&I))
+          if (LdI->isSimple() &&
+              getUnderlyingObject(LdI->getPointerOperand()) == StoreBase)
+            return true;
+    return false;
+  };
+  if (!HasSameBaseLoad())
+    return CacheAndReturn(false);
+
+  const LoopAccessInfo &LAI = LAIs->getInfo(*L);
+  const MemoryDepChecker &Dep = LAI.getDepChecker();
+  const auto *Deps = Dep.getDependences();
+  if (!Deps)
+    return CacheAndReturn(false);
+
+  // Collect the set of instructions still live in the loop. LAA's cached
+  // dependence list holds raw pointers that may dangle after earlier SLP
+  // vectorization passes delete instructions from the same loop.
+  SmallPtrSet<const Instruction *, 32> LiveInsts;
+  for (BasicBlock *BB : L->blocks())
+    for (Instruction &I : *BB)
+      LiveInsts.insert(&I);
+
+  // Pass 1: collect candidate loads from STLF-relevant dependences.
+  // LAA records one representative (store, load) pair per dependence —
+  // the chosen store is an internal LAA detail, so we must not require
+  // the representative to be a member of this particular chain. Instead,
+  // collect every simple load that appears in a backward / STLF-preventing
+  // dep and accesses the same underlying object as the chain. The
+  // chain-specific distance check happens in pass 2.
+  using DepType = MemoryDepChecker::Dependence::DepType;
+  SmallPtrSet<LoadInst *, 8> CandidateLoads;
+  for (const MemoryDepChecker::Dependence &D : *Deps) {
+    switch (D.Type) {
+    case DepType::Backward:
+    case DepType::BackwardVectorizable:
+    case DepType::BackwardVectorizableButPreventsForwarding:
+    case DepType::ForwardButPreventsForwarding:
+      break;
+    default:
+      continue;
+    }
+
+    Instruction *Src = D.getSource(Dep);
+    Instruction *Dst = D.getDestination(Dep);
+
+    if (!LiveInsts.contains(Src) || !LiveInsts.contains(Dst))
+      continue;
+
+    // Either side may be the load; pick whichever is a LoadInst.
+    LoadInst *LoadI = dyn_cast<LoadInst>(Src);
----------------
alexey-bataev wrote:

```suggestion
    auto *LoadI = dyn_cast<LoadInst>(Src);
```

https://github.com/llvm/llvm-project/pull/199606


More information about the llvm-commits mailing list