[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 05:22:28 PDT 2026
================
@@ -27384,6 +27418,142 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
return Changed;
}
+bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
+ unsigned VF) {
+ if (!BaseStore || !LAIs)
+ return false;
+
+ StoreInst *FirstStore = BaseStore;
+
+ // Cache lookup: avoid re-walking the LAA dependence list when the same
+ // chain is retried at multiple vector factors.
+ auto Key = std::make_pair(FirstStore, VF);
+ auto CacheIt = StlfConflictCache.find(Key);
+ if (CacheIt != StlfConflictCache.end())
+ return CacheIt->second;
+
+ auto CacheAndReturn = [&](bool Result) -> bool {
+ StlfConflictCache[Key] = Result;
+ return Result;
+ };
+
+ Loop *L = LI->getLoopFor(FirstStore->getParent());
+ if (!L)
+ return CacheAndReturn(false);
+
+ Type *ValueTy = FirstStore->getValueOperand()->getType();
+ TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+ if (StoreSize.isScalable())
+ return CacheAndReturn(false);
+ uint64_t ElementSize = StoreSize.getFixedValue();
+ if (ElementSize == 0)
+ return CacheAndReturn(false);
+ uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+ << " ElementSize=" << ElementSize
+ << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+ // Cheap early-gate: if the loop has no loads from the same underlying
+ // object as the store chain, STLF conflicts are impossible. This avoids
+ // paying for LAA on loops that obviously cannot conflict.
+ Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+ auto HasSameBaseLoad = [&]() {
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB)
+ if (auto *LdI = dyn_cast<LoadInst>(&I))
+ if (LdI->isSimple() &&
+ getUnderlyingObject(LdI->getPointerOperand()) == StoreBase)
+ return true;
+ return false;
+ };
+ if (!HasSameBaseLoad())
+ return CacheAndReturn(false);
+
+ const LoopAccessInfo &LAI = LAIs->getInfo(*L);
+ const MemoryDepChecker &Dep = LAI.getDepChecker();
+ const auto *Deps = Dep.getDependences();
+ if (!Deps)
+ return CacheAndReturn(false);
+
+ // Collect the set of instructions still live in the loop. LAA's cached
+ // dependence list holds raw pointers that may dangle after earlier SLP
+ // vectorization passes delete instructions from the same loop.
+ SmallPtrSet<const Instruction *, 32> LiveInsts;
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB)
+ LiveInsts.insert(&I);
+
+ // Pass 1: collect candidate loads from STLF-relevant dependences.
+ // LAA records one representative (store, load) pair per dependence —
+ // the chosen store is an internal LAA detail, so we must not require
+ // the representative to be a member of this particular chain. Instead,
+ // collect every simple load that appears in a backward / STLF-preventing
+ // dep and accesses the same underlying object as the chain. The
+ // chain-specific distance check happens in pass 2.
+ using DepType = MemoryDepChecker::Dependence::DepType;
+ SmallPtrSet<LoadInst *, 8> CandidateLoads;
+ for (const MemoryDepChecker::Dependence &D : *Deps) {
+ switch (D.Type) {
+ case DepType::Backward:
+ case DepType::BackwardVectorizable:
+ case DepType::BackwardVectorizableButPreventsForwarding:
+ case DepType::ForwardButPreventsForwarding:
+ break;
+ default:
+ continue;
+ }
+
+ Instruction *Src = D.getSource(Dep);
+ Instruction *Dst = D.getDestination(Dep);
+
+ if (!LiveInsts.contains(Src) || !LiveInsts.contains(Dst))
+ continue;
+
+ // Either side may be the load; pick whichever is a LoadInst.
+ LoadInst *LoadI = dyn_cast<LoadInst>(Src);
----------------
alexey-bataev wrote:
```suggestion
auto *LoadI = dyn_cast<LoadInst>(Src);
```
https://github.com/llvm/llvm-project/pull/199606
More information about the llvm-commits
mailing list