[llvm] [SLP] Split blocking build-vector stores in scalar chains (PR #194970)

[Yaxun Liu via llvm-commits]

================
@@ -27270,6 +27270,90 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
   Stores.clear();
   GEPs.clear();
 
+  SmallVector<StoreInst *> StoresToErase;
+  // Recognize a vector value built lane-by-lane with insertelement:
+  //
+  //   %v0 = insertelement <4 x float> poison, float %a, i32 0
+  //   %v1 = insertelement <4 x float> %v0,    float %b, i32 1
+  //
+  // Such a value can be split back into the scalar lane values without changing
+  // semantics. Do not handle shuffles or partial vectors here; those need the
+  // normal SLP cost model.
+  auto CollectBuildVector = [](Value *V, SmallVectorImpl<Value *> &Elts) {
+    auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+    if (!VecTy)
+      return false;
+    Elts.assign(VecTy->getNumElements(), nullptr);
+    Value *Cur = V;
+    while (auto *IE = dyn_cast<InsertElementInst>(Cur)) {
+      auto *Idx = dyn_cast<ConstantInt>(IE->getOperand(2));
+      if (!Idx || Idx->getValue().uge(Elts.size()))
+        return false;
+      Elts[Idx->getZExtValue()] = IE->getOperand(1);
+      Cur = IE->getOperand(0);
+    }
+    if (!isa<PoisonValue, UndefValue>(Cur))
+      return false;
+    return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
+  };
+
+  for (Instruction &I : make_early_inc_range(*BB)) {
+    auto *SI = dyn_cast<StoreInst>(&I);
+    if (!SI || !SI->isSimple())
+      continue;
+
+    SmallVector<Value *, 16> BuildVectorOpds;
+    if (!CollectBuildVector(SI->getValueOperand(), BuildVectorOpds))
+      continue;
+
+    Type *EltTy = BuildVectorOpds.front()->getType();
+    Value *UnderlyingObject = getUnderlyingObject(SI->getPointerOperand());
+    bool HasScalarBefore = false;
+    bool HasScalarAfter = false;
+    for (Instruction &J : *BB) {
+      auto *OtherSI = dyn_cast<StoreInst>(&J);
+      if (!OtherSI || OtherSI == SI || !OtherSI->isSimple() ||
+          OtherSI->getValueOperand()->getType() != EltTy ||
+          getUnderlyingObject(OtherSI->getPointerOperand()) != UnderlyingObject)
+        continue;
+
+      std::optional<int64_t> Diff =
+          getPointersDiff(EltTy, SI->getPointerOperand(), EltTy,
+                          OtherSI->getPointerOperand(), *DL, *SE,
+                          /*StrictCheck=*/true);
+      if (!Diff)
+        continue;
+      HasScalarBefore |= *Diff < 0;
+      HasScalarAfter |= *Diff >= static_cast<int64_t>(BuildVectorOpds.size());
+    }
+    if (!HasScalarBefore || !HasScalarAfter)
+      continue;
+
+    // An existing vector store can hide a better scalar store chain from SLP.
+    // For example:
+    //
+    //   store float %a, ptr %p
+    //   store <4 x float> %v, ptr %p+4
+    //   store float %b, ptr %p+20
+    //
+    // SLP collects scalar stores as seeds, so the middle vector store blocks it
+    // from seeing one continuous store range. Split only when scalar stores to
+    // the same object appear on both sides; the later store-chain vectorizer
+    // can then decide the final profitable vector layout.
+    IRBuilder<> Builder(SI);
+    uint64_t EltSize = DL->getTypeStoreSize(EltTy).getFixedValue();
+    Type *I8Ty = Type::getInt8Ty(SI->getContext());
+    for (auto [Idx, V] : enumerate(BuildVectorOpds)) {
+      Value *Ptr = Builder.CreateConstGEP1_64(I8Ty, SI->getPointerOperand(),
+                                              Idx * EltSize);
+      Builder.CreateAlignedStore(
----------------
yxsamliu wrote:

Thanks, that makes sense. I reworked this to model the build-vector store inside SLP instead of creating temporary scalar store IR.

https://github.com/llvm/llvm-project/pull/194970