[llvm] [SLP] Split blocking build-vector stores in scalar chains (PR #194970)

Alexey Bataev via llvm-commits llvm-commits at lists.llvm.org
Fri May 1 22:03:03 PDT 2026


================
@@ -27302,20 +27240,137 @@ bool SLPVectorizerPass::vectorizeStores(
                         return A && (!B || A->getStride() < B->getStride());
                       });
 
+    auto TryVectorizeVectorLaneContext = [&](StoreChainContext &Context,
+                                             unsigned VF) {
+      ArrayRef<StoreLane> Lanes = Context.getOperands();
+      bool HasScalarLane = any_of(
+          Lanes, [](const StoreLane &Lane) { return !Lane.IsVectorLane; });
+      // Keep the initial support transactional: every lane must belong to a
+      // full output group, so no original vector store is partly replaced.
+      if (!Context.hasVectorLane() || !HasScalarLane || Lanes.size() % VF != 0)
+        return false;
+
+      // A vector store can cover lanes that land in several output groups. For
+      // example, p[3..6] may be replaced by new stores p[0..3] and p[4..7].
+      // Only erase the original stores after every affected group is
+      // profitable.
+      SmallVector<Instruction *> StoresToReplace;
+      SmallDenseSet<Value *> UserIgnoreList;
+      for (const StoreLane &Lane : Lanes) {
+        if (is_contained(StoresToReplace, Lane.Store))
+          continue;
+        StoresToReplace.push_back(Lane.Store);
+        UserIgnoreList.insert(Lane.Store);
+        if (!Lane.IsVectorLane)
+          continue;
+        SmallVector<Value *, 16> Elts;
+        SmallVector<Instruction *, 16> BuildVectorInsts;
+        if (!collectBuildVector(Lane.Store->getValueOperand(), Elts,
+                                BuildVectorInsts))
+          return false;
+        for (Instruction *I : BuildVectorInsts)
+          UserIgnoreList.insert(I);
+      }
+
+      TypeSize EltSize =
+          DL->getTypeStoreSize(Lanes.front().ScalarValue->getType());
+      if (EltSize.isScalable())
+        return false;
+      Type *I8Ty = Type::getInt8Ty(Lanes.front().Store->getContext());
+
+      auto TryBuildStoreValue = [&](ArrayRef<StoreLane> Group,
+                                    Value **VectorizedValue = nullptr) {
+        SmallVector<Value *, 4> Values;
+        for (const StoreLane &Lane : Group)
+          Values.push_back(Lane.ScalarValue);
+        R.buildTree(Values, UserIgnoreList, /*IsReduction=*/false);
+        if (R.isTreeTinyAndNotFullyVectorizable())
+          return false;
+        if (R.isProfitableToReorder()) {
+          R.reorderTopToBottom();
+          R.reorderBottomToTop();
+        }
+        R.transformNodes();
+        R.computeMinimumValueSizes();
+        InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+        R.buildExternalUses();
+        InstructionCost Cost = R.getTreeCost(TreeCost);
+        if (Cost >= -SLPCostThreshold)
+          return false;
+        if (VectorizedValue)
+          *VectorizedValue = R.vectorizeTree();
+        return true;
+      };
+
+      for (unsigned I = 0, E = Lanes.size(); I != E; I += VF)
+        if (!TryBuildStoreValue(Lanes.slice(I, VF)))
+          return false;
+
+      for (unsigned I = 0, E = Lanes.size(); I != E; I += VF) {
+        ArrayRef<StoreLane> Group = Lanes.slice(I, VF);
+        Value *Vec = nullptr;
+        bool Profitable = TryBuildStoreValue(Group, &Vec);
----------------
alexey-bataev wrote:

Expensive approach, double run!

https://github.com/llvm/llvm-project/pull/194970


More information about the llvm-commits mailing list