[llvm] [SLP] Split blocking build-vector stores in scalar chains (PR #194970)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 22:03:03 PDT 2026
================
@@ -27302,20 +27240,137 @@ bool SLPVectorizerPass::vectorizeStores(
return A && (!B || A->getStride() < B->getStride());
});
+ auto TryVectorizeVectorLaneContext = [&](StoreChainContext &Context,
+ unsigned VF) {
+ ArrayRef<StoreLane> Lanes = Context.getOperands();
+ bool HasScalarLane = any_of(
+ Lanes, [](const StoreLane &Lane) { return !Lane.IsVectorLane; });
+ // Keep the initial support transactional: every lane must belong to a
+ // full output group, so no original vector store is partly replaced.
+ if (!Context.hasVectorLane() || !HasScalarLane || Lanes.size() % VF != 0)
+ return false;
+
+ // A vector store can cover lanes that land in several output groups. For
+ // example, p[3..6] may be replaced by new stores p[0..3] and p[4..7].
+ // Only erase the original stores after every affected group is
+ // profitable.
+ SmallVector<Instruction *> StoresToReplace;
+ SmallDenseSet<Value *> UserIgnoreList;
+ for (const StoreLane &Lane : Lanes) {
+ if (is_contained(StoresToReplace, Lane.Store))
+ continue;
+ StoresToReplace.push_back(Lane.Store);
+ UserIgnoreList.insert(Lane.Store);
+ if (!Lane.IsVectorLane)
+ continue;
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> BuildVectorInsts;
+ if (!collectBuildVector(Lane.Store->getValueOperand(), Elts,
+ BuildVectorInsts))
+ return false;
+ for (Instruction *I : BuildVectorInsts)
+ UserIgnoreList.insert(I);
+ }
+
+ TypeSize EltSize =
+ DL->getTypeStoreSize(Lanes.front().ScalarValue->getType());
+ if (EltSize.isScalable())
+ return false;
+ Type *I8Ty = Type::getInt8Ty(Lanes.front().Store->getContext());
+
+ auto TryBuildStoreValue = [&](ArrayRef<StoreLane> Group,
+ Value **VectorizedValue = nullptr) {
+ SmallVector<Value *, 4> Values;
+ for (const StoreLane &Lane : Group)
+ Values.push_back(Lane.ScalarValue);
+ R.buildTree(Values, UserIgnoreList, /*IsReduction=*/false);
+ if (R.isTreeTinyAndNotFullyVectorizable())
+ return false;
+ if (R.isProfitableToReorder()) {
+ R.reorderTopToBottom();
+ R.reorderBottomToTop();
+ }
+ R.transformNodes();
+ R.computeMinimumValueSizes();
+ InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+ R.buildExternalUses();
+ InstructionCost Cost = R.getTreeCost(TreeCost);
+ if (Cost >= -SLPCostThreshold)
+ return false;
+ if (VectorizedValue)
+ *VectorizedValue = R.vectorizeTree();
+ return true;
+ };
+
+ for (unsigned I = 0, E = Lanes.size(); I != E; I += VF)
+ if (!TryBuildStoreValue(Lanes.slice(I, VF)))
+ return false;
+
+ for (unsigned I = 0, E = Lanes.size(); I != E; I += VF) {
+ ArrayRef<StoreLane> Group = Lanes.slice(I, VF);
+ Value *Vec = nullptr;
+ bool Profitable = TryBuildStoreValue(Group, &Vec);
----------------
alexey-bataev wrote:
Expensive approach, double run!
https://github.com/llvm/llvm-project/pull/194970
More information about the llvm-commits
mailing list