[llvm] [SLP] Split blocking build-vector stores in scalar chains (PR #194970)
Yaxun Liu via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 22:02:32 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/194970
>From 0d5b7c1efe259918de9fe74028f0d98cc5f25e00 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 4 May 2026 01:05:54 -0400
Subject: [PATCH 1/4] [SLP] Recognize build-vector stores as store-chain
candidates
A build-vector store like `store <a, b, c, d>, p` (built from a
chain of insertelements) is currently invisible to SLP's store-chain
analysis.
Add helpers (collectBuildVector, getStoreChainType,
isBuildVectorStore) and relax the input gates so a build-vector
store of <N x T> is treated as compatible with scalar stores of T
at the same address.
No behavior change yet: vectorizeStores filters build-vector stores
back out at entry. Later commits remove the filter and merge mixed
chains.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 96 ++++++++++++++++---
1 file changed, 85 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a18df9c0882ab..cbcc5223c8d24 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26678,6 +26678,56 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
}
namespace {
+/// Walk a build-vector chain (a sequence of insertelement instructions feeding
+/// undef/poison) starting at \p V and collect the lane values plus the chain
+/// instructions. Returns true when V is a complete fixed-vector buildvector
+/// that fully covers all lanes.
+///
+/// The walk goes from the stored value toward the root, so for a lane that is
+/// inserted more than once, the later insert is seen first and wins (matching
+/// the runtime semantics of repeated insertelement).
+static bool collectBuildVector(Value *V, SmallVectorImpl<Value *> &Elts,
+ SmallVectorImpl<Instruction *> &Insts) {
+ auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!VecTy)
+ return false;
+ Elts.assign(VecTy->getNumElements(), nullptr);
+ Value *Cur = V;
+ while (auto *IE = dyn_cast<InsertElementInst>(Cur)) {
+ if (!IE->hasOneUse())
+ return false;
+ auto *Idx = dyn_cast<ConstantInt>(IE->getOperand(2));
+ if (!Idx || Idx->getValue().uge(Elts.size()))
+ return false;
+ unsigned LaneIdx = Idx->getZExtValue();
+ if (!Elts[LaneIdx])
+ Elts[LaneIdx] = IE->getOperand(1);
+ Insts.push_back(IE);
+ Cur = IE->getOperand(0);
+ }
+ if (!isa<UndefValue>(Cur))
+ return false;
+ return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
+}
+
+/// Return the lane-element type of \p SI. For a normal scalar store this is
+/// the value type; for a build-vector store it is the inserted-element type.
+static Type *getStoreChainType(StoreInst *SI) {
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> Insts;
+ if (collectBuildVector(SI->getValueOperand(), Elts, Insts))
+ return Elts.front()->getType();
+ return SI->getValueOperand()->getType();
+}
+
+/// Returns true if SI's value is built from a fixed-vector insertelement chain
+/// rooted at undef/poison.
+static bool isBuildVectorStore(StoreInst *SI) {
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> Insts;
+ return collectBuildVector(SI->getValueOperand(), Elts, Insts);
+}
+
/// A group of related stores which we are in the process of vectorizing,
/// a subset of which may already be vectorized. Stores context information
/// about the group as a whole as well as information about what VFs need
@@ -27249,6 +27299,22 @@ bool SLPVectorizerPass::vectorizeStores(
ArrayRef<StoreInst *> Stores, BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited) {
+ // Build-vector stores reach this entry through the lane-aware sorter/matcher
+ // updates earlier in this commit, but the chain machinery here still treats
+ // each StoreInst as one lane. Filter them out for now and process only the
+ // scalar stores; a later commit teaches vectorizeStores to expand build-
+ // vector stores into per-lane elements and dispatch through a dedicated
+ // TreeEntry state.
+ SmallVector<StoreInst *> ScalarOnly;
+ if (any_of(Stores, isBuildVectorStore)) {
+ for (StoreInst *SI : Stores)
+ if (!isBuildVectorStore(SI))
+ ScalarOnly.push_back(SI);
+ Stores = ScalarOnly;
+ }
+ if (Stores.size() < 2)
+ return false;
+
// We may run into multiple chains that merge into a single chain. We mark the
// stores that we vectorized so that we don't visit the same store twice.
BoUpSLP::ValueSet VectorizedStores;
@@ -27467,7 +27533,9 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
if (auto *SI = dyn_cast<StoreInst>(&I)) {
if (!SI->isSimple())
continue;
- if (!isValidElementType(SI->getValueOperand()->getType()))
+ // Build-vector stores are accepted: their per-lane element type drives
+ // the chain so a store of <4 x float> behaves like 4 lanes of float.
+ if (!isValidElementType(getStoreChainType(SI)))
continue;
Stores[getUnderlyingObject(SI->getPointerOperand())].push_back(SI);
}
@@ -31258,11 +31326,13 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
// compatible (have the same opcode, same parent), otherwise it is
// definitely not profitable to try to vectorize them.
auto &&StoreSorter = [this](StoreInst *V, StoreInst *V2) {
- if (V->getValueOperand()->getType()->getTypeID() <
- V2->getValueOperand()->getType()->getTypeID())
+ // Sort by lane-element type so build-vector stores group with scalar
+ // stores of the same element type (e.g. <4 x float> with float).
+ Type *Ty = getStoreChainType(V);
+ Type *Ty2 = getStoreChainType(V2);
+ if (Ty->getTypeID() < Ty2->getTypeID())
return true;
- if (V->getValueOperand()->getType()->getTypeID() >
- V2->getValueOperand()->getType()->getTypeID())
+ if (Ty->getTypeID() > Ty2->getTypeID())
return false;
if (V->getPointerOperandType()->getTypeID() <
V2->getPointerOperandType()->getTypeID())
@@ -31270,11 +31340,9 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
if (V->getPointerOperandType()->getTypeID() >
V2->getPointerOperandType()->getTypeID())
return false;
- if (V->getValueOperand()->getType()->getScalarSizeInBits() <
- V2->getValueOperand()->getType()->getScalarSizeInBits())
+ if (Ty->getScalarSizeInBits() < Ty2->getScalarSizeInBits())
return true;
- if (V->getValueOperand()->getType()->getScalarSizeInBits() >
- V2->getValueOperand()->getType()->getScalarSizeInBits())
+ if (Ty->getScalarSizeInBits() > Ty2->getScalarSizeInBits())
return false;
// UndefValues are compatible with all other values.
auto *I1 = dyn_cast<Instruction>(V->getValueOperand());
@@ -31308,10 +31376,16 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
StoreInst *V2 = VL.back();
if (V1 == V2)
return true;
- if (V1->getValueOperand()->getType() != V2->getValueOperand()->getType())
+ bool IsBuildVectorStore = isBuildVectorStore(V1) || isBuildVectorStore(V2);
+ if (getStoreChainType(V1) != getStoreChainType(V2))
return false;
if (V1->getPointerOperandType() != V2->getPointerOperandType())
return false;
+ // Build-vector stores skip the value-operand-instruction compatibility
+ // check below since their value comes from an insertelement chain. A
+ // later commit handles them via a dedicated TreeEntry path.
+ if (IsBuildVectorStore)
+ return true;
// Undefs are compatible with any other value.
if (isa<UndefValue>(V1->getValueOperand()) ||
isa<UndefValue>(V2->getValueOperand()))
@@ -31357,7 +31431,7 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length "
<< Pair.second.size() << ".\n");
- if (!isValidElementType(Pair.second.front()->getValueOperand()->getType()))
+ if (!isValidElementType(getStoreChainType(Pair.second.front())))
continue;
// Reverse stores to do bottom-to-top analysis. This is important if the
>From c5eb34f7bf93492a83a7666d94d22956c0d2d99d Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 4 May 2026 01:06:08 -0400
Subject: [PATCH 2/4] [SLP] Model store chains as lane sequences
Refactor RelatedStoreInsts and StoreChainContext to track stores as
"lanes" instead of raw StoreInst pointers, so one build-vector store
can contribute several lanes. Example:
store <a, b, c, d>, p[3]
is modeled as 4 lanes at p[3], p[4], p[5], p[6] sharing the same
StoreInst.
No behavior change: vectorizeOneVF's callback still declines mixed
slices. The follow-up commit lifts that decline.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 227 ++++++++++++------
1 file changed, 152 insertions(+), 75 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cbcc5223c8d24..6b887baba199d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26728,6 +26728,44 @@ static bool isBuildVectorStore(StoreInst *SI) {
return collectBuildVector(SI->getValueOperand(), Elts, Insts);
}
+/// A store-chain element after breaking stores into scalar lanes.
+///
+/// A scalar store contributes one lane. A simple build-vector store contributes
+/// one lane per inserted element, so a store of <a, b, c, d> to p[3] is
+/// modeled as lanes p[3] = a, p[4] = b, p[5] = c, p[6] = d.
+///
+/// Store: the original StoreInst that this lane belongs to (the same StoreInst
+/// is shared by every lane of a build-vector store).
+/// ScalarValue: the value stored at this lane (the value operand for a scalar
+/// store, or the i-th inserted element for a build-vector store).
+/// Offset: the chain distance, set to 0 at construction and rewritten by the
+/// chain analyzer to the lane's distance from the chain base. Distinct from
+/// Lane so that distance and build-vector position are not conflated.
+/// Lane: the lane's position inside its build-vector store (0 for scalar
+/// stores).
+/// IsVectorLane: true when this lane comes from a build-vector store.
+struct StoreLane {
+ StoreInst *Store = nullptr;
+ Value *ScalarValue = nullptr;
+ int64_t Offset = 0;
+ unsigned Lane = 0;
+ bool IsVectorLane = false;
+
+ StoreLane() = default;
+ StoreLane(StoreInst *Store, Value *ScalarValue, int64_t Offset, unsigned Lane,
+ bool IsVectorLane)
+ : Store(Store), ScalarValue(ScalarValue), Offset(Offset), Lane(Lane),
+ IsVectorLane(IsVectorLane) {}
+
+ /// Per-lane alignment, taking the lane offset within a build-vector store
+ /// into account.
+ Align getAlignment(TypeSize EltSize) const {
+ uint64_t StoreOffset = IsVectorLane ? Lane : 0;
+ return commonAlignment(Store->getAlign(),
+ StoreOffset * EltSize.getFixedValue());
+ }
+};
+
/// A group of related stores which we are in the process of vectorizing,
/// a subset of which may already be vectorized. Stores context information
/// about the group as a whole as well as information about what VFs need
@@ -26736,7 +26774,7 @@ class StoreChainContext {
public:
using SizePair = std::pair<unsigned, unsigned>;
- explicit StoreChainContext(ArrayRef<Value *> Ops,
+ explicit StoreChainContext(ArrayRef<StoreLane> Ops,
ArrayRef<SizePair> RangeSizes,
SmallVector<unsigned> &RangeSizesByIdx,
unsigned Stride)
@@ -26754,17 +26792,24 @@ class StoreChainContext {
unsigned getMaxVF() const { return MaxVF; }
/// Return the stride of the context
unsigned getStride() const { return Stride; }
+ /// Return the lane operands that compose this chain.
+ ArrayRef<StoreLane> getOperands() const { return Operands; }
+ /// Return true if the chain contains at least one build-vector store lane.
+ bool hasVectorLane() const {
+ return any_of(Operands,
+ [](const StoreLane &Lane) { return Lane.IsVectorLane; });
+ }
/// Attempt to vectorize Operands for the given VF
/// Returns false if no more attempts should be made for the context
bool vectorizeOneVF(const TargetTransformInfo &TTI, unsigned VF,
BoUpSLP::ValueSet &VectorizedStores, bool &Changed,
llvm::function_ref<std::optional<bool>(
- ArrayRef<Value *>, unsigned, unsigned, unsigned &)>
+ ArrayRef<StoreLane>, unsigned, unsigned, unsigned &)>
VectorizeStoreChain);
/// Add an additional store to the chain
/// \p Store store too append to Operands
/// \p Idx position within TryToVectorize::StoreSeq
- void addOperand(Value *Store, unsigned Idx);
+ void addOperand(StoreLane Store, unsigned Idx);
private:
bool isNotVectorized(const SizePair &P) const {
@@ -26841,8 +26886,8 @@ class StoreChainContext {
Type *StoreTy = nullptr;
/// Which VFs do we want to attempt for this chain
std::queue<unsigned> CandidateVFs;
- /// Stores that compose this chain
- BoUpSLP::ValueList Operands;
+ /// Stores that compose this chain.
+ SmallVector<StoreLane> Operands;
/// Track the TreeSizes of prior vectorization attempts using each element,
/// to help us find early exit cases
/// - first: contains pointer into RangeSizesByIdx to help us track
@@ -26867,7 +26912,7 @@ class StoreChainContext {
SmallDenseMap<Value *, SizePair> NonSchedulable;
};
-void StoreChainContext::addOperand(Value *Store, unsigned Idx) {
+void StoreChainContext::addOperand(StoreLane Store, unsigned Idx) {
Operands.push_back(Store);
RangeSizesStorage.push_back({Idx, 1});
}
@@ -26904,10 +26949,8 @@ bool StoreChainContext::initializeContext(
assert((Stride == 1 || !SLPReVec) &&
"Strided stores not supported for revectorization");
if (!Visited
- .insert({Operands.front(),
- cast<StoreInst>(Operands.front())->getValueOperand(),
- Operands.back(),
- cast<StoreInst>(Operands.back())->getValueOperand(),
+ .insert({Operands.front().Store, Operands.front().ScalarValue,
+ Operands.back().Store, Operands.back().ScalarValue,
Operands.size()})
.second)
return false;
@@ -26916,15 +26959,23 @@ bool StoreChainContext::initializeContext(
RangeSizes = MutableArrayRef(RangeSizesStorage);
unsigned MaxVecRegSize = R.getMaxVecRegSize();
- unsigned EltSize = R.getVectorElementSize(Operands[0]);
+ // Use the lane element type for sizing build-vector store lanes (one
+ // inserted element per lane). Scalar lanes preserve the StoreInst-specific
+ // sizing path (which can follow trunc/sext through the value).
+ unsigned EltSize =
+ Operands[0].IsVectorLane
+ ? DL.getTypeSizeInBits(Operands[0].ScalarValue->getType())
+ : R.getVectorElementSize(Operands[0].Store);
unsigned MaxElts = llvm::bit_floor(MaxVecRegSize / EltSize);
MaxVF = std::min(R.getMaximumVF(EltSize, Instruction::Store), MaxElts);
- auto *Store = cast<StoreInst>(Operands[0]);
- StoreTy = Store->getValueOperand()->getType();
+ StoreInst *Store = Operands[0].Store;
+ StoreTy = Operands[0].IsVectorLane ? Operands[0].ScalarValue->getType()
+ : Store->getValueOperand()->getType();
Type *ValueTy = StoreTy;
- if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
- ValueTy = Trunc->getSrcTy();
+ if (!Operands[0].IsVectorLane)
+ if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
+ ValueTy = Trunc->getSrcTy();
// When REVEC is enabled, StoreTy and ValueTy may be FixedVectorType. But
// getStoreMinimumVF only support scalar type as arguments. As a result,
// we need to use the element type of StoreTy and ValueTy to retrieve the
@@ -27098,7 +27149,7 @@ bool StoreChainContext::checkTreeSizes(const unsigned SliceStartIdx,
bool StoreChainContext::vectorizeOneVF(
const TargetTransformInfo &TTI, unsigned VF,
BoUpSLP::ValueSet &VectorizedStores, bool &Changed,
- llvm::function_ref<std::optional<bool>(ArrayRef<Value *>, unsigned,
+ llvm::function_ref<std::optional<bool>(ArrayRef<StoreLane>, unsigned,
unsigned, unsigned &)>
VectorizeStoreChain) {
bool AnyProfitableGraph = false;
@@ -27115,18 +27166,16 @@ bool StoreChainContext::vectorizeOneVF(
++SliceStartIdx;
continue;
}
- ArrayRef<Value *> Slice = ArrayRef(Operands).slice(SliceStartIdx, VF);
+ ArrayRef<StoreLane> Slice = ArrayRef(Operands).slice(SliceStartIdx, VF);
assert(all_of(Slice,
- [&](Value *V) {
- return cast<StoreInst>(V)->getValueOperand()->getType() ==
- cast<StoreInst>(Slice.front())
- ->getValueOperand()
- ->getType();
+ [&](const StoreLane &Lane) {
+ return Lane.ScalarValue->getType() ==
+ Slice.front().ScalarValue->getType();
}) &&
- "Expected all operands of same type.");
+ "Expected all lanes of same scalar type.");
if (!NonSchedulable.empty()) {
auto [NonSchedSizeMax, NonSchedSizeMin] =
- NonSchedulable.lookup(Slice.front());
+ NonSchedulable.lookup(Slice.front().Store);
if (NonSchedSizeMax > 0 && NonSchedSizeMin <= VF) {
// VF is too ambitious. Try to vectorize another slice before
// trying a smaller VF.
@@ -27140,13 +27189,15 @@ bool StoreChainContext::vectorizeOneVF(
if (!Res) {
// Update the range of non schedulable VFs for slices starting
// at SliceStartIdx.
- NonSchedulable.try_emplace(Slice.front(), std::make_pair(VF, VF))
+ NonSchedulable.try_emplace(Slice.front().Store, std::make_pair(VF, VF))
.first->getSecond()
.second = VF;
} else if (*Res) {
// Mark the vectorized stores so that we don't vectorize them
- // again.
- VectorizedStores.insert_range(Slice);
+ // again. Use the underlying StoreInsts since multiple lanes may
+ // share one StoreInst (build-vector store).
+ for (const StoreLane &Lane : Slice)
+ VectorizedStores.insert(Lane.Store);
AnyProfitableGraph = RepeatChanged = Changed = true;
// If we vectorized initial block, no need to try to vectorize
// it again.
@@ -27217,7 +27268,7 @@ bool StoreChainContext::vectorizeOneVF(
/// address of this group's BaseInstr.
class RelatedStoreInsts {
public:
- RelatedStoreInsts(unsigned BaseInstrIdx, ArrayRef<StoreInst *> AllStores)
+ RelatedStoreInsts(unsigned BaseInstrIdx, ArrayRef<StoreLane> AllStores)
: AllStores(AllStores) {
reset(BaseInstrIdx);
}
@@ -27243,14 +27294,21 @@ class RelatedStoreInsts {
const DistToInstMap &getStores() const { return Instrs; }
/// If \p SI is related to this group of stores, return the distance of its
- /// pointer operand to the one the group's BaseInstr.
- std::optional<int64_t> getPointerDiff(StoreInst &SI, const DataLayout &DL,
+ /// pointer operand to the group's BaseInstr. The distance accounts for the
+ /// build-vector lane offset within \p SI's underlying StoreInst, so for a
+ /// store of <a, b, c, d> at p[3] lane #1 reports distance 4 (in element
+ /// units) relative to a chain rooted at p[3].
+ std::optional<int64_t> getPointerDiff(const StoreLane &SI,
+ const DataLayout &DL,
ScalarEvolution &SE) const {
- StoreInst &BaseStore = *AllStores[BaseInstrIdx];
- return getPointersDiff(
- BaseStore.getValueOperand()->getType(), BaseStore.getPointerOperand(),
- SI.getValueOperand()->getType(), SI.getPointerOperand(), DL, SE,
+ const StoreLane &BaseStore = AllStores[BaseInstrIdx];
+ std::optional<int64_t> Diff = getPointersDiff(
+ BaseStore.ScalarValue->getType(), BaseStore.Store->getPointerOperand(),
+ SI.ScalarValue->getType(), SI.Store->getPointerOperand(), DL, SE,
/*StrictCheck=*/true);
+ if (!Diff)
+ return std::nullopt;
+ return *Diff + SI.Lane - BaseStore.Lane;
}
/// Recompute the pointer distances to be based on \p NewBaseInstIdx.
@@ -27273,7 +27331,7 @@ class RelatedStoreInsts {
void clearVectorizedStores(const BoUpSLP::ValueSet &VectorizedStores) {
DistToInstMap::reverse_iterator LastVectorizedStore = find_if(
reverse(Instrs), [&](const std::pair<int64_t, unsigned> &DistAndIdx) {
- return VectorizedStores.contains(AllStores[DistAndIdx.second]);
+ return VectorizedStores.contains(AllStores[DistAndIdx.second].Store);
});
// Get a forward iterator pointing after the last vectorized store and erase
@@ -27289,8 +27347,8 @@ class RelatedStoreInsts {
/// Maps a pointer distance from \p BaseInstrIdx to an instruction index.
DistToInstMap Instrs;
- /// Reference to all the stores in the BB being analyzed.
- ArrayRef<StoreInst *> AllStores;
+ /// Reference to all the lanes in the BB being analyzed.
+ ArrayRef<StoreLane> AllStores;
};
} // end anonymous namespace
@@ -27299,21 +27357,23 @@ bool SLPVectorizerPass::vectorizeStores(
ArrayRef<StoreInst *> Stores, BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited) {
- // Build-vector stores reach this entry through the lane-aware sorter/matcher
- // updates earlier in this commit, but the chain machinery here still treats
- // each StoreInst as one lane. Filter them out for now and process only the
- // scalar stores; a later commit teaches vectorizeStores to expand build-
- // vector stores into per-lane elements and dispatch through a dedicated
- // TreeEntry state.
- SmallVector<StoreInst *> ScalarOnly;
- if (any_of(Stores, isBuildVectorStore)) {
- for (StoreInst *SI : Stores)
- if (!isBuildVectorStore(SI))
- ScalarOnly.push_back(SI);
- Stores = ScalarOnly;
- }
- if (Stores.size() < 2)
- return false;
+ // Convert collected stores into per-lane operands. A scalar store
+ // contributes one lane; a build-vector store contributes one lane per
+ // inserted element. Lane.Offset is the chain distance and is filled in
+ // later by the chain analyzer; Lane.Lane is the build-vector position.
+ SmallVector<StoreLane> StoreLanes;
+ for (StoreInst *SI : Stores) {
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> Insts;
+ if (collectBuildVector(SI->getValueOperand(), Elts, Insts)) {
+ for (auto [Idx, V] : enumerate(Elts))
+ StoreLanes.emplace_back(SI, V, /*Offset=*/0, static_cast<unsigned>(Idx),
+ /*IsVectorLane=*/true);
+ continue;
+ }
+ StoreLanes.emplace_back(SI, SI->getValueOperand(), /*Offset=*/0, /*Lane=*/0,
+ /*IsVectorLane=*/false);
+ }
// We may run into multiple chains that merge into a single chain. We mark the
// stores that we vectorized so that we don't visit the same store twice.
@@ -27323,7 +27383,6 @@ bool SLPVectorizerPass::vectorizeStores(
auto TryToVectorize = [&](const RelatedStoreInsts::DistToInstMap &StoreSeq) {
SmallVector<unsigned> RangeSizesByIdx(StoreSeq.size(), 1);
SmallVector<std::unique_ptr<StoreChainContext>> AllContexts;
- BoUpSLP::ValueList Operands;
SmallVector<StoreChainContext::SizePair> RangeSizes;
const unsigned MaxStride = EnableStridedStores ? MaxProfitableStride : 1;
@@ -27338,8 +27397,8 @@ bool SLPVectorizerPass::vectorizeStores(
// Index into StoreSeq if not added to AllContexts yet
unsigned StoreSeqIdx;
};
- // If not added to AllContexts, what is the single store in the chain
- Value *FirstStore;
+ // If not added to AllContexts, what is the single lane in the chain
+ StoreLane FirstStore;
// What is the Stride of this chain
unsigned Stride;
};
@@ -27354,6 +27413,8 @@ bool SLPVectorizerPass::vectorizeStores(
int64_t LastDist;
for (auto [Idx, Data] : enumerate(StoreSeq)) {
auto &[Dist, InstIdx] = Data;
+ StoreLane Lane = StoreLanes[InstIdx];
+ Lane.Offset = Dist;
// Clean up chains that can't be continued
if (Idx > 0)
for (int64_t D = LastDist;
@@ -27367,12 +27428,12 @@ bool SLPVectorizerPass::vectorizeStores(
for (auto &Status : Chains[GetChainsKey(Dist)]) {
if (Status.AddedToAllContexts) {
// Chain already in AllContexts()
- AllContexts[Status.AllContextsIdx]->addOperand(Stores[InstIdx], Idx);
+ AllContexts[Status.AllContextsIdx]->addOperand(Lane, Idx);
} else {
// Chain just a single element, not yet in AllContexts()
SmallVector<StoreChainContext::SizePair> RS = {
{Status.StoreSeqIdx, 1}, {Idx, 1}};
- BoUpSLP::ValueList Ops = {Status.FirstStore, Stores[InstIdx]};
+ SmallVector<StoreLane> Ops = {Status.FirstStore, Lane};
AllContexts.emplace_back(std::make_unique<StoreChainContext>(
Ops, RS, RangeSizesByIdx, Status.Stride));
Status.AllContextsIdx = AllContexts.size() - 1;
@@ -27380,7 +27441,7 @@ bool SLPVectorizerPass::vectorizeStores(
unsigned Key = GetChainsKey(Status.Stride + Dist);
Chains[Key].push_back({/*AddedToAllContexts=*/true,
{Status.AllContextsIdx},
- /*FirstStore=*/nullptr,
+ StoreLane(),
Status.Stride});
FoundStrides[Status.Stride] = true;
}
@@ -27393,7 +27454,7 @@ bool SLPVectorizerPass::vectorizeStores(
unsigned Key = GetChainsKey(Dist + Stride);
Chains[Key].push_back({/*AddedToAllContexts=*/false,
{/*StoreSeqIdx=*/(unsigned)Idx},
- Stores[InstIdx],
+ Lane,
Stride});
}
}
@@ -27423,9 +27484,23 @@ bool SLPVectorizerPass::vectorizeStores(
unsigned VF = *VFUnval;
if (!Context.vectorizeOneVF(
*TTI, VF, VectorizedStores, Changed,
- [this, &R](ArrayRef<Value *> Chain, unsigned Idx,
- unsigned MinVF, unsigned &Size) {
- return vectorizeStoreChain(Chain, R, Idx, MinVF, Size);
+ [this, &R](ArrayRef<StoreLane> Chain, unsigned Idx,
+ unsigned MinVF,
+ unsigned &Size) -> std::optional<bool> {
+ // Mixed scalar/build-vector slices need a dedicated
+ // TreeEntry path which is added by a follow-up commit.
+ // For now decline them so the scalar-only path is
+ // unchanged but keep the slice attempt counted.
+ if (any_of(Chain, [](const StoreLane &Lane) {
+ return Lane.IsVectorLane;
+ })) {
+ Size = 1;
+ return false;
+ }
+ SmallVector<Value *> Stores;
+ for (const StoreLane &Lane : Chain)
+ Stores.push_back(Lane.Store);
+ return vectorizeStoreChain(Stores, R, Idx, MinVF, Size);
})) {
CtxPtr.reset();
break;
@@ -27470,17 +27545,18 @@ bool SLPVectorizerPass::vectorizeStores(
// after previous store with the same distance most likely have memory
// dependencies and no need to waste compile time to try to vectorize them.
// - Try to vectorize the sequence {1, {1, 0}, {3, 2}}.
- auto FillStoresSet = [&](unsigned Idx, StoreInst *SI) {
+ auto FillStoresSet = [&](unsigned Idx, const StoreLane &Lane) {
std::optional<int64_t> PtrDist;
- auto *RelatedStores = find_if(
- SortedStores, [&PtrDist, SI, this](const RelatedStoreInsts &StoreSeq) {
- PtrDist = StoreSeq.getPointerDiff(*SI, *DL, *SE);
- return PtrDist.has_value();
- });
+ auto *RelatedStores =
+ find_if(SortedStores,
+ [&PtrDist, &Lane, this](const RelatedStoreInsts &StoreSeq) {
+ PtrDist = StoreSeq.getPointerDiff(Lane, *DL, *SE);
+ return PtrDist.has_value();
+ });
// We did not find a comparable store, start a new group.
if (RelatedStores == SortedStores.end()) {
- SortedStores.emplace_back(Idx, Stores);
+ SortedStores.emplace_back(Idx, StoreLanes);
return;
}
@@ -27497,19 +27573,20 @@ bool SLPVectorizerPass::vectorizeStores(
}
};
Type *PrevValTy = nullptr;
- for (auto [I, SI] : enumerate(Stores)) {
- if (R.isDeleted(SI))
+ for (auto [I, Lane] : enumerate(StoreLanes)) {
+ if (R.isDeleted(Lane.Store))
continue;
+ Type *LaneTy = Lane.ScalarValue->getType();
if (!PrevValTy)
- PrevValTy = SI->getValueOperand()->getType();
+ PrevValTy = LaneTy;
// Check that we do not try to vectorize stores of different types.
- if (PrevValTy != SI->getValueOperand()->getType()) {
+ if (PrevValTy != LaneTy) {
for (RelatedStoreInsts &StoreSeq : SortedStores)
TryToVectorize(StoreSeq.getStores());
SortedStores.clear();
- PrevValTy = SI->getValueOperand()->getType();
+ PrevValTy = LaneTy;
}
- FillStoresSet(I, SI);
+ FillStoresSet(I, Lane);
}
// Final vectorization attempt.
>From 1ba47e020dd45c147cc3174c74ebdcd0860c7c17 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 4 May 2026 01:06:29 -0400
Subject: [PATCH 3/4] [SLP] Vectorize mixed scalar/build-vector store chains
Merge a chain of scalar stores and adjacent build-vector stores into
one wider vector store. Example:
store float %v0, p[0]
store float %v1, p[1]
store float %v2, p[2]
store <%v3, %v4, %v5, %v6>, p[3]
store float %v7, p[7]
becomes one `store <8 x float>` at p[0].
Approach: treat the slice as a "store reduction". Put the original
insertelement chain and StoreInsts into UserIgnoreList and call
buildTree(LaneValues, UserIgnoreList) so SLP's standard pipeline
(reorder/transform/cost) runs on the lane-value subtree. Add the
sink-specific cost (one new store minus the replaced ones); if
profitable, vectorizeTree() materializes the lane vector, the helper
emits the new store at the first owner, propagates metadata, and
erases the originals. A memory guard rejects slices with reordered
owner stores or unrelated memory ops between them.
Fires only on truly mixed slices; pure-scalar and pure-build-vector
slices are unchanged. Updates the precommit lit test (#195531) for
cases that change.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 162 ++++++++++++++++--
.../X86/buildvector-store-chains.ll | 89 +++++++---
2 files changed, 214 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6b887baba199d..8c6e6dc648a07 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27473,6 +27473,137 @@ bool SLPVectorizerPass::vectorizeStores(
return A && (!B || A->getStride() < B->getStride());
});
+ // Mixed scalar/build-vector slice handling.
+ //
+ // Conceptually a "store reduction": SLP builds a tree on the per-lane
+ // values, treating the original IE chain and StoreInsts together as
+ // the reduction-operation chain via UserIgnoreList. After the tree's
+ // standard cost analysis we add the store-sink-specific cost (one vector
+ // store, minus the replaced scalar store costs); if profitable we let
+ // the standard tree emitter materialize the lane-value vector, then
+ // emit the new <VF x T> store and erase the originals here. The IE
+ // chains feeding any erased build-vector store become trivially dead
+ // and are cleaned up by the standard pass at the end.
+ auto VectorizeBuildVectorStoreSinkSlice =
+ [&](ArrayRef<StoreLane> Chain) -> std::optional<bool> {
+ SmallVector<Value *> LaneValues;
+ SmallVector<std::pair<StoreInst *, unsigned>> Owners;
+ StoreInst *LastOwner = nullptr;
+ for (const StoreLane &Lane : Chain) {
+ LaneValues.push_back(Lane.ScalarValue);
+ if (Lane.Store == LastOwner) {
+ Owners.back().second++;
+ } else {
+ Owners.emplace_back(Lane.Store, 1);
+ LastOwner = Lane.Store;
+ }
+ }
+ // Each build-vector owner must be fully covered by this slice;
+ // otherwise erasing it would lose data outside the slice.
+ for (auto &[SI, Count] : Owners) {
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> Insts;
+ if (collectBuildVector(SI->getValueOperand(), Elts, Insts) &&
+ Count != Elts.size())
+ return false;
+ }
+ // UserIgnoreList: original stores plus any IE chain feeding a build-
+ // vector store. SLP treats this exactly like a reduction tree: the
+ // lane values' uses inside listed instructions are not counted as
+ // external uses, since those instructions will be erased after the
+ // new vector store is emitted.
+ SmallDenseSet<Value *> Ignored;
+ for (auto &[SI, _] : Owners) {
+ Ignored.insert(SI);
+ SmallVector<Value *, 16> Elts;
+ SmallVector<Instruction *, 16> Insts;
+ if (collectBuildVector(SI->getValueOperand(), Elts, Insts))
+ for (Instruction *I : Insts)
+ Ignored.insert(I);
+ }
+ // Conservatively require owner stores to appear in the same order in the
+ // block as they appear in the lane range, and reject any intervening
+ // instruction that may read or write memory. The standard scalar store
+ // path gets memory-dependence checks through store-root scheduling; this
+ // manual store sink must not move a wider store across unknown memory.
+ StoreInst *PrevStore = nullptr;
+ for (auto &[SI, _] : Owners) {
+ if (PrevStore && !PrevStore->comesBefore(SI))
+ return false;
+ PrevStore = SI;
+ }
+ StoreInst *FirstStore = Owners.front().first;
+ StoreInst *LastStore = Owners.back().first;
+ for (Instruction &I : make_range(std::next(FirstStore->getIterator()),
+ LastStore->getIterator())) {
+ if (Ignored.contains(&I))
+ continue;
+ if (I.mayReadOrWriteMemory())
+ return false;
+ }
+ R.buildTree(LaneValues, Ignored);
+ if (R.isTreeTinyAndNotFullyVectorizable())
+ return false;
+ if (R.isProfitableToReorder()) {
+ R.reorderTopToBottom();
+ R.reorderBottomToTop();
+ }
+ R.transformNodes();
+ R.computeMinimumValueSizes();
+ InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+ R.buildExternalUses();
+ InstructionCost Cost = R.getTreeCost(TreeCost);
+ // Add the store-sink-specific cost: one new vector store at the
+ // lowest-lane address, with alignment computed from each owner's
+ // base alignment shifted by its lane offset, minus the original
+ // store costs being replaced.
+ Type *EltTy = LaneValues.front()->getType();
+ auto *VecTy = getWidenedType(EltTy, LaneValues.size());
+ TypeSize EltSize = DL->getTypeStoreSize(EltTy);
+ Align CommonAlign = FirstStore->getAlign();
+ int64_t LaneOffset = 0;
+ for (auto &[SI, Count] : Owners) {
+ Align A = commonAlignment(SI->getAlign(),
+ LaneOffset * EltSize.getFixedValue());
+ CommonAlign = std::min(CommonAlign, A);
+ LaneOffset += Count;
+ }
+ InstructionCost VecStoreCost = TTI->getMemoryOpCost(
+ Instruction::Store, VecTy, CommonAlign,
+ FirstStore->getPointerAddressSpace(), TTI::TCK_RecipThroughput);
+ InstructionCost ScalarStoreCost = 0;
+ for (auto &[SI, Count] : Owners) {
+ TTI::OperandValueInfo OpInfo =
+ TTI::getOperandInfo(SI->getValueOperand());
+ ScalarStoreCost += TTI->getMemoryOpCost(
+ Instruction::Store, SI->getValueOperand()->getType(),
+ SI->getAlign(), SI->getPointerAddressSpace(),
+ TTI::TCK_RecipThroughput, OpInfo, SI);
+ }
+ Cost += VecStoreCost - ScalarStoreCost;
+ if (Cost >= -SLPCostThreshold)
+ return false;
+ // Standard emitter materializes the lane-value vector.
+ Value *VecValue = R.vectorizeTree();
+ // Emit the new vector store at the first owner's location.
+ IRBuilder<> Builder(FirstStore);
+ if (VecValue->getType() != VecTy) {
+ bool IsSigned = any_of(LaneValues, [&](Value *V) {
+ return !isKnownNonNegative(V, SimplifyQuery(*DL));
+ });
+ VecValue = Builder.CreateIntCast(VecValue, VecTy, IsSigned);
+ }
+ StoreInst *NewSI = Builder.CreateAlignedStore(
+ VecValue, FirstStore->getPointerOperand(), CommonAlign);
+ SmallVector<Value *> OwnerValues;
+ for (auto &[SI, _] : Owners)
+ OwnerValues.push_back(SI);
+ (void)::propagateMetadata(NewSI, OwnerValues);
+ for (auto &[SI, _] : Owners)
+ R.eraseInstruction(SI);
+ return true;
+ };
+
for (unsigned LimitVF = GlobalMaxVF; LimitVF > 0;
LimitVF = bit_ceil(LimitVF) / 2) {
for (auto &CtxPtr : AllContexts) {
@@ -27484,16 +27615,27 @@ bool SLPVectorizerPass::vectorizeStores(
unsigned VF = *VFUnval;
if (!Context.vectorizeOneVF(
*TTI, VF, VectorizedStores, Changed,
- [this, &R](ArrayRef<StoreLane> Chain, unsigned Idx,
- unsigned MinVF,
- unsigned &Size) -> std::optional<bool> {
- // Mixed scalar/build-vector slices need a dedicated
- // TreeEntry path which is added by a follow-up commit.
- // For now decline them so the scalar-only path is
- // unchanged but keep the slice attempt counted.
- if (any_of(Chain, [](const StoreLane &Lane) {
- return Lane.IsVectorLane;
- })) {
+ [&](ArrayRef<StoreLane> Chain, unsigned Idx, unsigned MinVF,
+ unsigned &Size) -> std::optional<bool> {
+ bool HasVec = any_of(Chain, [](const StoreLane &Lane) {
+ return Lane.IsVectorLane;
+ });
+ bool HasScalar = any_of(Chain, [](const StoreLane &Lane) {
+ return !Lane.IsVectorLane;
+ });
+ if (HasVec && HasScalar) {
+ // Mixed scalar/build-vector slice: store-reduction sink.
+ Size = 1;
+ std::optional<bool> Res =
+ VectorizeBuildVectorStoreSinkSlice(Chain);
+ if (Res)
+ Size = R.getTreeSize();
+ return Res;
+ }
+ if (HasVec) {
+ // All-build-vector slice: nothing to merge with
+ // surrounding scalar stores; fall through and decline
+ // so the existing scalar-only path is unaffected.
Size = 1;
return false;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
index ad55b6dd445c3..35900a23107a4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
@@ -11,23 +11,16 @@ define void @buildvector_store_middle(ptr %p, float %a0, float %a1, float %a2, f
; CHECK-LABEL: define void @buildvector_store_middle(
; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[A1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[V2:%.*]] = fadd float [[A2]], 1.000000e+00
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[A3]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[A4]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[A5]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A6]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = fadd <4 x float> [[TMP6]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[V7:%.*]] = fadd float [[A7]], 1.000000e+00
-; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[P]], align 4
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
-; CHECK-NEXT: store float [[V2]], ptr [[P2]], align 4
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[P3]], align 4
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds float, ptr [[P]], i64 7
-; CHECK-NEXT: store float [[V7]], ptr [[P7]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x float> [[TMP0]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x float> [[TMP1]], float [[A2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x float> [[TMP2]], float [[A3]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x float> [[TMP3]], float [[A4]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x float> [[TMP4]], float [[A5]], i32 5
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x float> [[TMP5]], float [[A6]], i32 6
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x float> [[TMP6]], float [[A7]], i32 7
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <8 x float> [[TMP7]], splat (float 1.000000e+00)
+; CHECK-NEXT: store <8 x float> [[TMP8]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -64,14 +57,17 @@ define void @buildvector_store_start(ptr %p, float %a0, float %a1, float %a2, fl
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A2]], i32 2
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A3]], i32 3
; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP3]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[A4]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A5]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[A6]], i32 2
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A7]], i32 3
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[TMP8]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[A4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[A5]], i32 1
+; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x float> [[TMP6]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[A6]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> [[TMP8]], float [[A7]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[TMP9]], splat (float 1.000000e+00)
; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[P]], align 4
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds float, ptr [[P]], i64 4
-; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[P4]], align 4
+; CHECK-NEXT: store <2 x float> [[TMP7]], ptr [[P4]], align 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds float, ptr [[P]], i64 6
+; CHECK-NEXT: store <2 x float> [[TMP10]], ptr [[P6]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -147,9 +143,8 @@ define void @buildvector_store_duplicate_offset(ptr %p, float %a0, float %a1, fl
; CHECK-LABEL: define void @buildvector_store_duplicate_offset(
; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[A1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[V0:%.*]] = fadd float [[A0]], 1.000000e+00
+; CHECK-NEXT: [[V1:%.*]] = fadd float [[A1]], 1.000000e+00
; CHECK-NEXT: [[V2:%.*]] = fadd float [[A2]], 1.000000e+00
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[A3]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[A4]], i32 1
@@ -157,8 +152,9 @@ define void @buildvector_store_duplicate_offset(ptr %p, float %a0, float %a1, fl
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A6]], i32 3
; CHECK-NEXT: [[TMP7:%.*]] = fadd <4 x float> [[TMP6]], splat (float 1.000000e+00)
; CHECK-NEXT: [[V7:%.*]] = fadd float [[A7]], 1.000000e+00
+; CHECK-NEXT: store float [[V0]], ptr [[P]], align 4
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
-; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[P]], align 4
+; CHECK-NEXT: store float [[V1]], ptr [[P1]], align 4
; CHECK-NEXT: store float [[V2]], ptr [[P1]], align 4
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[P2]], align 4
@@ -189,3 +185,42 @@ entry:
store float %v7, ptr %p6, align 4
ret void
}
+
+define float @buildvector_store_intervening_load(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4) {
+; CHECK-LABEL: define float @buildvector_store_intervening_load(
+; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V0:%.*]] = fadd float [[A0]], 1.000000e+00
+; CHECK-NEXT: [[V1:%.*]] = fadd float [[A1]], 1.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[A2]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[A3]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[V4:%.*]] = fadd float [[A4]], 1.000000e+00
+; CHECK-NEXT: store float [[V0]], ptr [[P]], align 4
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
+; CHECK-NEXT: [[OLD:%.*]] = load float, ptr [[P1]], align 4
+; CHECK-NEXT: store float [[V1]], ptr [[P1]], align 4
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
+; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[P2]], align 4
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds float, ptr [[P]], i64 4
+; CHECK-NEXT: store float [[V4]], ptr [[P4]], align 4
+; CHECK-NEXT: ret float [[OLD]]
+;
+entry:
+ %v0 = fadd float %a0, 1.000000e+00
+ %v1 = fadd float %a1, 1.000000e+00
+ %v2 = fadd float %a2, 1.000000e+00
+ %v3 = fadd float %a3, 1.000000e+00
+ %v4 = fadd float %a4, 1.000000e+00
+ store float %v0, ptr %p, align 4
+ %p1 = getelementptr inbounds float, ptr %p, i64 1
+ %old = load float, ptr %p1, align 4
+ store float %v1, ptr %p1, align 4
+ %p2 = getelementptr inbounds float, ptr %p, i64 2
+ %b0 = insertelement <2 x float> poison, float %v2, i32 0
+ %b1 = insertelement <2 x float> %b0, float %v3, i32 1
+ store <2 x float> %b1, ptr %p2, align 4
+ %p4 = getelementptr inbounds float, ptr %p, i64 4
+ store float %v4, ptr %p4, align 4
+ ret float %old
+}
>From 9f68753405e613086122c8c6637ca4a8e5227229 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 7 Jul 2026 00:15:36 -0400
Subject: [PATCH 4/4] [SLP] Skip reduction bitwidth analysis for ignored stores
The build-vector store path uses UserIgnoreList for instructions that will be erased, including stores. The existing min-bitwidth code treated the list as reduction values and queried type sizes, which asserts for void-typed stores.
Only run the reduction bitwidth analysis when all ignored values have sized integer types. Add a regression test for mixed scalar/build-vector stores with alternating zext/sext.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 8 +++-
.../X86/buildvector-store-chains.ll | 46 +++++++++++++++++++
2 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8c6e6dc648a07..c633ad1ec217a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26334,7 +26334,13 @@ void BoUpSLP::computeMinimumValueSizes() {
// modify.
// Add reduction ops sizes, if any.
if (UserIgnoreList &&
- isa<IntegerType>(VectorizableTree.front()->Scalars.front()->getType())) {
+ isa<IntegerType>(VectorizableTree.front()->Scalars.front()->getType()) &&
+ all_of(*UserIgnoreList, [](Value *V) {
+ if (isa<PoisonValue>(V))
+ return true;
+ Type *Ty = V->getType();
+ return Ty->isSized() && isa<IntegerType>(Ty->getScalarType());
+ })) {
// Convert vector_reduce_add(ZExt(<n x i1>)) to ZExtOrTrunc(ctpop(bitcast <n
// x i1> to in)).
if (all_of(*UserIgnoreList,
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
index 35900a23107a4..17e561bfbb72a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-chains.ll
@@ -224,3 +224,49 @@ entry:
store float %v4, ptr %p4, align 4
ret float %old
}
+
+define i32 @mixed_ext(ptr %p, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7) {
+; CHECK-LABEL: define i32 @mixed_ext(
+; CHECK: ret i32
+entry:
+ %v0 = zext i8 %a0 to i32
+ %v1 = sext i8 %a1 to i32
+ %v2 = zext i8 %a2 to i32
+ %v3 = sext i8 %a3 to i32
+ %v4 = zext i8 %a4 to i32
+ %v5 = sext i8 %a5 to i32
+ %v6 = zext i8 %a6 to i32
+ %v7 = sext i8 %a7 to i32
+ store i32 %v0, ptr %p, align 4
+ %p1 = getelementptr inbounds i32, ptr %p, i64 1
+ store i32 %v1, ptr %p1, align 4
+ %p2 = getelementptr inbounds i32, ptr %p, i64 2
+ store i32 %v2, ptr %p2, align 4
+ %p3 = getelementptr inbounds i32, ptr %p, i64 3
+ %b0 = insertelement <4 x i32> poison, i32 %v3, i32 0
+ %b1 = insertelement <4 x i32> %b0, i32 %v4, i32 1
+ %b2 = insertelement <4 x i32> %b1, i32 %v5, i32 2
+ %b3 = insertelement <4 x i32> %b2, i32 %v6, i32 3
+ store <4 x i32> %b3, ptr %p3, align 4
+ %p7 = getelementptr inbounds i32, ptr %p, i64 7
+ store i32 %v7, ptr %p7, align 4
+ %l0 = load i32, ptr %p, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %s1 = add i32 %l0, %l1
+ %l2 = load i32, ptr %p2, align 4
+ %s2 = add i32 %s1, %l2
+ %l3 = load i32, ptr %p3, align 4
+ %s3 = add i32 %s2, %l3
+ %l4p = getelementptr inbounds i32, ptr %p, i64 4
+ %l4 = load i32, ptr %l4p, align 4
+ %s4 = add i32 %s3, %l4
+ %l5p = getelementptr inbounds i32, ptr %p, i64 5
+ %l5 = load i32, ptr %l5p, align 4
+ %s5 = add i32 %s4, %l5
+ %l6p = getelementptr inbounds i32, ptr %p, i64 6
+ %l6 = load i32, ptr %l6p, align 4
+ %s6 = add i32 %s5, %l6
+ %l7 = load i32, ptr %p7, align 4
+ %s7 = add i32 %s6, %l7
+ ret i32 %s7
+}
More information about the llvm-commits
mailing list