[llvm] [SLP] Split blocking build-vector stores in scalar chains (PR #194970)
Yaxun Liu via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 18:30:27 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/194970
>From e03c5ad9112b0b892511fb4806fb651b45738fe7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 29 Apr 2026 17:35:37 -0400
Subject: [PATCH 1/5] [SLP] Split blocking build-vector stores in scalar chains
An existing build-vector store can split an otherwise contiguous scalar store chain. For example, scalar stores to p[0..2], then a <4 x float> store to p[3..6], then a scalar store to p[7] can make SLP form a mixed <4 x float> plus <2 x float> / scalar-tail layout.
When the middle vector value is built lane-by-lane and scalar stores to the same object appear on both sides, split that store back into scalar stores before seed collection. This exposes the full store range to SLP so the normal cost model can rebuild a cleaner vector layout, such as two full <4 x float> stores.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 84 +++++++++++++++++++
.../buildvector-store-blocks-store-chain.ll | 60 +++++++++++++
2 files changed, 144 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cc5a206cacbd0..1eb8cc984a473 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27270,6 +27270,90 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
Stores.clear();
GEPs.clear();
+ SmallVector<StoreInst *> StoresToErase;
+ // Recognize a vector value built lane-by-lane with insertelement:
+ //
+ // %v0 = insertelement <4 x float> poison, float %a, i32 0
+ // %v1 = insertelement <4 x float> %v0, float %b, i32 1
+ //
+ // Such a value can be split back into the scalar lane values without changing
+ // semantics. Do not handle shuffles or partial vectors here; those need the
+ // normal SLP cost model.
+ auto CollectBuildVector = [](Value *V, SmallVectorImpl<Value *> &Elts) {
+ auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!VecTy)
+ return false;
+ Elts.assign(VecTy->getNumElements(), nullptr);
+ Value *Cur = V;
+ while (auto *IE = dyn_cast<InsertElementInst>(Cur)) {
+ auto *Idx = dyn_cast<ConstantInt>(IE->getOperand(2));
+ if (!Idx || Idx->getValue().uge(Elts.size()))
+ return false;
+ Elts[Idx->getZExtValue()] = IE->getOperand(1);
+ Cur = IE->getOperand(0);
+ }
+ if (!isa<PoisonValue, UndefValue>(Cur))
+ return false;
+ return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
+ };
+
+ for (Instruction &I : make_early_inc_range(*BB)) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI || !SI->isSimple())
+ continue;
+
+ SmallVector<Value *, 16> BuildVectorOpds;
+ if (!CollectBuildVector(SI->getValueOperand(), BuildVectorOpds))
+ continue;
+
+ Type *EltTy = BuildVectorOpds.front()->getType();
+ Value *UnderlyingObject = getUnderlyingObject(SI->getPointerOperand());
+ bool HasScalarBefore = false;
+ bool HasScalarAfter = false;
+ for (Instruction &J : *BB) {
+ auto *OtherSI = dyn_cast<StoreInst>(&J);
+ if (!OtherSI || OtherSI == SI || !OtherSI->isSimple() ||
+ OtherSI->getValueOperand()->getType() != EltTy ||
+ getUnderlyingObject(OtherSI->getPointerOperand()) != UnderlyingObject)
+ continue;
+
+ std::optional<int64_t> Diff =
+ getPointersDiff(EltTy, SI->getPointerOperand(), EltTy,
+ OtherSI->getPointerOperand(), *DL, *SE,
+ /*StrictCheck=*/true);
+ if (!Diff)
+ continue;
+ HasScalarBefore |= *Diff < 0;
+ HasScalarAfter |= *Diff >= static_cast<int64_t>(BuildVectorOpds.size());
+ }
+ if (!HasScalarBefore || !HasScalarAfter)
+ continue;
+
+ // An existing vector store can hide a better scalar store chain from SLP.
+ // For example:
+ //
+ // store float %a, ptr %p
+ // store <4 x float> %v, ptr %p+4
+ // store float %b, ptr %p+20
+ //
+ // SLP collects scalar stores as seeds, so the middle vector store blocks it
+ // from seeing one continuous store range. Split only when scalar stores to
+ // the same object appear on both sides; the later store-chain vectorizer can
+ // then decide the final profitable vector layout.
+ IRBuilder<> Builder(SI);
+ uint64_t EltSize = DL->getTypeStoreSize(EltTy).getFixedValue();
+ Type *I8Ty = Type::getInt8Ty(SI->getContext());
+ for (auto [Idx, V] : enumerate(BuildVectorOpds)) {
+ Value *Ptr = Builder.CreateConstGEP1_64(I8Ty, SI->getPointerOperand(),
+ Idx * EltSize);
+ Builder.CreateAlignedStore(
+ V, Ptr, commonAlignment(SI->getAlign(), Idx * EltSize));
+ }
+ StoresToErase.push_back(SI);
+ }
+ for (StoreInst *SI : StoresToErase)
+ SI->eraseFromParent();
+
// Visit the store and getelementptr instructions in BB and organize them in
// Stores and GEPs according to the underlying objects of their pointer
// operands.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
new file mode 100644
index 0000000000000..262a799f6ef23
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; The input has a scalar store chain with an explicit build-vector store in the
+; middle:
+;
+; p[0] = v0; p[1] = v1; p[2] = v2;
+; store <v3, v4, v5, v6> to &p[3];
+; p[7] = v7;
+;
+; Splitting the middle build-vector store lets SLP see one continuous chain and
+; rebuild it as two full <4 x float> stores instead of a mixed <4 x float> plus
+; <2 x float> / scalar tail shape.
+
+define void @buildvector_store_blocks_store_chain(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
+; CHECK-LABEL: define void @buildvector_store_blocks_store_chain(
+; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A3]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP3]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[A4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A5]], i32 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[A6]], i32 2
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A7]], i32 3
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[TMP8]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> [[TMP9]], <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[P]], align 4
+; CHECK-NEXT: [[P4:%.*]] = getelementptr i8, ptr [[P3]], i64 4
+; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[P4]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v0 = fadd float %a0, 1.000000e+00
+ %v1 = fadd float %a1, 1.000000e+00
+ %v2 = fadd float %a2, 1.000000e+00
+ %v3 = fadd float %a3, 1.000000e+00
+ %v4 = fadd float %a4, 1.000000e+00
+ %v5 = fadd float %a5, 1.000000e+00
+ %v6 = fadd float %a6, 1.000000e+00
+ %v7 = fadd float %a7, 1.000000e+00
+ store float %v0, ptr %p, align 4
+ %p1 = getelementptr inbounds float, ptr %p, i64 1
+ store float %v1, ptr %p1, align 4
+ %p2 = getelementptr inbounds float, ptr %p, i64 2
+ store float %v2, ptr %p2, align 4
+ %p3 = getelementptr inbounds float, ptr %p, i64 3
+ %b0 = insertelement <4 x float> poison, float %v3, i32 0
+ %b1 = insertelement <4 x float> %b0, float %v4, i32 1
+ %b2 = insertelement <4 x float> %b1, float %v5, i32 2
+ %b3 = insertelement <4 x float> %b2, float %v6, i32 3
+ store <4 x float> %b3, ptr %p3, align 4
+ %p7 = getelementptr inbounds float, ptr %p, i64 7
+ store float %v7, ptr %p7, align 4
+ ret void
+}
>From 9f00a5b0911494b94319d8d5e3a801b65b82a211 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 29 Apr 2026 18:50:03 -0400
Subject: [PATCH 2/5] [SLP] Fix comment formatting
Wrap the new SLP comment to satisfy clang-format.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1eb8cc984a473..08ac66ed5280c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27338,8 +27338,8 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
//
// SLP collects scalar stores as seeds, so the middle vector store blocks it
// from seeing one continuous store range. Split only when scalar stores to
- // the same object appear on both sides; the later store-chain vectorizer can
- // then decide the final profitable vector layout.
+ // the same object appear on both sides; the later store-chain vectorizer
+ // can then decide the final profitable vector layout.
IRBuilder<> Builder(SI);
uint64_t EltSize = DL->getTypeStoreSize(EltTy).getFixedValue();
Type *I8Ty = Type::getInt8Ty(SI->getContext());
>From ec05db1efd46a60b7f673a964c87fcf2077f4fe8 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 29 Apr 2026 21:28:04 -0400
Subject: [PATCH 3/5] [SLP] Model blocking build-vector stores without scalar
IR
Address review feedback by retiling the build-vector store inside SLP instead of first materializing intermediate scalar stores.
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 4 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 233 ++++++++++++------
.../buildvector-store-blocks-store-chain.ll | 11 +-
3 files changed, 173 insertions(+), 75 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 877c83291170b..dbbe6638ad5d4 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -108,6 +108,10 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
/// Vectorize the store instructions collected in Stores.
bool vectorizeStoreChains(slpvectorizer::BoUpSLP &R);
+ /// Try to retile explicit build-vector stores that block scalar store chains.
+ bool vectorizeBuildVectorStoreChains(BasicBlock *BB,
+ slpvectorizer::BoUpSLP &R);
+
/// Vectorize the index computations of the getelementptr instructions
/// collected in GEPs.
bool vectorizeGEPIndices(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 08ac66ed5280c..fd9a3e9e169ec 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26364,6 +26364,8 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()))
continue;
+ Changed |= vectorizeBuildVectorStoreChains(BB, R);
+
// Start new block - clear the list of reduction roots.
R.clearReductionData();
collectSeedInstructions(BB);
@@ -27265,94 +27267,187 @@ bool SLPVectorizerPass::vectorizeStores(
return Changed;
}
-void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
- // Initialize the collections. We will make a single pass over the block.
- Stores.clear();
- GEPs.clear();
-
- SmallVector<StoreInst *> StoresToErase;
- // Recognize a vector value built lane-by-lane with insertelement:
- //
- // %v0 = insertelement <4 x float> poison, float %a, i32 0
- // %v1 = insertelement <4 x float> %v0, float %b, i32 1
- //
- // Such a value can be split back into the scalar lane values without changing
- // semantics. Do not handle shuffles or partial vectors here; those need the
- // normal SLP cost model.
- auto CollectBuildVector = [](Value *V, SmallVectorImpl<Value *> &Elts) {
- auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
- if (!VecTy)
- return false;
- Elts.assign(VecTy->getNumElements(), nullptr);
- Value *Cur = V;
- while (auto *IE = dyn_cast<InsertElementInst>(Cur)) {
- auto *Idx = dyn_cast<ConstantInt>(IE->getOperand(2));
- if (!Idx || Idx->getValue().uge(Elts.size()))
- return false;
- Elts[Idx->getZExtValue()] = IE->getOperand(1);
- Cur = IE->getOperand(0);
- }
- if (!isa<PoisonValue, UndefValue>(Cur))
+static bool collectBuildVector(Value *V, SmallVectorImpl<Value *> &Elts,
+ SmallVectorImpl<Instruction *> *Insts = nullptr) {
+ auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!VecTy)
+ return false;
+ Elts.assign(VecTy->getNumElements(), nullptr);
+ Value *Cur = V;
+ while (auto *IE = dyn_cast<InsertElementInst>(Cur)) {
+ auto *Idx = dyn_cast<ConstantInt>(IE->getOperand(2));
+ if (!Idx || Idx->getValue().uge(Elts.size()))
return false;
- return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
- };
+ Elts[Idx->getZExtValue()] = IE->getOperand(1);
+ if (Insts)
+ Insts->push_back(IE);
+ Cur = IE->getOperand(0);
+ }
+ if (!isa<PoisonValue, UndefValue>(Cur))
+ return false;
+ return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
+}
+/// Retile a scalar-store / build-vector-store / scalar-store sequence without
+/// first materializing scalar stores in the IR. The build-vector store is
+/// treated as covering its lane range, and BoUpSLP decides whether each final
+/// full-width store group is profitable.
+bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
+ BoUpSLP &R) {
+ bool Changed = false;
for (Instruction &I : make_early_inc_range(*BB)) {
- auto *SI = dyn_cast<StoreInst>(&I);
- if (!SI || !SI->isSimple())
+ auto *VecStore = dyn_cast<StoreInst>(&I);
+ if (!VecStore || !VecStore->isSimple() || R.isDeleted(VecStore))
+ continue;
+
+ SmallVector<Value *, 16> VecElts;
+ SmallVector<Instruction *, 16> BuildVectorInsts;
+ if (!collectBuildVector(VecStore->getValueOperand(), VecElts,
+ &BuildVectorInsts))
continue;
- SmallVector<Value *, 16> BuildVectorOpds;
- if (!CollectBuildVector(SI->getValueOperand(), BuildVectorOpds))
+ Type *EltTy = VecElts.front()->getType();
+ TypeSize EltSize = DL->getTypeStoreSize(EltTy);
+ if (EltSize.isScalable())
continue;
- Type *EltTy = BuildVectorOpds.front()->getType();
- Value *UnderlyingObject = getUnderlyingObject(SI->getPointerOperand());
- bool HasScalarBefore = false;
- bool HasScalarAfter = false;
+ unsigned VF = VecElts.size();
+ Value *UnderlyingObject = getUnderlyingObject(VecStore->getPointerOperand());
+ MapVector<int64_t, std::pair<Value *, StoreInst *>> Lanes;
+ for (auto [Idx, V] : enumerate(VecElts))
+ Lanes[Idx] = {V, VecStore};
+
for (Instruction &J : *BB) {
- auto *OtherSI = dyn_cast<StoreInst>(&J);
- if (!OtherSI || OtherSI == SI || !OtherSI->isSimple() ||
- OtherSI->getValueOperand()->getType() != EltTy ||
- getUnderlyingObject(OtherSI->getPointerOperand()) != UnderlyingObject)
+ auto *SI = dyn_cast<StoreInst>(&J);
+ if (!SI || SI == VecStore || !SI->isSimple() || R.isDeleted(SI) ||
+ SI->getValueOperand()->getType() != EltTy ||
+ getUnderlyingObject(SI->getPointerOperand()) != UnderlyingObject)
continue;
std::optional<int64_t> Diff =
- getPointersDiff(EltTy, SI->getPointerOperand(), EltTy,
- OtherSI->getPointerOperand(), *DL, *SE,
+ getPointersDiff(EltTy, VecStore->getPointerOperand(), EltTy,
+ SI->getPointerOperand(), *DL, *SE,
/*StrictCheck=*/true);
- if (!Diff)
+ if (!Diff || (*Diff >= 0 && *Diff < static_cast<int64_t>(VF)))
continue;
- HasScalarBefore |= *Diff < 0;
- HasScalarAfter |= *Diff >= static_cast<int64_t>(BuildVectorOpds.size());
+ Lanes[*Diff] = {SI->getValueOperand(), SI};
+ }
+
+ int64_t MinOffset = Lanes.front().first;
+ int64_t MaxOffset = Lanes.front().first;
+ for (auto &Lane : Lanes) {
+ MinOffset = std::min(MinOffset, Lane.first);
+ MaxOffset = std::max(MaxOffset, Lane.first);
}
- if (!HasScalarBefore || !HasScalarAfter)
+ if (Lanes.size() <= VF || MinOffset >= 0 ||
+ MaxOffset < static_cast<int64_t>(VF))
+ continue;
+ if (static_cast<unsigned>(MaxOffset - MinOffset + 1) != Lanes.size() ||
+ Lanes.size() % VF != 0)
continue;
- // An existing vector store can hide a better scalar store chain from SLP.
- // For example:
- //
- // store float %a, ptr %p
- // store <4 x float> %v, ptr %p+4
- // store float %b, ptr %p+20
- //
- // SLP collects scalar stores as seeds, so the middle vector store blocks it
- // from seeing one continuous store range. Split only when scalar stores to
- // the same object appear on both sides; the later store-chain vectorizer
- // can then decide the final profitable vector layout.
- IRBuilder<> Builder(SI);
- uint64_t EltSize = DL->getTypeStoreSize(EltTy).getFixedValue();
- Type *I8Ty = Type::getInt8Ty(SI->getContext());
- for (auto [Idx, V] : enumerate(BuildVectorOpds)) {
- Value *Ptr = Builder.CreateConstGEP1_64(I8Ty, SI->getPointerOperand(),
- Idx * EltSize);
+ SmallVector<Instruction *> StoresToReplace;
+ for (auto &Lane : Lanes)
+ StoresToReplace.push_back(Lane.second.second);
+ sort(StoresToReplace);
+ StoresToReplace.erase(llvm::unique(StoresToReplace), StoresToReplace.end());
+
+ Instruction *FirstStore = StoresToReplace.front();
+ Instruction *LastStore = StoresToReplace.front();
+ for (Instruction *SI : StoresToReplace) {
+ if (SI->comesBefore(FirstStore))
+ FirstStore = SI;
+ if (LastStore->comesBefore(SI))
+ LastStore = SI;
+ }
+
+ bool SawFirst = false;
+ bool SafeToMove = true;
+ for (Instruction &J : *BB) {
+ if (&J == FirstStore)
+ SawFirst = true;
+ if (!SawFirst)
+ continue;
+ if (J.mayReadOrWriteMemory() && !is_contained(StoresToReplace, &J)) {
+ SafeToMove = false;
+ break;
+ }
+ if (&J == LastStore)
+ break;
+ }
+ if (!SafeToMove)
+ continue;
+
+ SmallDenseSet<Value *> UserIgnoreList;
+ for (Instruction *SI : StoresToReplace)
+ UserIgnoreList.insert(SI);
+ for (Instruction *I : BuildVectorInsts)
+ UserIgnoreList.insert(I);
+
+ SmallVector<SmallVector<Value *, 4>, 4> Groups;
+ SmallVector<int64_t, 4> GroupOffsets;
+ for (int64_t Start = MinOffset; Start <= MaxOffset; Start += VF) {
+ Groups.emplace_back();
+ GroupOffsets.push_back(Start);
+ for (int64_t Offset = Start; Offset < Start + static_cast<int64_t>(VF);
+ ++Offset)
+ Groups.back().push_back(Lanes[Offset].first);
+ }
+
+ bool AllProfitable = true;
+ for (ArrayRef<Value *> Group : Groups) {
+ R.buildTree(Group, UserIgnoreList);
+ if (R.isTreeTinyAndNotFullyVectorizable()) {
+ AllProfitable = false;
+ break;
+ }
+ if (R.isProfitableToReorder()) {
+ R.reorderTopToBottom();
+ R.reorderBottomToTop();
+ }
+ R.transformNodes();
+ R.computeMinimumValueSizes();
+ InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+ R.buildExternalUses();
+ InstructionCost Cost = R.getTreeCost(TreeCost);
+ if (Cost >= -SLPCostThreshold) {
+ AllProfitable = false;
+ break;
+ }
+ }
+ if (!AllProfitable)
+ continue;
+
+ IRBuilder<> Builder(VecStore);
+ Type *I8Ty = Type::getInt8Ty(VecStore->getContext());
+ for (auto [Idx, Group] : enumerate(Groups)) {
+ R.buildTree(Group, UserIgnoreList);
+ Value *Vec = R.vectorizeTree();
+ Builder.SetInsertPoint(VecStore);
+ Value *Ptr = Builder.CreateConstGEP1_64(
+ I8Ty, VecStore->getPointerOperand(),
+ GroupOffsets[Idx] * EltSize.getFixedValue());
Builder.CreateAlignedStore(
- V, Ptr, commonAlignment(SI->getAlign(), Idx * EltSize));
+ Vec, Ptr,
+ commonAlignment(VecStore->getAlign(),
+ std::abs(GroupOffsets[Idx]) *
+ EltSize.getFixedValue()));
}
- StoresToErase.push_back(SI);
+
+ for (Instruction *SI : StoresToReplace)
+ R.eraseInstruction(SI);
+ for (Instruction *I : BuildVectorInsts)
+ if (isInstructionTriviallyDead(I))
+ R.eraseInstruction(I);
+ Changed = true;
}
- for (StoreInst *SI : StoresToErase)
- SI->eraseFromParent();
+ return Changed;
+}
+
+void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
+ // Initialize the collections. We will make a single pass over the block.
+ Stores.clear();
+ GEPs.clear();
// Visit the store and getelementptr instructions in BB and organize them in
// Stores and GEPs according to the underlying objects of their pointer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
index 262a799f6ef23..e14bfeac6e04b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
@@ -8,9 +8,9 @@
; store <v3, v4, v5, v6> to &p[3];
; p[7] = v7;
;
-; Splitting the middle build-vector store lets SLP see one continuous chain and
-; rebuild it as two full <4 x float> stores instead of a mixed <4 x float> plus
-; <2 x float> / scalar tail shape.
+; Modeling the middle build-vector store as part of the surrounding store range
+; lets SLP rebuild the chain as two full <4 x float> stores instead of a mixed
+; <4 x float> plus <2 x float> / scalar tail shape.
define void @buildvector_store_blocks_store_chain(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
; CHECK-LABEL: define void @buildvector_store_blocks_store_chain(
@@ -27,9 +27,8 @@ define void @buildvector_store_blocks_store_chain(ptr %p, float %a0, float %a1,
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A7]], i32 3
; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[TMP8]], splat (float 1.000000e+00)
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> [[TMP9]], <4 x i32> <i32 0, i32 4, i32 5, i32 6>
-; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[P]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[P3]], i64 -12
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[TMP10]], align 4
; CHECK-NEXT: [[P4:%.*]] = getelementptr i8, ptr [[P3]], i64 4
; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[P4]], align 4
; CHECK-NEXT: ret void
>From 1c02d55d9c52ebf21b691553373dc15e74f3971c Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 30 Apr 2026 14:31:33 -0400
Subject: [PATCH 4/5] [SLP] Address build-vector store review comments
Remove an unnecessary optional parameter from the build-vector collector and use the existing UndefValue check for poison or undef bases.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fd9a3e9e169ec..cb10544ed92f3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27268,7 +27268,7 @@ bool SLPVectorizerPass::vectorizeStores(
}
static bool collectBuildVector(Value *V, SmallVectorImpl<Value *> &Elts,
- SmallVectorImpl<Instruction *> *Insts = nullptr) {
+ SmallVectorImpl<Instruction *> &Insts) {
auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
if (!VecTy)
return false;
@@ -27279,11 +27279,10 @@ static bool collectBuildVector(Value *V, SmallVectorImpl<Value *> &Elts,
if (!Idx || Idx->getValue().uge(Elts.size()))
return false;
Elts[Idx->getZExtValue()] = IE->getOperand(1);
- if (Insts)
- Insts->push_back(IE);
+ Insts.push_back(IE);
Cur = IE->getOperand(0);
}
- if (!isa<PoisonValue, UndefValue>(Cur))
+ if (!isa<UndefValue>(Cur))
return false;
return all_of(Elts, [](Value *Elt) { return Elt != nullptr; });
}
@@ -27303,7 +27302,7 @@ bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
SmallVector<Value *, 16> VecElts;
SmallVector<Instruction *, 16> BuildVectorInsts;
if (!collectBuildVector(VecStore->getValueOperand(), VecElts,
- &BuildVectorInsts))
+ BuildVectorInsts))
continue;
Type *EltTy = VecElts.front()->getType();
>From 661863a5c13b6117e0cf72ab056bef34476830ff Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 30 Apr 2026 21:29:57 -0400
Subject: [PATCH 5/5] [SLP] Handle build-vector store chain edge cases
Model duplicate store offsets conservatively and cover build-vector stores at the start or end of the scalar store chain.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 34 ++++-
.../buildvector-store-blocks-store-chain.ll | 138 ++++++++++++++++++
2 files changed, 164 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cb10544ed92f3..fa0147bdd72e1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27311,11 +27311,13 @@ bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
continue;
unsigned VF = VecElts.size();
- Value *UnderlyingObject = getUnderlyingObject(VecStore->getPointerOperand());
+ Value *UnderlyingObject =
+ getUnderlyingObject(VecStore->getPointerOperand());
MapVector<int64_t, std::pair<Value *, StoreInst *>> Lanes;
for (auto [Idx, V] : enumerate(VecElts))
Lanes[Idx] = {V, VecStore};
+ bool HasDuplicateOffset = false;
for (Instruction &J : *BB) {
auto *SI = dyn_cast<StoreInst>(&J);
if (!SI || SI == VecStore || !SI->isSimple() || R.isDeleted(SI) ||
@@ -27329,8 +27331,14 @@ bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
/*StrictCheck=*/true);
if (!Diff || (*Diff >= 0 && *Diff < static_cast<int64_t>(VF)))
continue;
+ if (Lanes.contains(*Diff)) {
+ HasDuplicateOffset = true;
+ break;
+ }
Lanes[*Diff] = {SI->getValueOperand(), SI};
}
+ if (HasDuplicateOffset)
+ continue;
int64_t MinOffset = Lanes.front().first;
int64_t MaxOffset = Lanes.front().first;
@@ -27338,8 +27346,7 @@ bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
MinOffset = std::min(MinOffset, Lane.first);
MaxOffset = std::max(MaxOffset, Lane.first);
}
- if (Lanes.size() <= VF || MinOffset >= 0 ||
- MaxOffset < static_cast<int64_t>(VF))
+ if (Lanes.size() <= VF)
continue;
if (static_cast<unsigned>(MaxOffset - MinOffset + 1) != Lanes.size() ||
Lanes.size() % VF != 0)
@@ -27419,18 +27426,29 @@ bool SLPVectorizerPass::vectorizeBuildVectorStoreChains(BasicBlock *BB,
IRBuilder<> Builder(VecStore);
Type *I8Ty = Type::getInt8Ty(VecStore->getContext());
+ auto GetLaneAlign = [&](int64_t LaneOffset) {
+ StoreInst *SI = Lanes[LaneOffset].second;
+ int64_t StoreOffset = SI == VecStore ? 0 : LaneOffset;
+ return commonAlignment(SI->getAlign(),
+ std::abs(StoreOffset - LaneOffset) *
+ EltSize.getFixedValue());
+ };
for (auto [Idx, Group] : enumerate(Groups)) {
R.buildTree(Group, UserIgnoreList);
Value *Vec = R.vectorizeTree();
Builder.SetInsertPoint(VecStore);
+ Align StoreAlign = GetLaneAlign(GroupOffsets[Idx]);
+ for (int64_t Lane = GroupOffsets[Idx] + 1;
+ Lane < GroupOffsets[Idx] + static_cast<int64_t>(VF); ++Lane) {
+ Align LaneAlignAtGroupBase =
+ commonAlignment(GetLaneAlign(Lane), (Lane - GroupOffsets[Idx]) *
+ EltSize.getFixedValue());
+ StoreAlign = std::min(StoreAlign, LaneAlignAtGroupBase);
+ }
Value *Ptr = Builder.CreateConstGEP1_64(
I8Ty, VecStore->getPointerOperand(),
GroupOffsets[Idx] * EltSize.getFixedValue());
- Builder.CreateAlignedStore(
- Vec, Ptr,
- commonAlignment(VecStore->getAlign(),
- std::abs(GroupOffsets[Idx]) *
- EltSize.getFixedValue()));
+ Builder.CreateAlignedStore(Vec, Ptr, StoreAlign);
}
for (Instruction *SI : StoresToReplace)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
index e14bfeac6e04b..45079c32439c3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-store-blocks-store-chain.ll
@@ -57,3 +57,141 @@ entry:
store float %v7, ptr %p7, align 4
ret void
}
+
+define void @buildvector_store_starts_store_chain(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
+; CHECK-LABEL: define void @buildvector_store_starts_store_chain(
+; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A3]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP3]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[A4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A5]], i32 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[A6]], i32 2
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A7]], i32 3
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[TMP8]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 0
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[TMP10]], align 4
+; CHECK-NEXT: [[P4:%.*]] = getelementptr i8, ptr [[P]], i64 16
+; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[P4]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v0 = fadd float %a0, 1.000000e+00
+ %v1 = fadd float %a1, 1.000000e+00
+ %v2 = fadd float %a2, 1.000000e+00
+ %v3 = fadd float %a3, 1.000000e+00
+ %v4 = fadd float %a4, 1.000000e+00
+ %v5 = fadd float %a5, 1.000000e+00
+ %v6 = fadd float %a6, 1.000000e+00
+ %v7 = fadd float %a7, 1.000000e+00
+ %b0 = insertelement <4 x float> poison, float %v0, i32 0
+ %b1 = insertelement <4 x float> %b0, float %v1, i32 1
+ %b2 = insertelement <4 x float> %b1, float %v2, i32 2
+ %b3 = insertelement <4 x float> %b2, float %v3, i32 3
+ store <4 x float> %b3, ptr %p, align 4
+ %p4 = getelementptr inbounds float, ptr %p, i64 4
+ store float %v4, ptr %p4, align 4
+ %p5 = getelementptr inbounds float, ptr %p, i64 5
+ store float %v5, ptr %p5, align 4
+ %p6 = getelementptr inbounds float, ptr %p, i64 6
+ store float %v6, ptr %p6, align 4
+ %p7 = getelementptr inbounds float, ptr %p, i64 7
+ store float %v7, ptr %p7, align 4
+ ret void
+}
+
+define void @buildvector_store_ends_store_chain(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
+; CHECK-LABEL: define void @buildvector_store_ends_store_chain(
+; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A3]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP3]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[A4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A5]], i32 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[A6]], i32 2
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A7]], i32 3
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[TMP8]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds float, ptr [[P]], i64 4
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[P4]], i64 -16
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[P4]], i64 0
+; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[TMP11]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v0 = fadd float %a0, 1.000000e+00
+ %v1 = fadd float %a1, 1.000000e+00
+ %v2 = fadd float %a2, 1.000000e+00
+ %v3 = fadd float %a3, 1.000000e+00
+ %v4 = fadd float %a4, 1.000000e+00
+ %v5 = fadd float %a5, 1.000000e+00
+ %v6 = fadd float %a6, 1.000000e+00
+ %v7 = fadd float %a7, 1.000000e+00
+ store float %v0, ptr %p, align 4
+ %p1 = getelementptr inbounds float, ptr %p, i64 1
+ store float %v1, ptr %p1, align 4
+ %p2 = getelementptr inbounds float, ptr %p, i64 2
+ store float %v2, ptr %p2, align 4
+ %p3 = getelementptr inbounds float, ptr %p, i64 3
+ store float %v3, ptr %p3, align 4
+ %p4 = getelementptr inbounds float, ptr %p, i64 4
+ %b0 = insertelement <4 x float> poison, float %v4, i32 0
+ %b1 = insertelement <4 x float> %b0, float %v5, i32 1
+ %b2 = insertelement <4 x float> %b1, float %v6, i32 2
+ %b3 = insertelement <4 x float> %b2, float %v7, i32 3
+ store <4 x float> %b3, ptr %p4, align 4
+ ret void
+}
+
+define void @buildvector_store_duplicate_offset(ptr %p, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
+; CHECK-LABEL: define void @buildvector_store_duplicate_offset(
+; CHECK-SAME: ptr [[P:%.*]], float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]], float [[A5:%.*]], float [[A6:%.*]], float [[A7:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[V2:%.*]] = fadd float [[A2]], 1.000000e+00
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[A3]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[A4]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[A5]], i32 2
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[A6]], i32 3
+; CHECK-NEXT: [[TMP7:%.*]] = fadd <4 x float> [[TMP6]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[V7:%.*]] = fadd float [[A7]], 1.000000e+00
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
+; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[P]], align 4
+; CHECK-NEXT: store float [[V2]], ptr [[P1]], align 4
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
+; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[P2]], align 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds float, ptr [[P]], i64 6
+; CHECK-NEXT: store float [[V7]], ptr [[P6]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v0 = fadd float %a0, 1.000000e+00
+ %v1 = fadd float %a1, 1.000000e+00
+ %v2 = fadd float %a2, 1.000000e+00
+ %v3 = fadd float %a3, 1.000000e+00
+ %v4 = fadd float %a4, 1.000000e+00
+ %v5 = fadd float %a5, 1.000000e+00
+ %v6 = fadd float %a6, 1.000000e+00
+ %v7 = fadd float %a7, 1.000000e+00
+ store float %v0, ptr %p, align 4
+ %p1 = getelementptr inbounds float, ptr %p, i64 1
+ store float %v1, ptr %p1, align 4
+ store float %v2, ptr %p1, align 4
+ %p2 = getelementptr inbounds float, ptr %p, i64 2
+ %b0 = insertelement <4 x float> poison, float %v3, i32 0
+ %b1 = insertelement <4 x float> %b0, float %v4, i32 1
+ %b2 = insertelement <4 x float> %b1, float %v5, i32 2
+ %b3 = insertelement <4 x float> %b2, float %v6, i32 3
+ store <4 x float> %b3, ptr %p2, align 4
+ %p6 = getelementptr inbounds float, ptr %p, i64 6
+ store float %v7, ptr %p6, align 4
+ ret void
+}
More information about the llvm-commits
mailing list