[llvm] [SLP] Initial support for masked stores (PR #204893)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 09:01:28 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/204893
>From 6f056467204a12495674df92ab5b2f22e53827bc Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 19 Jun 2026 13:52:44 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 3 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 264 +++++++++++++++++-
.../SLPVectorizer/X86/masked-stores.ll | 212 ++------------
3 files changed, 272 insertions(+), 207 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..23df37b606220 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -174,7 +174,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
- &Visited);
+ &Visited,
+ bool AllowMaskedStores = true);
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9b7d154598c40..c18ddf9a66a06 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -221,6 +221,11 @@ static cl::opt<bool>
cl::desc("Enable SLP trees to be built from strided "
"store chains."));
+static cl::opt<bool> EnableMaskedStores(
+ "slp-enable-masked-stores", cl::init(true), cl::Hidden,
+ cl::desc("Enable vectorization of non-consecutive stores as a single "
+ "masked store, when the target supports masked stores."));
+
static cl::opt<bool>
DisableTreeReorder("slp-disable-tree-reorder", cl::init(false), cl::Hidden,
cl::desc("Disable tree reordering even if it is "
@@ -4321,6 +4326,8 @@ class slpvectorizer::BoUpSLP {
Vectorize, ///< The node is regularly vectorized.
ScatterVectorize, ///< Masked scatter/gather node.
StridedVectorize, ///< Strided loads (and stores)
+ ExpandVectorize, ///< Masked stores, the values are expanded into
+ ///< a wider vector and vectorized with a mask.
CompressVectorize, ///< (Masked) load with compress.
NeedToGather, ///< Gather/buildvector node.
CombinedVectorize, ///< Vectorized node, combined with its user into more
@@ -4607,6 +4614,9 @@ class slpvectorizer::BoUpSLP {
case StridedVectorize:
dbgs() << "StridedVectorize\n";
break;
+ case ExpandVectorize:
+ dbgs() << "ExpandVectorize\n";
+ break;
case CompressVectorize:
dbgs() << "CompressVectorize\n";
break;
@@ -4915,7 +4925,8 @@ class slpvectorizer::BoUpSLP {
TreeEntry::EntryState getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo);
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo,
+ SmallVectorImpl<int> &ReuseShuffleIndices);
/// Maps a specific scalar to its tree entry(ies).
SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarToTreeEntries;
@@ -6663,6 +6674,7 @@ struct llvm::DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
return "color=red";
if (Entry->State == TreeEntry::ScatterVectorize ||
Entry->State == TreeEntry::StridedVectorize ||
+ Entry->State == TreeEntry::ExpandVectorize ||
Entry->State == TreeEntry::CompressVectorize)
return "color=blue";
return "";
@@ -7463,6 +7475,55 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
CompressMask, LoadVecTy);
}
+/// Checks if the stores \p VL with pointers \p PointerOps can be lowered as a
+/// single masked store. On success \p StoreVecTy is the widened store type and
+/// \p ReuseShuffleIndices is the expand mask that places each stored value at
+/// its element offset from the base (poison in the gaps).
+static bool isMaskedStoreCompress(
+ ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
+ ArrayRef<unsigned> Order, const TargetTransformInfo &TTI,
+ const DataLayout &DL, ScalarEvolution &SE, Align CommonAlignment,
+ SmallVectorImpl<int> &ReuseShuffleIndices, FixedVectorType *&StoreVecTy) {
+ Type *ScalarTy = cast<StoreInst>(VL.front())->getValueOperand()->getType();
+ const size_t Sz = VL.size();
+ // Only simple scalar element types are supported.
+ if (Sz < 2 || (!ScalarTy->isIntOrPtrTy() && !ScalarTy->isFloatingPointTy()))
+ return false;
+ Value *Ptr0 = Order.empty() ? PointerOps.front() : PointerOps[Order.front()];
+ Value *PtrN = Order.empty() ? PointerOps.back() : PointerOps[Order.back()];
+ std::optional<int64_t> Diff =
+ getPointersDiff(ScalarTy, Ptr0, ScalarTy, PtrN, DL, SE);
+ if (!Diff || *Diff <= 0)
+ return false;
+ // Avoid widened vectors with very large gaps between the stored elements.
+ const unsigned MaxRegSize =
+ TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+ .getFixedValue();
+ const unsigned ScalarBits = DL.getTypeSizeInBits(ScalarTy).getFixedValue();
+ if (ScalarBits == 0 ||
+ static_cast<uint64_t>(*Diff) / Sz >= MaxRegSize / ScalarBits)
+ return false;
+ StoreVecTy = cast<FixedVectorType>(getWidenedType(ScalarTy, *Diff + 1));
+ unsigned AS = cast<StoreInst>(VL.front())->getPointerAddressSpace();
+ if (!TTI.isLegalMaskedStore(StoreVecTy, CommonAlignment, AS,
+ TTI::ConstantMask))
+ return false;
+ // Build the expand mask: store I (in address-sorted order) is placed at its
+ // element offset from the base, other widened lanes are poison.
+ ReuseShuffleIndices.assign(*Diff + 1, PoisonMaskElem);
+ int64_t Prev = -1;
+ for (unsigned I : seq<unsigned>(Sz)) {
+ Value *Ptr = Order.empty() ? PointerOps[I] : PointerOps[Order[I]];
+ std::optional<int64_t> Off =
+ getPointersDiff(ScalarTy, Ptr0, ScalarTy, Ptr, DL, SE);
+ if (!Off || *Off <= Prev || *Off > *Diff)
+ return false;
+ ReuseShuffleIndices[*Off] = static_cast<int>(I);
+ Prev = *Off;
+ }
+ return true;
+}
+
/// Checks if strided loads can be generated out of \p VL loads with pointers \p
/// PointerOps:
/// 1. Target with strided load support is detected.
@@ -8470,6 +8531,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
if (TE.State == TreeEntry::SplitVectorize ||
((TE.State == TreeEntry::Vectorize ||
TE.State == TreeEntry::StridedVectorize ||
+ TE.State == TreeEntry::ExpandVectorize ||
TE.State == TreeEntry::CompressVectorize) &&
(isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
(TopToBottom && isa<StoreInst, InsertElementInst, InsertValueInst>(
@@ -8974,6 +9036,7 @@ void BoUpSLP::reorderTopToBottom() {
VFToOrderedEntries[TE->getVectorFactor()].insert(TE.get());
if (!(TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
+ TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::SplitVectorize ||
TE->State == TreeEntry::CompressVectorize) ||
!TE->ReuseShuffleIndices.empty())
@@ -9112,7 +9175,8 @@ void BoUpSLP::reorderTopToBottom() {
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
// Just do the reordering for the nodes with the given VF.
if (TE->Scalars.size() != VF) {
- if (TE->ReuseShuffleIndices.size() == VF) {
+ if (TE->ReuseShuffleIndices.size() == VF &&
+ TE->State != TreeEntry::ExpandVectorize) {
assert(TE->State != TreeEntry::SplitVectorize &&
"Split vectorized not expected.");
// Need to reorder the reuses masks of the operands with smaller VF to
@@ -9149,6 +9213,7 @@ void BoUpSLP::reorderTopToBottom() {
TE->ReuseShuffleIndices.empty()) ||
((TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
+ TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize) &&
(isa<ExtractElementInst, ExtractValueInst, LoadInst, StoreInst,
InsertElementInst, InsertValueInst>(TE->getMainOp()) ||
@@ -9170,9 +9235,12 @@ void BoUpSLP::reorderTopToBottom() {
"Expected empty reorder sequence.");
reorderScalars(TE->Scalars, Mask);
}
- if (!TE->ReuseShuffleIndices.empty()) {
+ if (!TE->ReuseShuffleIndices.empty() &&
+ TE->State != TreeEntry::ExpandVectorize) {
// Apply reversed order to keep the original ordering of the reused
- // elements to avoid extra reorder indices shuffling.
+ // elements to avoid extra reorder indices shuffling. An ExpandVectorize
+ // store keeps its expand mask fixed and carries the reorder in
+ // ReorderIndices, so it is excluded here.
OrdersType CurrentOrder;
reorderOrder(CurrentOrder, MaskOrder);
SmallVector<int> NewReuses;
@@ -9197,6 +9265,7 @@ void BoUpSLP::buildReorderableOperands(
return OpData.first == I &&
(OpData.second->State == TreeEntry::Vectorize ||
OpData.second->State == TreeEntry::StridedVectorize ||
+ OpData.second->State == TreeEntry::ExpandVectorize ||
OpData.second->State == TreeEntry::CompressVectorize ||
OpData.second->State == TreeEntry::SplitVectorize);
}))
@@ -9212,7 +9281,8 @@ void BoUpSLP::buildReorderableOperands(
continue;
if (UserTE->getOpcode() == Instruction::Store && I == 1 &&
(UserTE->State == TreeEntry::Vectorize ||
- UserTE->State == TreeEntry::StridedVectorize))
+ UserTE->State == TreeEntry::StridedVectorize ||
+ UserTE->State == TreeEntry::ExpandVectorize))
continue;
if (UserTE->getOpcode() == Instruction::Load &&
(UserTE->State == TreeEntry::Vectorize ||
@@ -9258,6 +9328,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (TE->State != TreeEntry::Vectorize &&
TE->State != TreeEntry::StridedVectorize &&
+ TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
TE->State != TreeEntry::SplitVectorize)
NonVectorized.insert(TE.get());
@@ -9266,6 +9337,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
Queue.push(TE.get());
if (!(TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
+ TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::SplitVectorize) ||
!TE->ReuseShuffleIndices.empty())
@@ -9295,6 +9367,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
for (TreeEntry *TE : OrderedOps) {
if (!(TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
+ TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::SplitVectorize ||
(TE->isGather() && GathersToOrders.contains(TE))) ||
@@ -9604,6 +9677,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
// Gathers are processed separately.
if (TE->State != TreeEntry::Vectorize &&
TE->State != TreeEntry::StridedVectorize &&
+ TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
TE->State != TreeEntry::SplitVectorize &&
(TE->State != TreeEntry::ScatterVectorize ||
@@ -10839,7 +10913,8 @@ static bool allStructUsersAreExtractValueInsts(ArrayRef<Value *> VL) {
BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo) {
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo,
+ SmallVectorImpl<int> &ReuseShuffleIndices) {
assert(S.getMainOp() &&
"Expected instructions with same/alternate opcodes only.");
@@ -11195,6 +11270,16 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
CurrentOrder, *Dist, Ptr0, SPtrInfo))
return TreeEntry::StridedVectorize;
+ // If the stores are not consecutive but the target supports masked stores
+ // for the widened type, lower them as a masked store.
+ FixedVectorType *StoreVecTy = nullptr;
+ if (EnableMaskedStores &&
+ isMaskedStoreCompress(VL, PointerOps, CurrentOrder, *TTI, *DL, *SE,
+ CommonAlignment, ReuseShuffleIndices,
+ StoreVecTy)) {
+ SPtrInfo.Ty = StoreVecTy;
+ return TreeEntry::ExpandVectorize;
+ }
}
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
@@ -12984,8 +13069,10 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
OrdersType CurrentOrder;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
+ SmallVector<int> ExpandShuffleMask;
TreeEntry::EntryState State = getScalarsVectorizationState(
- S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, SPtrInfo);
+ S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, SPtrInfo,
+ ExpandShuffleMask);
if (State == TreeEntry::NeedToGather) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
@@ -13269,6 +13356,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
<< "SLP: added a new TreeEntry (non-consecutive LoadInst).\n";
TE->dump());
break;
+ case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
case TreeEntry::NeedToGather:
@@ -13450,6 +13538,21 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
return;
}
+ if (State == TreeEntry::ExpandVectorize) {
+ assert(ReuseShuffleIndices.empty() &&
+ "Expected no reuse shuffle for an expanded masked store.");
+ TreeEntry *TE =
+ newTreeEntry(VL, TreeEntry::ExpandVectorize, Bundle, S, UserTreeIdx,
+ ExpandShuffleMask, CurrentOrder);
+ TreeEntryToStridedPtrInfoMap[TE] = SPtrInfo;
+ LLVM_DEBUG(
+ dbgs()
+ << "SLP: added a new TreeEntry (expanded masked StoreInst).\n";
+ TE->dump());
+ TE->setOperands(Operands);
+ buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
+ return;
+ }
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices, CurrentOrder);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (StoreInst).\n";
@@ -15320,6 +15423,8 @@ void BoUpSLP::transformNodes() {
break;
}
case Instruction::Store: {
+ if (E.State == TreeEntry::ExpandVectorize)
+ break;
Type *ScalarTy =
cast<StoreInst>(E.getMainOp())->getValueOperand()->getType();
auto *VecTy =
@@ -17000,6 +17105,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert((E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::ScatterVectorize ||
E->State == TreeEntry::StridedVectorize ||
+ E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize) &&
"Unhandled state");
assert(E->getOpcode() &&
@@ -17099,9 +17205,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto GetGEPCostDiff = [=](ArrayRef<Value *> Ptrs, Value *BasePtr) {
assert((E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
+ E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize) &&
- "Entry state expected to be Vectorize, StridedVectorize or "
- "MaskedLoadCompressVectorize here.");
+ "Entry state expected to be Vectorize, StridedVectorize, "
+ "ExpandVectorize or CompressVectorize here.");
InstructionCost ScalarCost = 0;
InstructionCost VecCost = 0;
std::tie(ScalarCost, VecCost) =
@@ -17885,6 +17992,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
CostKind);
break;
}
+ case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
case TreeEntry::NeedToGather:
@@ -17935,7 +18043,31 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
VecStCost +=
TTI->getCastInstrCost(Instruction::BitCast, VecTy, StridedStoreTy,
getCastContextHint(*E), CostKind);
-
+ } else if (E->State == TreeEntry::ExpandVectorize) {
+ const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
+ FixedVectorType *MaskedStoreTy = SPtrInfo.Ty;
+ assert(MaskedStoreTy && "Missing StridedPointerInfo for tree entry.");
+ Align CommonAlignment =
+ computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
+ // Masked store: the values are expanded into the widened vector and
+ // stored with a constant mask.
+ VecStCost = TTI->getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::masked_store, MaskedStoreTy,
+ CommonAlignment,
+ BaseSI->getPointerAddressSpace()),
+ CostKind);
+ const TreeEntry *ValOpTE = getOperandEntry(E, 0);
+ if (auto ValBWIt = MinBWs.find(ValOpTE);
+ ValBWIt != MinBWs.end() && ValOpTE->hasState() &&
+ UnaryInstruction::isCast(ValOpTE->getOpcode())) {
+ auto *SrcVecTy = getWidenedType(
+ IntegerType::get(ScalarTy->getContext(), ValBWIt->second.first),
+ VL.size());
+ if (SrcVecTy != VecTy)
+ VecStCost += TTI->getCastInstrCost(
+ ValBWIt->second.second ? Instruction::SExt : Instruction::ZExt,
+ VecTy, SrcVecTy, getCastContextHint(*ValOpTE), CostKind);
+ }
} else {
assert(E->State == TreeEntry::Vectorize &&
"Expected either strided or consecutive stores.");
@@ -18182,6 +18314,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
if (VectorizableTree.size() == 1 &&
(VectorizableTree[0]->State == TreeEntry::Vectorize ||
VectorizableTree[0]->State == TreeEntry::StridedVectorize ||
+ VectorizableTree[0]->State == TreeEntry::ExpandVectorize ||
VectorizableTree[0]->State == TreeEntry::CompressVectorize ||
(ForReduction &&
AreVectorizableGathers(VectorizableTree[0].get(),
@@ -18206,6 +18339,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
(VectorizableTree[1]->isGather() &&
VectorizableTree[0]->State != TreeEntry::ScatterVectorize &&
VectorizableTree[0]->State != TreeEntry::StridedVectorize &&
+ VectorizableTree[0]->State != TreeEntry::ExpandVectorize &&
VectorizableTree[0]->State != TreeEntry::CompressVectorize))
return false;
@@ -23240,7 +23374,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
ShuffleBuilder.addOrdered(V, {});
} else if (E->getOpcode() == Instruction::Store &&
(E->State == TreeEntry::Vectorize ||
- E->State == TreeEntry::StridedVectorize)) {
+ E->State == TreeEntry::StridedVectorize ||
+ E->State == TreeEntry::ExpandVectorize)) {
ArrayRef<int> Mask =
ArrayRef(reinterpret_cast<const int *>(E->ReorderIndices.begin()),
E->ReorderIndices.size());
@@ -24010,9 +24145,29 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Instruction *ST;
if (E->State == TreeEntry::Vectorize) {
ST = Builder.CreateAlignedStore(VecValue, Ptr, SI->getAlign());
+ } else if (E->State == TreeEntry::ExpandVectorize) {
+ const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
+ FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
+ assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
+ Align CommonAlignment = computeCommonAlignment<StoreInst>(E->Scalars);
+ ArrayRef<int> ExpandMask = E->ReuseShuffleIndices;
+ unsigned VecNumElts = getNumElements(StridedStoreTy);
+ assert(VecValue->getType() == StridedStoreTy &&
+ "FinalShuffle must expand the value to the masked store type.");
+ assert(ExpandMask.size() == VecNumElts &&
+ "Reuse mask must match the masked store type.");
+ Value *BasePtr =
+ cast<StoreInst>(E->Scalars.front())->getPointerOperand();
+ SmallVector<Constant *> MaskValues(
+ VecNumElts, ConstantInt::getFalse(VecTy->getContext()));
+ for (unsigned I : seq<unsigned>(VecNumElts))
+ if (ExpandMask[I] != PoisonMaskElem)
+ MaskValues[I] = ConstantInt::getTrue(VecTy->getContext());
+ ST = Builder.CreateMaskedStore(VecValue, BasePtr, CommonAlignment,
+ ConstantVector::get(MaskValues));
} else {
assert(E->State == TreeEntry::StridedVectorize &&
- "Expected either strided or consecutive stores.");
+ "Expected either strided, masked or consecutive stores.");
bool IsReverseOrder =
!E->ReorderIndices.empty() && isReverseOrder(E->ReorderIndices);
if (IsReverseOrder) {
@@ -24811,11 +24966,14 @@ Value *BoUpSLP::vectorizeTree(
return !UseEntries.empty() &&
(E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
+ E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize) &&
any_of(UseEntries, [&, TTI = TTI](TreeEntry *UseEntry) {
return (UseEntry->State == TreeEntry::Vectorize ||
UseEntry->State ==
TreeEntry::StridedVectorize ||
+ UseEntry->State ==
+ TreeEntry::ExpandVectorize ||
UseEntry->State ==
TreeEntry::CompressVectorize) &&
doesInTreeUserNeedToExtract(
@@ -28293,8 +28451,8 @@ class RelatedStoreInsts {
bool SLPVectorizerPass::vectorizeStores(
ArrayRef<StoreInst *> Stores, BoUpSLP &R,
- DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
- &Visited) {
+ DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>> &Visited,
+ bool AllowMaskedStores) {
// We may run into multiple chains that merge into a single chain. We mark the
// stores that we vectorized so that we don't visit the same store twice.
BoUpSLP::ValueSet VectorizedStores;
@@ -28498,6 +28656,55 @@ bool SLPVectorizerPass::vectorizeStores(
ExtendContexts(StoreSeq.getStores());
ActuallyVectorizeContexts();
+
+ // When some stores are still not vectorized, try to lower them together as a
+ // single masked store (expand + masked.store).
+ if (!AllowMaskedStores)
+ return Changed;
+
+ SmallVector<Value *> Remaining;
+ for (StoreInst *SI : Stores)
+ if (!R.isDeleted(SI) && !VectorizedStores.contains(SI))
+ Remaining.push_back(SI);
+ if (Remaining.size() < 2)
+ return Changed;
+
+ Type *ValTy = getValueType(Remaining.front());
+ if (any_of(Remaining, [&](Value *V) {
+ return cast<StoreInst>(V)->getValueOperand()->getType() != ValTy;
+ }))
+ return Changed;
+
+ SmallVector<Value *> PointerOps;
+ PointerOps.reserve(Remaining.size());
+ for (Value *V : Remaining)
+ PointerOps.push_back(cast<StoreInst>(V)->getPointerOperand());
+ SmallVector<unsigned> Order;
+ if (!sortPtrAccesses(PointerOps, ValTy, *DL, *SE, Order))
+ return Changed;
+
+ SmallVector<Value *> Sorted(Remaining.size());
+ for (unsigned I : seq<unsigned>(Remaining.size()))
+ Sorted[I] = Remaining[Order.empty() ? I : Order[I]];
+
+ // No need to re-check for consecutive (stride-1) pairs here: the caller
+ // already disables masked stores for the whole base-object group when any
+ // consecutive pair exists, so this subset cannot contain one.
+ if (!Visited
+ .insert({Sorted.front(),
+ cast<StoreInst>(Sorted.front())->getValueOperand(),
+ Sorted.back(),
+ cast<StoreInst>(Sorted.back())->getValueOperand(),
+ Sorted.size()})
+ .second)
+ return Changed;
+
+ unsigned Size = 0;
+ if (vectorizeStoreChain(Sorted, R, /*Idx=*/0, /*MinVF=*/2, Size)
+ .value_or(false)) {
+ VectorizedStores.insert_range(Sorted);
+ Changed = true;
+ }
return Changed;
}
@@ -32868,6 +33075,28 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
V2->getValueOperand()->getValueID();
};
+ // Returns true if any two stores in the group are at consecutive (stride-1)
+ // addresses. Masked stores are disabled for the entire base-object group when
+ // this is true - consecutive vectorization is always preferred.
+ auto HasConsecutiveStores = [&](ArrayRef<StoreInst *> SIs) {
+ if (SIs.size() < 2)
+ return false;
+ Type *Ty = SIs.front()->getValueOperand()->getType();
+ Value *Base = SIs.front()->getPointerOperand();
+ SmallVector<int64_t> Offsets;
+ Offsets.reserve(SIs.size());
+ for (StoreInst *SI : SIs) {
+ if (std::optional<int64_t> Off =
+ getPointersDiff(Ty, Base, Ty, SI->getPointerOperand(), *DL, *SE))
+ Offsets.push_back(*Off);
+ }
+ sort(Offsets);
+ for (size_t I = 1, E = Offsets.size(); I < E; ++I)
+ if (Offsets[I] - Offsets[I - 1] == 1)
+ return true;
+ return false;
+ };
+
// Attempt to sort and vectorize each of the store-groups.
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>> Attempted;
for (auto &Pair : Stores) {
@@ -32880,6 +33109,11 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
if (!isValidElementType(Pair.second.front()->getValueOperand()->getType()))
continue;
+ // Masked stores are only attempted when no consecutive pair exists in the
+ // full base-object group.
+ const bool AllowMaskedStores =
+ !EnableMaskedStores || !HasConsecutiveStores(Pair.second);
+
// Reverse stores to do bottom-to-top analysis. This is important if the
// values are stores to the same addresses several times, in this case need
// to follow the stores order (reversed to meet the memory dependecies).
@@ -32888,7 +33122,7 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
Changed |= tryToVectorizeSequence<StoreInst>(
ReversedStores, StoreSorter, AreCompatibleStores,
[&](ArrayRef<StoreInst *> Candidates, bool) {
- return vectorizeStores(Candidates, R, Attempted);
+ return vectorizeStores(Candidates, R, Attempted, AllowMaskedStores);
},
/*MaxVFOnly=*/false, R);
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/masked-stores.ll b/llvm/test/Transforms/SLPVectorizer/X86/masked-stores.ll
index 500a9c5673cac..b38453b35da51 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/masked-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/masked-stores.ll
@@ -5,30 +5,9 @@ define void @const_stride_2_i8(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @const_stride_2_i8(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i8>, ptr [[PL]], align 1
-; CHECK-NEXT: [[E0:%.*]] = extractelement <8 x i8> [[L]], i32 0
-; CHECK-NEXT: [[E1:%.*]] = extractelement <8 x i8> [[L]], i32 1
-; CHECK-NEXT: [[E2:%.*]] = extractelement <8 x i8> [[L]], i32 2
-; CHECK-NEXT: [[E3:%.*]] = extractelement <8 x i8> [[L]], i32 3
-; CHECK-NEXT: [[E4:%.*]] = extractelement <8 x i8> [[L]], i32 4
-; CHECK-NEXT: [[E5:%.*]] = extractelement <8 x i8> [[L]], i32 5
-; CHECK-NEXT: [[E6:%.*]] = extractelement <8 x i8> [[L]], i32 6
-; CHECK-NEXT: [[E7:%.*]] = extractelement <8 x i8> [[L]], i32 7
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 14
-; CHECK-NEXT: store i8 [[E0]], ptr [[S0]], align 1
-; CHECK-NEXT: store i8 [[E1]], ptr [[S1]], align 1
-; CHECK-NEXT: store i8 [[E2]], ptr [[S2]], align 1
-; CHECK-NEXT: store i8 [[E3]], ptr [[S3]], align 1
-; CHECK-NEXT: store i8 [[E4]], ptr [[S4]], align 1
-; CHECK-NEXT: store i8 [[E5]], ptr [[S5]], align 1
-; CHECK-NEXT: store i8 [[E6]], ptr [[S6]], align 1
-; CHECK-NEXT: store i8 [[E7]], ptr [[S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> [[L]], <8 x i8> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v15i8.p0(<15 x i8> [[TMP1]], ptr align 1 [[S0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i8>, ptr %pl, align 1
@@ -63,30 +42,9 @@ define void @const_stride_2_i32(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @const_stride_2_i32(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i32>, ptr [[PL]], align 4
-; CHECK-NEXT: [[E0:%.*]] = extractelement <8 x i32> [[L]], i32 0
-; CHECK-NEXT: [[E1:%.*]] = extractelement <8 x i32> [[L]], i32 1
-; CHECK-NEXT: [[E2:%.*]] = extractelement <8 x i32> [[L]], i32 2
-; CHECK-NEXT: [[E3:%.*]] = extractelement <8 x i32> [[L]], i32 3
-; CHECK-NEXT: [[E4:%.*]] = extractelement <8 x i32> [[L]], i32 4
-; CHECK-NEXT: [[E5:%.*]] = extractelement <8 x i32> [[L]], i32 5
-; CHECK-NEXT: [[E6:%.*]] = extractelement <8 x i32> [[L]], i32 6
-; CHECK-NEXT: [[E7:%.*]] = extractelement <8 x i32> [[L]], i32 7
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 14
-; CHECK-NEXT: store i32 [[E0]], ptr [[S0]], align 4
-; CHECK-NEXT: store i32 [[E1]], ptr [[S1]], align 4
-; CHECK-NEXT: store i32 [[E2]], ptr [[S2]], align 4
-; CHECK-NEXT: store i32 [[E3]], ptr [[S3]], align 4
-; CHECK-NEXT: store i32 [[E4]], ptr [[S4]], align 4
-; CHECK-NEXT: store i32 [[E5]], ptr [[S5]], align 4
-; CHECK-NEXT: store i32 [[E6]], ptr [[S6]], align 4
-; CHECK-NEXT: store i32 [[E7]], ptr [[S7]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[L]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[S0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i32>, ptr %pl, align 4
@@ -157,38 +115,10 @@ define void @const_stride_2_i32_zext_value(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @const_stride_2_i32_zext_value(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i8>, ptr [[PL]], align 1
-; CHECK-NEXT: [[E0:%.*]] = extractelement <8 x i8> [[L]], i32 0
-; CHECK-NEXT: [[E1:%.*]] = extractelement <8 x i8> [[L]], i32 1
-; CHECK-NEXT: [[E2:%.*]] = extractelement <8 x i8> [[L]], i32 2
-; CHECK-NEXT: [[E3:%.*]] = extractelement <8 x i8> [[L]], i32 3
-; CHECK-NEXT: [[E4:%.*]] = extractelement <8 x i8> [[L]], i32 4
-; CHECK-NEXT: [[E5:%.*]] = extractelement <8 x i8> [[L]], i32 5
-; CHECK-NEXT: [[E6:%.*]] = extractelement <8 x i8> [[L]], i32 6
-; CHECK-NEXT: [[E7:%.*]] = extractelement <8 x i8> [[L]], i32 7
-; CHECK-NEXT: [[Z0:%.*]] = zext i8 [[E0]] to i32
-; CHECK-NEXT: [[Z1:%.*]] = zext i8 [[E1]] to i32
-; CHECK-NEXT: [[Z2:%.*]] = zext i8 [[E2]] to i32
-; CHECK-NEXT: [[Z3:%.*]] = zext i8 [[E3]] to i32
-; CHECK-NEXT: [[Z4:%.*]] = zext i8 [[E4]] to i32
-; CHECK-NEXT: [[Z5:%.*]] = zext i8 [[E5]] to i32
-; CHECK-NEXT: [[Z6:%.*]] = zext i8 [[E6]] to i32
-; CHECK-NEXT: [[Z7:%.*]] = zext i8 [[E7]] to i32
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 14
-; CHECK-NEXT: store i32 [[Z0]], ptr [[S0]], align 4
-; CHECK-NEXT: store i32 [[Z1]], ptr [[S1]], align 4
-; CHECK-NEXT: store i32 [[Z2]], ptr [[S2]], align 4
-; CHECK-NEXT: store i32 [[Z3]], ptr [[S3]], align 4
-; CHECK-NEXT: store i32 [[Z4]], ptr [[S4]], align 4
-; CHECK-NEXT: store i32 [[Z5]], ptr [[S5]], align 4
-; CHECK-NEXT: store i32 [[Z6]], ptr [[S6]], align 4
-; CHECK-NEXT: store i32 [[Z7]], ptr [[S7]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <8 x i8> [[L]] to <8 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP2]], ptr align 4 [[S0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i8>, ptr %pl, align 1
@@ -231,40 +161,12 @@ define void @const_stride_2_i32_add_value(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @const_stride_2_i32_add_value(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i8>, ptr [[PL]], align 1
+; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = zext <8 x i8> [[L]] to <8 x i16>
; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i16> [[TMP1]], splat (i16 5)
-; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 14
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i16> [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[S0]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT: store i32 [[TMP6]], ptr [[S1]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP2]], i32 2
-; CHECK-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
-; CHECK-NEXT: store i32 [[TMP8]], ptr [[S2]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP10:%.*]] = zext i16 [[TMP9]] to i32
-; CHECK-NEXT: store i32 [[TMP10]], ptr [[S3]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP2]], i32 4
-; CHECK-NEXT: [[TMP12:%.*]] = zext i16 [[TMP11]] to i32
-; CHECK-NEXT: store i32 [[TMP12]], ptr [[S4]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i16> [[TMP2]], i32 5
-; CHECK-NEXT: [[TMP14:%.*]] = zext i16 [[TMP13]] to i32
-; CHECK-NEXT: store i32 [[TMP14]], ptr [[S5]], align 4
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i16> [[TMP2]], i32 6
-; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP15]] to i32
-; CHECK-NEXT: store i32 [[TMP16]], ptr [[S6]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i16> [[TMP2]], i32 7
-; CHECK-NEXT: [[TMP18:%.*]] = zext i16 [[TMP17]] to i32
-; CHECK-NEXT: store i32 [[TMP18]], ptr [[S7]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP4]], ptr align 4 [[S0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i8>, ptr %pl, align 1
@@ -315,31 +217,10 @@ define void @const_stride_2_i32_reverse_values(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @const_stride_2_i32_reverse_values(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i32>, ptr [[PL]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[L]], splat (i32 1)
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 14
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[S0]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
-; CHECK-NEXT: store i32 [[TMP3]], ptr [[S1]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[S2]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
-; CHECK-NEXT: store i32 [[TMP5]], ptr [[S3]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: store i32 [[TMP6]], ptr [[S4]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: store i32 [[TMP7]], ptr [[S5]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: store i32 [[TMP8]], ptr [[S6]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: store i32 [[TMP9]], ptr [[S7]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[L]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <15 x i32> <i32 7, i32 poison, i32 6, i32 poison, i32 5, i32 poison, i32 4, i32 poison, i32 3, i32 poison, i32 2, i32 poison, i32 1, i32 poison, i32 0>
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP2]], ptr align 4 [[S0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i32>, ptr %pl, align 4
@@ -382,30 +263,9 @@ define void @non_const_stride_i32(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @non_const_stride_i32(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i32>, ptr [[PL]], align 4
-; CHECK-NEXT: [[E0:%.*]] = extractelement <8 x i32> [[L]], i32 0
-; CHECK-NEXT: [[E1:%.*]] = extractelement <8 x i32> [[L]], i32 1
-; CHECK-NEXT: [[E2:%.*]] = extractelement <8 x i32> [[L]], i32 2
-; CHECK-NEXT: [[E3:%.*]] = extractelement <8 x i32> [[L]], i32 3
-; CHECK-NEXT: [[E4:%.*]] = extractelement <8 x i32> [[L]], i32 4
-; CHECK-NEXT: [[E5:%.*]] = extractelement <8 x i32> [[L]], i32 5
-; CHECK-NEXT: [[E6:%.*]] = extractelement <8 x i32> [[L]], i32 6
-; CHECK-NEXT: [[E7:%.*]] = extractelement <8 x i32> [[L]], i32 7
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 8
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 10
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 12
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 15
-; CHECK-NEXT: store i32 [[E0]], ptr [[S0]], align 4
-; CHECK-NEXT: store i32 [[E1]], ptr [[S1]], align 4
-; CHECK-NEXT: store i32 [[E2]], ptr [[S2]], align 4
-; CHECK-NEXT: store i32 [[E3]], ptr [[S3]], align 4
-; CHECK-NEXT: store i32 [[E4]], ptr [[S4]], align 4
-; CHECK-NEXT: store i32 [[E5]], ptr [[S5]], align 4
-; CHECK-NEXT: store i32 [[E6]], ptr [[S6]], align 4
-; CHECK-NEXT: store i32 [[E7]], ptr [[S7]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[L]], <8 x i32> poison, <16 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP1]], ptr align 4 [[S0]], <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i32>, ptr %pl, align 4
@@ -440,30 +300,9 @@ define void @irregular_gaps_i8(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @irregular_gaps_i8(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <8 x i8>, ptr [[PL]], align 1
-; CHECK-NEXT: [[E0:%.*]] = extractelement <8 x i8> [[L]], i32 0
-; CHECK-NEXT: [[E1:%.*]] = extractelement <8 x i8> [[L]], i32 1
-; CHECK-NEXT: [[E2:%.*]] = extractelement <8 x i8> [[L]], i32 2
-; CHECK-NEXT: [[E3:%.*]] = extractelement <8 x i8> [[L]], i32 3
-; CHECK-NEXT: [[E4:%.*]] = extractelement <8 x i8> [[L]], i32 4
-; CHECK-NEXT: [[E5:%.*]] = extractelement <8 x i8> [[L]], i32 5
-; CHECK-NEXT: [[E6:%.*]] = extractelement <8 x i8> [[L]], i32 6
-; CHECK-NEXT: [[E7:%.*]] = extractelement <8 x i8> [[L]], i32 7
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 5
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 9
-; CHECK-NEXT: [[S4:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 11
-; CHECK-NEXT: [[S5:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 14
-; CHECK-NEXT: [[S6:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 18
-; CHECK-NEXT: [[S7:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 20
-; CHECK-NEXT: store i8 [[E0]], ptr [[S0]], align 1
-; CHECK-NEXT: store i8 [[E1]], ptr [[S1]], align 1
-; CHECK-NEXT: store i8 [[E2]], ptr [[S2]], align 1
-; CHECK-NEXT: store i8 [[E3]], ptr [[S3]], align 1
-; CHECK-NEXT: store i8 [[E4]], ptr [[S4]], align 1
-; CHECK-NEXT: store i8 [[E5]], ptr [[S5]], align 1
-; CHECK-NEXT: store i8 [[E6]], ptr [[S6]], align 1
-; CHECK-NEXT: store i8 [[E7]], ptr [[S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> [[L]], <8 x i8> poison, <21 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v21i8.p0(<21 x i8> [[TMP1]], ptr align 1 [[S0]], <21 x i1> <i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <8 x i8>, ptr %pl, align 1
@@ -498,19 +337,10 @@ define void @reordered_values_i32(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @reordered_values_i32(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = load <4 x i32>, ptr [[PL]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[L]], splat (i32 1)
; CHECK-NEXT: [[S0:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 4
-; CHECK-NEXT: [[S3:%.*]] = getelementptr inbounds i32, ptr [[PS]], i64 6
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[S0]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: store i32 [[TMP3]], ptr [[S1]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[S2]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: store i32 [[TMP5]], ptr [[S3]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[L]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <7 x i32> <i32 1, i32 poison, i32 0, i32 poison, i32 3, i32 poison, i32 2>
+; CHECK-NEXT: call void @llvm.masked.store.v7i32.p0(<7 x i32> [[TMP2]], ptr align 4 [[S0]], <7 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%l = load <4 x i32>, ptr %pl, align 4
More information about the llvm-commits
mailing list