[llvm] [SLP][modularisation][NFC] Extract mask, shuffle and AA helpers into SLUtils 3/3 (PR #209973)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 22:58:48 PDT 2026
https://github.com/madhur13490 created https://github.com/llvm/llvm-project/pull/209973
Move the mask/shuffle construction and alias-analysis free helpers out of SLPVectorizer.cpp into SLPVectorizer/SLPUtils.{h,cpp} (namespace llvm::slpvectorizer).
Moved:
transformScalarShuffleIndiciesToVector
getShufflevectorNumGroups
calculateShufflevectorMask
UseMask (enum)
buildUseMask
isUndefVector
doesInTreeUserNeedToExtract
getLocation
isSimple
addMask
fixupOrderingIndices
getAltInstrMask
replicateMask
RFC: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
>From b19351cba9452f2043126f0b4b863708c808fc84 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Wed, 15 Jul 2026 22:45:42 -0700
Subject: [PATCH] [SLP][modularisation][NFC] Extract mask, shuffle and AA
helpers into SLPUtils (3/3)
Patch 3 of 3. Move the mask/shuffle construction and alias-analysis free
helpers out of SLPVectorizer.cpp into SLPVectorizer/SLPUtils.{h,cpp}
(namespace llvm::slpvectorizer).
Moved:
transformScalarShuffleIndiciesToVector
getShufflevectorNumGroups
calculateShufflevectorMask
UseMask (enum)
buildUseMask
isUndefVector
doesInTreeUserNeedToExtract
getLocation
isSimple
addMask
fixupOrderingIndices
getAltInstrMask
replicateMask
RFC: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 351 +-----------------
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 259 +++++++++++++
.../Vectorize/SLPVectorizer/SLPUtils.h | 112 ++++++
3 files changed, 380 insertions(+), 342 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b76cb80426456..d0bf68d70a6bc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -421,200 +421,6 @@ getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
return (Sz / RegVF) * RegVF;
}
-static void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements,
- SmallVectorImpl<int> &Mask) {
- // The ShuffleBuilder implementation use shufflevector to splat an "element".
- // But the element have different meaning for SLP (scalar) and REVEC
- // (vector). We need to expand Mask into masks which shufflevector can use
- // directly.
- SmallVector<int> NewMask(Mask.size() * VecTyNumElements);
- for (unsigned I : seq<unsigned>(Mask.size()))
- for (auto [J, MaskV] : enumerate(MutableArrayRef(NewMask).slice(
- I * VecTyNumElements, VecTyNumElements)))
- MaskV = Mask[I] == PoisonMaskElem ? PoisonMaskElem
- : Mask[I] * VecTyNumElements + J;
- Mask.swap(NewMask);
-}
-
-/// \returns the number of groups of shufflevector
-/// A group has the following features
-/// 1. All of value in a group are shufflevector.
-/// 2. The mask of all shufflevector is isExtractSubvectorMask.
-/// 3. The mask of all shufflevector uses all of the elements of the source.
-/// e.g., it is 1 group (%0)
-/// %1 = shufflevector <16 x i8> %0, <16 x i8> poison,
-/// <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-/// %2 = shufflevector <16 x i8> %0, <16 x i8> poison,
-/// <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-/// it is 2 groups (%3 and %4)
-/// %5 = shufflevector <8 x i16> %3, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-/// %6 = shufflevector <8 x i16> %3, <8 x i16> poison,
-/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-/// %7 = shufflevector <8 x i16> %4, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-/// %8 = shufflevector <8 x i16> %4, <8 x i16> poison,
-/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-/// it is 0 group
-/// %12 = shufflevector <8 x i16> %10, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-/// %13 = shufflevector <8 x i16> %11, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-static unsigned getShufflevectorNumGroups(ArrayRef<Value *> VL) {
- if (VL.empty())
- return 0;
- if (!all_of(VL, IsaPred<ShuffleVectorInst>))
- return 0;
- auto *SV = cast<ShuffleVectorInst>(VL.front());
- unsigned SVNumElements =
- cast<FixedVectorType>(SV->getOperand(0)->getType())->getNumElements();
- unsigned ShuffleMaskSize = SV->getShuffleMask().size();
- if (SVNumElements % ShuffleMaskSize != 0)
- return 0;
- unsigned GroupSize = SVNumElements / ShuffleMaskSize;
- if (GroupSize == 0 || (VL.size() % GroupSize) != 0)
- return 0;
- unsigned NumGroup = 0;
- for (size_t I = 0, E = VL.size(); I != E; I += GroupSize) {
- auto *SV = cast<ShuffleVectorInst>(VL[I]);
- Value *Src = SV->getOperand(0);
- ArrayRef<Value *> Group = VL.slice(I, GroupSize);
- SmallBitVector ExpectedIndex(GroupSize);
- if (!all_of(Group, [&](Value *V) {
- auto *SV = cast<ShuffleVectorInst>(V);
- // From the same source.
- if (SV->getOperand(0) != Src)
- return false;
- int Index;
- if (!SV->isExtractSubvectorMask(Index))
- return false;
- ExpectedIndex.set(Index / ShuffleMaskSize);
- return true;
- }))
- return 0;
- if (!ExpectedIndex.all())
- return 0;
- ++NumGroup;
- }
- assert(NumGroup == (VL.size() / GroupSize) && "Unexpected number of groups");
- return NumGroup;
-}
-
-/// \returns a shufflevector mask which is used to vectorize shufflevectors
-/// e.g.,
-/// %5 = shufflevector <8 x i16> %3, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-/// %6 = shufflevector <8 x i16> %3, <8 x i16> poison,
-/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-/// %7 = shufflevector <8 x i16> %4, <8 x i16> poison,
-/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-/// %8 = shufflevector <8 x i16> %4, <8 x i16> poison,
-/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-/// the result is
-/// <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>
-static SmallVector<int> calculateShufflevectorMask(ArrayRef<Value *> VL) {
- assert(getShufflevectorNumGroups(VL) && "Not supported shufflevector usage.");
- auto *SV = cast<ShuffleVectorInst>(VL.front());
- unsigned SVNumElements =
- cast<FixedVectorType>(SV->getOperand(0)->getType())->getNumElements();
- SmallVector<int> Mask;
- unsigned AccumulateLength = 0;
- for (Value *V : VL) {
- auto *SV = cast<ShuffleVectorInst>(V);
- for (int M : SV->getShuffleMask())
- Mask.push_back(M == PoisonMaskElem ? PoisonMaskElem
- : AccumulateLength + M);
- AccumulateLength += SVNumElements;
- }
- return Mask;
-}
-
-namespace {
-/// Specifies the way the mask should be analyzed for undefs/poisonous elements
-/// in the shuffle mask.
-enum class UseMask {
- FirstArg, ///< The mask is expected to be for permutation of 1-2 vectors,
- ///< check for the mask elements for the first argument (mask
- ///< indices are in range [0:VF)).
- SecondArg, ///< The mask is expected to be for permutation of 2 vectors, check
- ///< for the mask elements for the second argument (mask indices
- ///< are in range [VF:2*VF))
- UndefsAsMask ///< Consider undef mask elements (-1) as placeholders for
- ///< future shuffle elements and mark them as ones as being used
- ///< in future. Non-undef elements are considered as unused since
- ///< they're already marked as used in the mask.
-};
-} // namespace
-
-/// Prepares a use bitset for the given mask either for the first argument or
-/// for the second.
-static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask,
- UseMask MaskArg) {
- SmallBitVector UseMask(VF, true);
- for (auto [Idx, Value] : enumerate(Mask)) {
- if (Value == PoisonMaskElem) {
- if (MaskArg == UseMask::UndefsAsMask)
- UseMask.reset(Idx);
- continue;
- }
- if (MaskArg == UseMask::FirstArg && Value < VF)
- UseMask.reset(Value);
- else if (MaskArg == UseMask::SecondArg && Value >= VF)
- UseMask.reset(Value - VF);
- }
- return UseMask;
-}
-
-/// Checks if the given value is actually an undefined constant vector.
-/// Also, if the \p UseMask is not empty, tries to check if the non-masked
-/// elements actually mask the insertelement buildvector, if any.
-template <bool IsPoisonOnly = false>
-static SmallBitVector isUndefVector(const Value *V,
- const SmallBitVector &UseMask = {}) {
- SmallBitVector Res(UseMask.empty() ? 1 : UseMask.size(), true);
- using T = std::conditional_t<IsPoisonOnly, PoisonValue, UndefValue>;
- if (isa<T>(V))
- return Res;
- auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
- if (!VecTy)
- return Res.reset();
- auto *C = dyn_cast<Constant>(V);
- if (!C) {
- if (!UseMask.empty()) {
- const Value *Base = V;
- while (auto *II = dyn_cast<InsertElementInst>(Base)) {
- Base = II->getOperand(0);
- if (isa<T>(II->getOperand(1)))
- continue;
- std::optional<unsigned> Idx = getElementIndex(II);
- if (!Idx) {
- Res.reset();
- return Res;
- }
- if (*Idx < UseMask.size() && !UseMask.test(*Idx))
- Res.reset(*Idx);
- }
- // TODO: Add analysis for shuffles here too.
- if (V == Base) {
- Res.reset();
- } else {
- SmallBitVector SubMask(UseMask.size(), false);
- Res &= isUndefVector<IsPoisonOnly>(Base, SubMask);
- }
- } else {
- Res.reset();
- }
- return Res;
- }
- for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I) {
- if (Constant *Elem = C->getAggregateElement(I))
- if (!isa<T>(Elem) &&
- (UseMask.empty() || (I < UseMask.size() && !UseMask.test(I))))
- Res.reset(I);
- }
- return Res;
-}
-
/// Checks if the vector of instructions can be represented as a shuffle, like:
/// %x0 = extractelement <4 x i8> %x, i32 0
/// %x3 = extractelement <4 x i8> %x, i32 3
@@ -1591,145 +1397,6 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
return S;
}
-/// \returns True if in-tree use also needs extract. This refers to
-/// possible scalar operand in vectorized instruction.
-static bool doesInTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
- TargetLibraryInfo *TLI,
- const TargetTransformInfo *TTI) {
- if (!UserInst)
- return false;
- unsigned Opcode = UserInst->getOpcode();
- switch (Opcode) {
- case Instruction::Load: {
- LoadInst *LI = cast<LoadInst>(UserInst);
- return (LI->getPointerOperand() == Scalar);
- }
- case Instruction::Store: {
- StoreInst *SI = cast<StoreInst>(UserInst);
- return (SI->getPointerOperand() == Scalar);
- }
- case Instruction::Call: {
- CallInst *CI = cast<CallInst>(UserInst);
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
- return any_of(enumerate(CI->args()), [&](auto &&Arg) {
- return isVectorIntrinsicWithScalarOpAtArg(ID, Arg.index(), TTI) &&
- Arg.value().get() == Scalar;
- });
- }
- default:
- return false;
- }
-}
-
-/// \returns the AA location that is being access by the instruction.
-static MemoryLocation getLocation(Instruction *I) {
- if (StoreInst *SI = dyn_cast<StoreInst>(I))
- return MemoryLocation::get(SI);
- if (LoadInst *LI = dyn_cast<LoadInst>(I))
- return MemoryLocation::get(LI);
- return MemoryLocation();
-}
-
-/// \returns True if the instruction is not a volatile or atomic load/store.
-static bool isSimple(Instruction *I) {
- if (LoadInst *LI = dyn_cast<LoadInst>(I))
- return LI->isSimple();
- if (StoreInst *SI = dyn_cast<StoreInst>(I))
- return SI->isSimple();
- if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
- return !MI->isVolatile();
- return true;
-}
-
-/// Shuffles \p Mask in accordance with the given \p SubMask.
-/// \param ExtendingManyInputs Supports reshuffling of the mask with not only
-/// one but two input vectors.
-static void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
- bool ExtendingManyInputs = false) {
- if (SubMask.empty())
- return;
- assert(
- (!ExtendingManyInputs || SubMask.size() > Mask.size() ||
- // Check if input scalars were extended to match the size of other node.
- (SubMask.size() == Mask.size() && Mask.back() == PoisonMaskElem)) &&
- "SubMask with many inputs support must be larger than the mask.");
- if (Mask.empty()) {
- Mask.append(SubMask.begin(), SubMask.end());
- return;
- }
- SmallVector<int> NewMask(SubMask.size(), PoisonMaskElem);
- int TermValue = std::min(Mask.size(), SubMask.size());
- for (int I = 0, E = SubMask.size(); I < E; ++I) {
- if (SubMask[I] == PoisonMaskElem ||
- (!ExtendingManyInputs &&
- (SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue)))
- continue;
- NewMask[I] = Mask[SubMask[I]];
- }
- Mask.swap(NewMask);
-}
-
-/// Order may have elements assigned special value (size) which is out of
-/// bounds. Such indices only appear on places which correspond to undef values
-/// (see canReuseExtract for details) and used in order to avoid undef values
-/// have effect on operands ordering.
-/// The first loop below simply finds all unused indices and then the next loop
-/// nest assigns these indices for undef values positions.
-/// As an example below Order has two undef positions and they have assigned
-/// values 3 and 7 respectively:
-/// before: 6 9 5 4 9 2 1 0
-/// after: 6 3 5 4 7 2 1 0
-static void fixupOrderingIndices(MutableArrayRef<unsigned> Order) {
- const size_t Sz = Order.size();
- SmallBitVector UnusedIndices(Sz, /*t=*/true);
- SmallBitVector MaskedIndices(Sz);
- for (unsigned I = 0; I < Sz; ++I) {
- if (Order[I] < Sz)
- UnusedIndices.reset(Order[I]);
- else
- MaskedIndices.set(I);
- }
- if (MaskedIndices.none())
- return;
- assert(UnusedIndices.count() == MaskedIndices.count() &&
- "Non-synced masked/available indices.");
- int Idx = UnusedIndices.find_first();
- int MIdx = MaskedIndices.find_first();
- while (MIdx >= 0) {
- assert(Idx >= 0 && "Indices must be synced.");
- Order[MIdx] = Idx;
- Idx = UnusedIndices.find_next(Idx);
- MIdx = MaskedIndices.find_next(MIdx);
- }
-}
-
-/// \returns a bitset for selecting opcodes. false for Opcode0 and true for
-/// Opcode1.
-static SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, Type *ScalarTy,
- unsigned Opcode0, unsigned Opcode1) {
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- SmallBitVector OpcodeMask(VL.size() * ScalarTyNumElements, false);
- for (unsigned Lane : seq<unsigned>(VL.size())) {
- if (isa<PoisonValue>(VL[Lane]))
- continue;
- if (cast<Instruction>(VL[Lane])->getOpcode() == Opcode1)
- OpcodeMask.set(Lane * ScalarTyNumElements,
- Lane * ScalarTyNumElements + ScalarTyNumElements);
- }
- return OpcodeMask;
-}
-
-/// Replicates the given \p Val \p VF times.
-static SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val,
- unsigned VF) {
- assert(none_of(Val, [](Constant *C) { return C->getType()->isVectorTy(); }) &&
- "Expected scalar constants.");
- SmallVector<Constant *> NewVal(Val.size() * VF);
- for (auto [I, V] : enumerate(Val))
- std::fill_n(NewVal.begin() + I * VF, VF, V);
- return NewVal;
-}
-
/// Returns true if widened type of \p Ty elements with size \p Sz represents
/// full vector type, i.e. adding extra element results in extra parts upon type
/// legalization.
@@ -3994,7 +3661,7 @@ class slpvectorizer::BoUpSLP {
return {};
SmallVector<int> Mask;
inversePermutation(ReorderIndices, Mask);
- ::addMask(Mask, ReuseShuffleIndices);
+ addMask(Mask, ReuseShuffleIndices);
return Mask;
}
@@ -4047,7 +3714,7 @@ class slpvectorizer::BoUpSLP {
if (VL.size() == Scalars.size())
return IsSame(Scalars, Mask);
if (VL.size() == ReuseShuffleIndices.size()) {
- ::addMask(Mask, ReuseShuffleIndices);
+ addMask(Mask, ReuseShuffleIndices);
return IsSame(Scalars, Mask);
}
return false;
@@ -8269,7 +7936,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
SmallVector<int> Mask;
fixupOrderingIndices(*CurrentOrder);
inversePermutation(*CurrentOrder, Mask);
- ::addMask(Mask, TE.ReuseShuffleIndices);
+ addMask(Mask, TE.ReuseShuffleIndices);
OrdersType Res(TE.getVectorFactor(), TE.getVectorFactor());
unsigned Sz = TE.Scalars.size();
for (int K = 0, E = TE.getVectorFactor() / Sz; K < E; ++K) {
@@ -8295,7 +7962,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
std::iota(ReorderMask.begin(), ReorderMask.end(), 0);
else
inversePermutation(TE.ReorderIndices, ReorderMask);
- ::addMask(ReorderMask, TE.ReuseShuffleIndices);
+ addMask(ReorderMask, TE.ReuseShuffleIndices);
unsigned VF = ReorderMask.size();
OrdersType ResOrder(VF, VF);
unsigned NumParts = divideCeil(VF, Sz);
@@ -16156,7 +15823,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
[](int Idx) { return Idx != PoisonMaskElem && Idx != 0; })) {
SmallVector<int> ReorderMask;
inversePermutation(E->ReorderIndices, ReorderMask);
- ::addMask(CommonMask, ReorderMask);
+ addMask(CommonMask, ReorderMask);
}
} else if (V1 && P2.isNull()) {
// Shuffle single vector.
@@ -17085,10 +16752,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
} else {
inversePermutation(E->ReorderIndices, NewMask);
}
- ::addMask(Mask, NewMask);
+ addMask(Mask, NewMask);
}
if (!E->ReuseShuffleIndices.empty())
- ::addMask(Mask, E->ReuseShuffleIndices);
+ addMask(Mask, E->ReuseShuffleIndices);
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
assert(!isa<StructType>(FinalVecTy) &&
"Expected non-struct vector type for shuffle cost calculation.");
@@ -19601,10 +19268,10 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
} else {
inversePermutation(TE->ReorderIndices, NewMask);
}
- ::addMask(Mask, NewMask);
+ addMask(Mask, NewMask);
}
if (!TE->ReuseShuffleIndices.empty())
- ::addMask(Mask, TE->ReuseShuffleIndices);
+ addMask(Mask, TE->ReuseShuffleIndices);
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, EntryVF))
GatherCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(VecTy), Mask);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 3eef3c52c0b7d..19d4833acddf1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -9,6 +9,7 @@
#include "SLPUtils.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Constants.h"
@@ -20,6 +21,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
#include <string>
#include <type_traits>
@@ -348,4 +350,261 @@ bool doesNotNeedToSchedule(ArrayRef<Value *> VL) {
(all_of(VL, isUsedOutsideBlock) || all_of(VL, areAllOperandsNonInsts));
}
+void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements,
+ SmallVectorImpl<int> &Mask) {
+ // The ShuffleBuilder implementation use shufflevector to splat an "element".
+ // But the element have different meaning for SLP (scalar) and REVEC
+ // (vector). We need to expand Mask into masks which shufflevector can use
+ // directly.
+ SmallVector<int> NewMask(Mask.size() * VecTyNumElements);
+ for (unsigned I : seq<unsigned>(Mask.size()))
+ for (auto [J, MaskV] : enumerate(MutableArrayRef(NewMask).slice(
+ I * VecTyNumElements, VecTyNumElements)))
+ MaskV = Mask[I] == PoisonMaskElem ? PoisonMaskElem
+ : Mask[I] * VecTyNumElements + J;
+ Mask.swap(NewMask);
+}
+
+unsigned getShufflevectorNumGroups(ArrayRef<Value *> VL) {
+ if (VL.empty())
+ return 0;
+ if (!all_of(VL, IsaPred<ShuffleVectorInst>))
+ return 0;
+ auto *SV = cast<ShuffleVectorInst>(VL.front());
+ unsigned SVNumElements =
+ cast<FixedVectorType>(SV->getOperand(0)->getType())->getNumElements();
+ unsigned ShuffleMaskSize = SV->getShuffleMask().size();
+ if (SVNumElements % ShuffleMaskSize != 0)
+ return 0;
+ unsigned GroupSize = SVNumElements / ShuffleMaskSize;
+ if (GroupSize == 0 || (VL.size() % GroupSize) != 0)
+ return 0;
+ unsigned NumGroup = 0;
+ for (size_t I = 0, E = VL.size(); I != E; I += GroupSize) {
+ auto *SV = cast<ShuffleVectorInst>(VL[I]);
+ Value *Src = SV->getOperand(0);
+ ArrayRef<Value *> Group = VL.slice(I, GroupSize);
+ SmallBitVector ExpectedIndex(GroupSize);
+ if (!all_of(Group, [&](Value *V) {
+ auto *SV = cast<ShuffleVectorInst>(V);
+ // From the same source.
+ if (SV->getOperand(0) != Src)
+ return false;
+ int Index;
+ if (!SV->isExtractSubvectorMask(Index))
+ return false;
+ ExpectedIndex.set(Index / ShuffleMaskSize);
+ return true;
+ }))
+ return 0;
+ if (!ExpectedIndex.all())
+ return 0;
+ ++NumGroup;
+ }
+ assert(NumGroup == (VL.size() / GroupSize) && "Unexpected number of groups");
+ return NumGroup;
+}
+
+SmallVector<int> calculateShufflevectorMask(ArrayRef<Value *> VL) {
+ assert(getShufflevectorNumGroups(VL) && "Not supported shufflevector usage.");
+ auto *SV = cast<ShuffleVectorInst>(VL.front());
+ unsigned SVNumElements =
+ cast<FixedVectorType>(SV->getOperand(0)->getType())->getNumElements();
+ SmallVector<int> Mask;
+ unsigned AccumulateLength = 0;
+ for (Value *V : VL) {
+ auto *SV = cast<ShuffleVectorInst>(V);
+ for (int M : SV->getShuffleMask())
+ Mask.push_back(M == PoisonMaskElem ? PoisonMaskElem
+ : AccumulateLength + M);
+ AccumulateLength += SVNumElements;
+ }
+ return Mask;
+}
+
+SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask, UseMask MaskArg) {
+ SmallBitVector UseMask(VF, true);
+ for (auto [Idx, Value] : enumerate(Mask)) {
+ if (Value == PoisonMaskElem) {
+ if (MaskArg == UseMask::UndefsAsMask)
+ UseMask.reset(Idx);
+ continue;
+ }
+ if (MaskArg == UseMask::FirstArg && Value < VF)
+ UseMask.reset(Value);
+ else if (MaskArg == UseMask::SecondArg && Value >= VF)
+ UseMask.reset(Value - VF);
+ }
+ return UseMask;
+}
+
+template <bool IsPoisonOnly>
+SmallBitVector isUndefVector(const Value *V, const SmallBitVector &UseMask) {
+ SmallBitVector Res(UseMask.empty() ? 1 : UseMask.size(), true);
+ using T = std::conditional_t<IsPoisonOnly, PoisonValue, UndefValue>;
+ if (isa<T>(V))
+ return Res;
+ auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!VecTy)
+ return Res.reset();
+ auto *C = dyn_cast<Constant>(V);
+ if (!C) {
+ if (!UseMask.empty()) {
+ const Value *Base = V;
+ while (auto *II = dyn_cast<InsertElementInst>(Base)) {
+ Base = II->getOperand(0);
+ if (isa<T>(II->getOperand(1)))
+ continue;
+ std::optional<unsigned> Idx = getElementIndex(II);
+ if (!Idx) {
+ Res.reset();
+ return Res;
+ }
+ if (*Idx < UseMask.size() && !UseMask.test(*Idx))
+ Res.reset(*Idx);
+ }
+ // TODO: Add analysis for shuffles here too.
+ if (V == Base) {
+ Res.reset();
+ } else {
+ SmallBitVector SubMask(UseMask.size(), false);
+ Res &= isUndefVector<IsPoisonOnly>(Base, SubMask);
+ }
+ } else {
+ Res.reset();
+ }
+ return Res;
+ }
+ for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I) {
+ if (Constant *Elem = C->getAggregateElement(I))
+ if (!isa<T>(Elem) &&
+ (UseMask.empty() || (I < UseMask.size() && !UseMask.test(I))))
+ Res.reset(I);
+ }
+ return Res;
+}
+
+template SmallBitVector isUndefVector<false>(const Value *,
+ const SmallBitVector &);
+template SmallBitVector isUndefVector<true>(const Value *,
+ const SmallBitVector &);
+
+bool doesInTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
+ TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI) {
+ if (!UserInst)
+ return false;
+ unsigned Opcode = UserInst->getOpcode();
+ switch (Opcode) {
+ case Instruction::Load: {
+ LoadInst *LI = cast<LoadInst>(UserInst);
+ return (LI->getPointerOperand() == Scalar);
+ }
+ case Instruction::Store: {
+ StoreInst *SI = cast<StoreInst>(UserInst);
+ return (SI->getPointerOperand() == Scalar);
+ }
+ case Instruction::Call: {
+ CallInst *CI = cast<CallInst>(UserInst);
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
+ return any_of(enumerate(CI->args()), [&](auto &&Arg) {
+ return isVectorIntrinsicWithScalarOpAtArg(ID, Arg.index(), TTI) &&
+ Arg.value().get() == Scalar;
+ });
+ }
+ default:
+ return false;
+ }
+}
+
+MemoryLocation getLocation(Instruction *I) {
+ if (StoreInst *SI = dyn_cast<StoreInst>(I))
+ return MemoryLocation::get(SI);
+ if (LoadInst *LI = dyn_cast<LoadInst>(I))
+ return MemoryLocation::get(LI);
+ return MemoryLocation();
+}
+
+bool isSimple(Instruction *I) {
+ if (LoadInst *LI = dyn_cast<LoadInst>(I))
+ return LI->isSimple();
+ if (StoreInst *SI = dyn_cast<StoreInst>(I))
+ return SI->isSimple();
+ if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
+ return !MI->isVolatile();
+ return true;
+}
+
+void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
+ bool ExtendingManyInputs) {
+ if (SubMask.empty())
+ return;
+ assert(
+ (!ExtendingManyInputs || SubMask.size() > Mask.size() ||
+ // Check if input scalars were extended to match the size of other node.
+ (SubMask.size() == Mask.size() && Mask.back() == PoisonMaskElem)) &&
+ "SubMask with many inputs support must be larger than the mask.");
+ if (Mask.empty()) {
+ Mask.append(SubMask.begin(), SubMask.end());
+ return;
+ }
+ SmallVector<int> NewMask(SubMask.size(), PoisonMaskElem);
+ int TermValue = std::min(Mask.size(), SubMask.size());
+ for (int I = 0, E = SubMask.size(); I < E; ++I) {
+ if (SubMask[I] == PoisonMaskElem ||
+ (!ExtendingManyInputs &&
+ (SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue)))
+ continue;
+ NewMask[I] = Mask[SubMask[I]];
+ }
+ Mask.swap(NewMask);
+}
+
+void fixupOrderingIndices(MutableArrayRef<unsigned> Order) {
+ const size_t Sz = Order.size();
+ SmallBitVector UnusedIndices(Sz, /*t=*/true);
+ SmallBitVector MaskedIndices(Sz);
+ for (unsigned I = 0; I < Sz; ++I) {
+ if (Order[I] < Sz)
+ UnusedIndices.reset(Order[I]);
+ else
+ MaskedIndices.set(I);
+ }
+ if (MaskedIndices.none())
+ return;
+ assert(UnusedIndices.count() == MaskedIndices.count() &&
+ "Non-synced masked/available indices.");
+ int Idx = UnusedIndices.find_first();
+ int MIdx = MaskedIndices.find_first();
+ while (MIdx >= 0) {
+ assert(Idx >= 0 && "Indices must be synced.");
+ Order[MIdx] = Idx;
+ Idx = UnusedIndices.find_next(Idx);
+ MIdx = MaskedIndices.find_next(MIdx);
+ }
+}
+
+SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, Type *ScalarTy,
+ unsigned Opcode0, unsigned Opcode1) {
+ unsigned ScalarTyNumElements = getNumElements(ScalarTy);
+ SmallBitVector OpcodeMask(VL.size() * ScalarTyNumElements, false);
+ for (unsigned Lane : seq<unsigned>(VL.size())) {
+ if (isa<PoisonValue>(VL[Lane]))
+ continue;
+ if (cast<Instruction>(VL[Lane])->getOpcode() == Opcode1)
+ OpcodeMask.set(Lane * ScalarTyNumElements,
+ Lane * ScalarTyNumElements + ScalarTyNumElements);
+ }
+ return OpcodeMask;
+}
+
+SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF) {
+ assert(none_of(Val, [](Constant *C) { return C->getType()->isVectorTy(); }) &&
+ "Expected scalar constants.");
+ SmallVector<Constant *> NewVal(Val.size() * VF);
+ for (auto [I, V] : enumerate(Val))
+ std::fill_n(NewVal.begin() + I * VF, VF, V);
+ return NewVal;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 9597b58157cd1..51181b99078c5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -17,13 +17,18 @@
#define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/MemoryLocation.h"
#include <optional>
#include <string>
namespace llvm {
+class Constant;
class Instruction;
+class TargetLibraryInfo;
+class TargetTransformInfo;
class Type;
class Value;
} // namespace llvm
@@ -161,6 +166,113 @@ template <typename T>
std::optional<unsigned> getInsertExtractIndex(const Value *Inst,
unsigned Offset);
+void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements,
+ SmallVectorImpl<int> &Mask);
+
+/// \returns the number of groups of shufflevector
+/// A group has the following features
+/// 1. All of value in a group are shufflevector.
+/// 2. The mask of all shufflevector is isExtractSubvectorMask.
+/// 3. The mask of all shufflevector uses all of the elements of the source.
+/// e.g., it is 1 group (%0)
+/// %1 = shufflevector <16 x i8> %0, <16 x i8> poison,
+/// <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+/// %2 = shufflevector <16 x i8> %0, <16 x i8> poison,
+/// <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+/// it is 2 groups (%3 and %4)
+/// %5 = shufflevector <8 x i16> %3, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+/// %6 = shufflevector <8 x i16> %3, <8 x i16> poison,
+/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+/// %7 = shufflevector <8 x i16> %4, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+/// %8 = shufflevector <8 x i16> %4, <8 x i16> poison,
+/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+/// it is 0 group
+/// %12 = shufflevector <8 x i16> %10, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+/// %13 = shufflevector <8 x i16> %11, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+unsigned getShufflevectorNumGroups(ArrayRef<Value *> VL);
+
+/// \returns a shufflevector mask which is used to vectorize shufflevectors
+/// e.g.,
+/// %5 = shufflevector <8 x i16> %3, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+/// %6 = shufflevector <8 x i16> %3, <8 x i16> poison,
+/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+/// %7 = shufflevector <8 x i16> %4, <8 x i16> poison,
+/// <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+/// %8 = shufflevector <8 x i16> %4, <8 x i16> poison,
+/// <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+/// the result is
+/// <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>
+SmallVector<int> calculateShufflevectorMask(ArrayRef<Value *> VL);
+
+/// Specifies the way the mask should be analyzed for undefs/poisonous elements
+/// in the shuffle mask.
+enum class UseMask {
+ FirstArg, ///< The mask is expected to be for permutation of 1-2 vectors,
+ ///< check for the mask elements for the first argument (mask
+ ///< indices are in range [0:VF)).
+ SecondArg, ///< The mask is expected to be for permutation of 2 vectors, check
+ ///< for the mask elements for the second argument (mask indices
+ ///< are in range [VF:2*VF))
+ UndefsAsMask ///< Consider undef mask elements (-1) as placeholders for
+ ///< future shuffle elements and mark them as ones as being used
+ ///< in future. Non-undef elements are considered as unused since
+ ///< they're already marked as used in the mask.
+};
+
+/// Prepares a use bitset for the given mask either for the first argument or
+/// for the second.
+SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask, UseMask MaskArg);
+
+/// Checks if the given value is actually an undefined constant vector.
+/// Also, if the \p UseMask is not empty, tries to check if the non-masked
+/// elements actually mask the insertelement buildvector, if any.
+template <bool IsPoisonOnly = false>
+SmallBitVector isUndefVector(const Value *V,
+ const SmallBitVector &UseMask = {});
+
+/// \returns True if in-tree use also needs extract. This refers to
+/// possible scalar operand in vectorized instruction.
+bool doesInTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
+ TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI);
+
+/// \returns the AA location that is being access by the instruction.
+MemoryLocation getLocation(Instruction *I);
+
+/// \returns True if the instruction is not a volatile or atomic load/store.
+bool isSimple(Instruction *I);
+
+/// Shuffles \p Mask in accordance with the given \p SubMask.
+/// \param ExtendingManyInputs Supports reshuffling of the mask with not only
+/// one but two input vectors.
+void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
+ bool ExtendingManyInputs = false);
+
+/// Order may have elements assigned special value (size) which is out of
+/// bounds. Such indices only appear on places which correspond to undef values
+/// (see canReuseExtract for details) and used in order to avoid undef values
+/// have effect on operands ordering.
+/// The first loop below simply finds all unused indices and then the next loop
+/// nest assigns these indices for undef values positions.
+/// As an example below Order has two undef positions and they have assigned
+/// values 3 and 7 respectively:
+/// before: 6 9 5 4 9 2 1 0
+/// after: 6 3 5 4 7 2 1 0
+void fixupOrderingIndices(MutableArrayRef<unsigned> Order);
+
+/// \returns a bitset for selecting opcodes. false for Opcode0 and true for
+/// Opcode1.
+SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, Type *ScalarTy,
+ unsigned Opcode0, unsigned Opcode1);
+
+/// Replicates the given \p Val \p VF times.
+SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
More information about the llvm-commits
mailing list