[llvm] 6f43d28 - [IR]Add NumSrcElts param to is..Mask static function in ShuffleVectorInst.
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 3 10:26:22 PDT 2023
Author: Alexey Bataev
Date: 2023-10-03T10:26:11-07:00
New Revision: 6f43d28f3452b3ef598bc12b761cfc2dbd0f34c9
URL: https://github.com/llvm/llvm-project/commit/6f43d28f3452b3ef598bc12b761cfc2dbd0f34c9
DIFF: https://github.com/llvm/llvm-project/commit/6f43d28f3452b3ef598bc12b761cfc2dbd0f34c9.diff
LOG: [IR]Add NumSrcElts param to is..Mask static function in ShuffleVectorInst.
Need to add NumSrcElts param to is..Mask functions in
ShuffleVectorInstruction class for better mask analysis. Mask.size() not
always matches the sizes of the permuted vector(s). Allows to better
estimate the cost in SLP and fix uses of the functions in other cases.
Differential Revision: https://reviews.llvm.org/D158449
Added:
Modified:
llvm/include/llvm/CodeGen/BasicTTIImpl.h
llvm/include/llvm/IR/Instructions.h
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
llvm/unittests/IR/InstructionsTest.cpp
llvm/unittests/IR/ShuffleVectorInstTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index c11d558a73e9d09..8508086ea3a4ed6 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -935,31 +935,28 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
ArrayRef<int> Mask,
VectorType *Ty, int &Index,
VectorType *&SubTy) const {
- int Limit = Mask.size() * 2;
- if (Mask.empty() ||
- // Extra check required by isSingleSourceMaskImpl function (called by
- // ShuffleVectorInst::isSingleSourceMask).
- any_of(Mask, [Limit](int I) { return I >= Limit; }))
+ if (Mask.empty())
return Kind;
+ int NumSrcElts = Ty->getElementCount().getKnownMinValue();
switch (Kind) {
case TTI::SK_PermuteSingleSrc:
- if (ShuffleVectorInst::isReverseMask(Mask))
+ if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts))
return TTI::SK_Reverse;
- if (ShuffleVectorInst::isZeroEltSplatMask(Mask))
+ if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts))
return TTI::SK_Broadcast;
break;
case TTI::SK_PermuteTwoSrc: {
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
- Mask, Mask.size(), NumSubElts, Index)) {
+ Mask, NumSrcElts, NumSubElts, Index)) {
SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts);
return TTI::SK_InsertSubvector;
}
- if (ShuffleVectorInst::isSelectMask(Mask))
+ if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts))
return TTI::SK_Select;
- if (ShuffleVectorInst::isTransposeMask(Mask))
+ if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts))
return TTI::SK_Transpose;
- if (ShuffleVectorInst::isSpliceMask(Mask, Index))
+ if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index))
return TTI::SK_Splice;
break;
}
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index a0c8406fe4ec1e5..c85727ce30a94a4 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2068,13 +2068,14 @@ class ShuffleVectorInst : public Instruction {
/// Return true if this shuffle mask chooses elements from exactly one source
/// vector.
/// Example: <7,5,undef,7>
- /// This assumes that vector operands are the same length as the mask.
- static bool isSingleSourceMask(ArrayRef<int> Mask);
- static bool isSingleSourceMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSingleSourceMask(MaskAsInts);
+ return isSingleSourceMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2082,7 +2083,8 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(ShuffleMask);
+ return !changesLength() &&
+ isSingleSourceMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2090,8 +2092,8 @@ class ShuffleVectorInst : public Instruction {
/// necessarily a no-op because it may change the number of elements from its
/// input vectors or it may provide demanded bits knowledge via undef lanes.
/// Example: <undef,undef,2,3>
- static bool isIdentityMask(ArrayRef<int> Mask);
- static bool isIdentityMask(const Constant *Mask) {
+ static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isIdentityMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
// Not possible to express a shuffle mask for a scalable vector for this
@@ -2101,7 +2103,7 @@ class ShuffleVectorInst : public Instruction {
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isIdentityMask(MaskAsInts);
+ return isIdentityMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2114,7 +2116,7 @@ class ShuffleVectorInst : public Instruction {
if (isa<ScalableVectorType>(getType()))
return false;
- return !changesLength() && isIdentityMask(ShuffleMask);
+ return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2138,12 +2140,12 @@ class ShuffleVectorInst : public Instruction {
/// In that case, the shuffle is better classified as an identity shuffle.
/// This assumes that vector operands are the same length as the mask
/// (a length-changing shuffle can never be equivalent to a vector select).
- static bool isSelectMask(ArrayRef<int> Mask);
- static bool isSelectMask(const Constant *Mask) {
+ static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isSelectMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSelectMask(MaskAsInts);
+ return isSelectMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from its source vectors
@@ -2155,19 +2157,20 @@ class ShuffleVectorInst : public Instruction {
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(ShuffleMask);
+ return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask swaps the order of elements from exactly
/// one source vector.
/// Example: <7,6,undef,4>
- /// This assumes that vector operands are the same length as the mask.
- static bool isReverseMask(ArrayRef<int> Mask);
- static bool isReverseMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isReverseMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isReverseMask(MaskAsInts);
+ return isReverseMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle swaps the order of elements from exactly
@@ -2175,19 +2178,20 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(ShuffleMask);
+ return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask chooses all elements with the same value
/// as the first element of exactly one source vector.
/// Example: <4,undef,undef,4>
- /// This assumes that vector operands are the same length as the mask.
- static bool isZeroEltSplatMask(ArrayRef<int> Mask);
- static bool isZeroEltSplatMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isZeroEltSplatMask(MaskAsInts);
+ return isZeroEltSplatMask(MaskAsInts, NumSrcElts);
}
/// Return true if all elements of this shuffle are the same value as the
@@ -2197,7 +2201,8 @@ class ShuffleVectorInst : public Instruction {
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(ShuffleMask);
+ return !changesLength() &&
+ isZeroEltSplatMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2232,12 +2237,12 @@ class ShuffleVectorInst : public Instruction {
/// ; Transposed matrix
/// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
/// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
- static bool isTransposeMask(ArrayRef<int> Mask);
- static bool isTransposeMask(const Constant *Mask) {
+ static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isTransposeMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isTransposeMask(MaskAsInts);
+ return isTransposeMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle transposes the elements of its inputs without
@@ -2246,19 +2251,21 @@ class ShuffleVectorInst : public Instruction {
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(ShuffleMask);
+ return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask is a splice mask, concatenating the two
/// inputs together and then extracts an original width vector starting from
/// the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
- static bool isSpliceMask(ArrayRef<int> Mask, int &Index);
- static bool isSpliceMask(const Constant *Mask, int &Index) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts, int &Index);
+ static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSpliceMask(MaskAsInts, Index);
+ return isSpliceMask(MaskAsInts, NumSrcElts, Index);
}
/// Return true if this shuffle splices two inputs without changing the length
@@ -2266,7 +2273,8 @@ class ShuffleVectorInst : public Instruction {
/// then extracts an original width vector starting from the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
bool isSplice(int &Index) const {
- return !changesLength() && isSpliceMask(ShuffleMask, Index);
+ return !changesLength() &&
+ isSpliceMask(ShuffleMask, ShuffleMask.size(), Index);
}
/// Return true if this shuffle mask is an extract subvector mask.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 05a19ed989befdd..f2bd5137bd20935 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24151,7 +24151,7 @@ static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
// Profitability check: only deal with extractions from the first subvector
// unless the mask becomes an identity mask.
- if (!ShuffleVectorInst::isIdentityMask(NewMask) ||
+ if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) ||
any_of(NewMask, [](int M) { return M < 0; }))
for (auto &DemandedSubvector : DemandedSubvectors)
if (DemandedSubvector.second != 0)
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index b76e355996ea530..ece3b58792dd1f8 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2136,10 +2136,10 @@ static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return UsesLHS || UsesRHS;
}
-bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isSingleSourceMaskImpl(Mask, Mask.size());
+ return isSingleSourceMaskImpl(Mask, NumSrcElts);
}
static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
@@ -2154,65 +2154,75 @@ static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return true;
}
-bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isIdentityMaskImpl(Mask, Mask.size());
+ return isIdentityMaskImpl(Mask, NumSrcElts);
}
-bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask) {
- if (!isSingleSourceMask(Mask))
+bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
+ if (!isSingleSourceMask(Mask, NumSrcElts))
return false;
// The number of elements in the mask must be at least 2.
- int NumElts = Mask.size();
- if (NumElts < 2)
+ if (NumSrcElts < 2)
return false;
- for (int i = 0; i < NumElts; ++i) {
- if (Mask[i] == -1)
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i))
+ if (Mask[I] != (NumSrcElts - 1 - I) &&
+ Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))
return false;
}
return true;
}
-bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask) {
- if (!isSingleSourceMask(Mask))
+bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
return false;
- for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
- if (Mask[i] == -1)
+ if (!isSingleSourceMask(Mask, NumSrcElts))
+ return false;
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != 0 && Mask[i] != NumElts)
+ if (Mask[I] != 0 && Mask[I] != NumSrcElts)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// Select is
diff erentiated from identity. It requires using both sources.
- if (isSingleSourceMask(Mask))
+ if (isSingleSourceMask(Mask, NumSrcElts))
return false;
- for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
- if (Mask[i] == -1)
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != i && Mask[i] != (NumElts + i))
+ if (Mask[I] != I && Mask[I] != (NumSrcElts + I))
return false;
}
return true;
}
-bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
// Example masks that will return true:
// v1 = <a, b, c, d>
// v2 = <e, f, g, h>
// trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>
// trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// 1. The number of elements in the mask must be a power-of-2 and at least 2.
- int NumElts = Mask.size();
- if (NumElts < 2 || !isPowerOf2_32(NumElts))
+ int Sz = Mask.size();
+ if (Sz < 2 || !isPowerOf2_32(Sz))
return false;
// 2. The first element of the mask must be either a 0 or a 1.
@@ -2221,23 +2231,26 @@ bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
// 3. The
diff erence between the first 2 elements must be equal to the
// number of elements in the mask.
- if ((Mask[1] - Mask[0]) != NumElts)
+ if ((Mask[1] - Mask[0]) != NumSrcElts)
return false;
// 4. The
diff erence between consecutive even-numbered and odd-numbered
// elements must be equal to 2.
- for (int i = 2; i < NumElts; ++i) {
- int MaskEltVal = Mask[i];
+ for (int I = 2; I < Sz; ++I) {
+ int MaskEltVal = Mask[I];
if (MaskEltVal == -1)
return false;
- int MaskEltPrevVal = Mask[i - 2];
+ int MaskEltPrevVal = Mask[I - 2];
if (MaskEltVal - MaskEltPrevVal != 2)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
+bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
+ int &Index) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
int StartIndex = -1;
for (int I = 0, E = Mask.size(); I != E; ++I) {
@@ -2248,7 +2261,7 @@ bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
if (StartIndex == -1) {
// Don't support a StartIndex that begins in the second input, or if the
// first non-undef index would access below the StartIndex.
- if (MaskEltVal < I || E <= (MaskEltVal - I))
+ if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))
return false;
StartIndex = MaskEltVal - I;
@@ -2543,7 +2556,7 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {
// case.
if (isa<ScalableVectorType>(getType()))
return false;
- if (!isSingleSourceMask(ShuffleMask))
+ if (!isSingleSourceMask(ShuffleMask, VF))
return false;
return isOneUseSingleSourceMask(ShuffleMask, VF);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 669cd8e6018b2f1..8c42572c374dd84 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11813,7 +11813,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) ||
(VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) &&
- ShuffleVectorInst::isReverseMask(ShuffleMask)) {
+ ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) {
SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
DAG.getConstant(8, dl, MVT::i32));
@@ -25837,7 +25837,8 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SDLoc DL(Op);
unsigned MinSVESize = Subtarget.getMinSVEVectorSizeInBits();
unsigned MaxSVESize = Subtarget.getMaxSVEVectorSizeInBits();
- bool IsSingleOp = ShuffleVectorInst::isSingleSourceMask(ShuffleMask);
+ bool IsSingleOp =
+ ShuffleVectorInst::isSingleSourceMask(ShuffleMask, ShuffleMask.size());
// Ignore two operands if no SVE2 or all index numbers couldn't
// be represented.
@@ -26020,7 +26021,8 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits();
unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits();
if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) {
- if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) {
+ if (ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size()) &&
+ Op2.isUndef()) {
Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1);
return convertFromScalableVector(DAG, VT, Op);
}
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 52ce370bbacc642..8cca167e32ee791 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -8375,7 +8375,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
unsigned EltSize = VT.getScalarSizeInBits();
if (EltSize >= 32 ||
ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
- ShuffleVectorInst::isIdentityMask(M) ||
+ ShuffleVectorInst::isIdentityMask(M, M.size()) ||
isVREVMask(M, VT, 64) ||
isVREVMask(M, VT, 32) ||
isVREVMask(M, VT, 16))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 413af1ff4b9439a..937ce7850702832 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -4339,7 +4339,8 @@ static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
assert(VT.getVectorElementType() == MVT::i1);
- if (!ShuffleVectorInst::isReverseMask(SVN->getMask()) ||
+ if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
+ SVN->getMask().size()) ||
!SVN->getOperand(1).isUndef())
return SDValue();
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 2f510a215d80bf2..e947d4d1e8acdee 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -322,7 +322,10 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
LT.second.getVectorElementType().getSizeInBits() ==
Tp->getElementType()->getPrimitiveSizeInBits() &&
LT.second.getVectorNumElements() <
- cast<FixedVectorType>(Tp)->getNumElements()) {
+ cast<FixedVectorType>(Tp)->getNumElements() &&
+ divideCeil(Mask.size(),
+ cast<FixedVectorType>(Tp)->getNumElements()) ==
+ static_cast<unsigned>(*LT.first.getValue())) {
unsigned NumRegs = *LT.first.getValue();
unsigned VF = cast<FixedVectorType>(Tp)->getNumElements();
unsigned SubVF = PowerOf2Ceil(VF / NumRegs);
@@ -498,7 +501,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
InstructionCost Cost = MemCost;
for (unsigned Index : Indices) {
FixedVectorType *SubVecTy =
- FixedVectorType::get(FVTy->getElementType(), VF);
+ FixedVectorType::get(FVTy->getElementType(), VF * Factor);
auto Mask = createStrideMask(Index, Factor, VF);
InstructionCost ShuffleCost =
getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index d838d1f96e310d8..5a6413653908ec8 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1596,7 +1596,6 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
BaseTp->getElementType()->getPrimitiveSizeInBits() &&
LegalVT.getVectorNumElements() <
cast<FixedVectorType>(BaseTp)->getNumElements()) {
-
unsigned VecTySize = DL.getTypeStoreSize(BaseTp);
unsigned LegalVTSize = LegalVT.getStoreSize();
// Number of source vectors after legalization:
@@ -1621,6 +1620,10 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
// copy of the previous destination register (the cost is
// TTI::TCC_Basic). If the source register is just reused, the cost for
// this operation is 0.
+ NumOfDests =
+ getTypeLegalizationCost(
+ FixedVectorType::get(BaseTp->getElementType(), Mask.size()))
+ .first;
unsigned E = *NumOfDests.getValue();
unsigned NormalizedVF =
LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
@@ -1635,7 +1638,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
[this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
&Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
- if (!ShuffleVectorInst::isIdentityMask(RegMask)) {
+ if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) {
// Check if the previous register can be just copied to the next
// one.
if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 69d7593d5fc1e36..c8b58c51d4e6ecb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2122,8 +2122,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) {
NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i];
// A select mask with undef elements might look like an identity mask.
- assert((ShuffleVectorInst::isSelectMask(NewMask) ||
- ShuffleVectorInst::isIdentityMask(NewMask)) &&
+ assert((ShuffleVectorInst::isSelectMask(NewMask, NumElts) ||
+ ShuffleVectorInst::isIdentityMask(NewMask, NumElts)) &&
"Unexpected shuffle mask");
return new ShuffleVectorInst(X, Y, NewMask);
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bd67b69e0b55d9e..70fcfd4cbde99a1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3812,7 +3812,7 @@ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
inversePermutation(Order, MaskOrder);
}
reorderReuses(MaskOrder, Mask);
- if (ShuffleVectorInst::isIdentityMask(MaskOrder)) {
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) {
Order.clear();
return;
}
@@ -4333,7 +4333,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
static bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask,
unsigned Sz) {
ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
- if (ShuffleVectorInst::isIdentityMask(FirstCluster))
+ if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
return false;
for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
ArrayRef<int> Cluster = Mask.slice(I, Sz);
@@ -6646,9 +6646,24 @@ class BaseShuffleAnalysis {
bool IsStrict) {
int Limit = Mask.size();
int VF = VecTy->getNumElements();
- return (VF == Limit || !IsStrict) &&
- all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(Mask);
+ int Index = -1;
+ if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit))
+ return true;
+ if (!IsStrict) {
+ // Consider extract subvector starting from index 0.
+ if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
+ Index == 0)
+ return true;
+ // All VF-size submasks are identity (e.g.
+ // <poison,poison,poison,poison,0,1,2,poison,poison,1,2,3> etc. for VF 4).
+ if (Limit % VF == 0 && all_of(seq<int>(0, Limit / VF), [=](int Idx) {
+ ArrayRef<int> Slice = Mask.slice(Idx * VF, VF);
+ return all_of(Slice, [](int I) { return I == PoisonMaskElem; }) ||
+ ShuffleVectorInst::isIdentityMask(Slice, VF);
+ }))
+ return true;
+ }
+ return false;
}
/// Tries to combine 2
diff erent masks into single one.
@@ -6718,7 +6733,8 @@ class BaseShuffleAnalysis {
if (isIdentityMask(Mask, SVTy, /*IsStrict=*/false)) {
if (!IdentityOp || !SinglePermute ||
(isIdentityMask(Mask, SVTy, /*IsStrict=*/true) &&
- !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask))) {
+ !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask,
+ IdentityMask.size()))) {
IdentityOp = SV;
// Store current mask in the IdentityMask so later we did not lost
// this info if IdentityOp is selected as the best candidate for the
@@ -6788,7 +6804,7 @@ class BaseShuffleAnalysis {
}
if (auto *OpTy = dyn_cast<FixedVectorType>(Op->getType());
!OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) ||
- ShuffleVectorInst::isZeroEltSplatMask(Mask)) {
+ ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) {
if (IdentityOp) {
V = IdentityOp;
assert(Mask.size() == IdentityMask.size() &&
@@ -6804,7 +6820,7 @@ class BaseShuffleAnalysis {
/*IsStrict=*/true) ||
(Shuffle && Mask.size() == Shuffle->getShuffleMask().size() &&
Shuffle->isZeroEltSplat() &&
- ShuffleVectorInst::isZeroEltSplatMask(Mask)));
+ ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())));
}
V = Op;
return false;
@@ -6909,11 +6925,9 @@ class BaseShuffleAnalysis {
CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
}
}
- const int Limit = CombinedMask1.size() * 2;
- if (Op1 == Op2 && Limit == 2 * VF &&
- all_of(CombinedMask1, [=](int Idx) { return Idx < Limit; }) &&
- (ShuffleVectorInst::isIdentityMask(CombinedMask1) ||
- (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1) &&
+ if (Op1 == Op2 &&
+ (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) ||
+ (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) &&
isa<ShuffleVectorInst>(Op1) &&
cast<ShuffleVectorInst>(Op1)->getShuffleMask() ==
ArrayRef(CombinedMask1))))
@@ -6953,6 +6967,20 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
SmallPtrSetImpl<Value *> &CheckedExtracts;
constexpr static TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
+ if (Ty->getScalarType()->isPointerTy()) {
+ Constant *Res = ConstantExpr::getIntToPtr(
+ ConstantInt::getAllOnesValue(
+ IntegerType::get(Ty->getContext(),
+ DL.getTypeStoreSizeInBits(Ty->getScalarType()))),
+ Ty->getScalarType());
+ if (auto *VTy = dyn_cast<VectorType>(Ty))
+ Res = ConstantVector::getSplat(VTy->getElementCount(), Res);
+ return Res;
+ }
+ return Constant::getAllOnesValue(Ty);
+ }
+
InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) {
if ((!Root && allConstant(VL)) || all_of(VL, UndefValue::classof))
return TTI::TCC_Free;
@@ -7183,11 +7211,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const TargetTransformInfo &TTI;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
- int Limit = 2 * VF;
+ int Index = -1;
return Mask.empty() ||
(VF == Mask.size() &&
- all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(Mask));
+ ShuffleVectorInst::isIdentityMask(Mask, VF)) ||
+ (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
+ Index == 0);
}
public:
@@ -7200,21 +7229,17 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return TTI.getShuffleCost(
- TTI::SK_PermuteTwoSrc,
- FixedVectorType::get(
- cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
- Mask);
+ return TTI.getShuffleCost(TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(V1->getType()), Mask);
}
InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask) const {
// Empty mask or identity mask are free.
- if (isEmptyOrIdentity(Mask, Mask.size()))
+ unsigned VF =
+ cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
+ if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return TTI.getShuffleCost(
- TTI::SK_PermuteSingleSrc,
- FixedVectorType::get(
- cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
- Mask);
+ return TTI.getShuffleCost(TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(V1->getType()), Mask);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
@@ -7231,47 +7256,119 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const PointerUnion<Value *, const TreeEntry *> &P2,
ArrayRef<int> Mask) {
ShuffleCostBuilder Builder(TTI);
+ SmallVector<int> CommonMask(Mask.begin(), Mask.end());
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
- unsigned CommonVF = 0;
- if (!V1) {
+ unsigned CommonVF = Mask.size();
+ if (!V1 && !V2 && !P2.isNull()) {
+ // Shuffle 2 entry nodes.
const TreeEntry *E = P1.get<const TreeEntry *>();
unsigned VF = E->getVectorFactor();
- if (V2) {
- unsigned V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
- if (V2VF != VF && V2VF == E->Scalars.size())
- VF = E->Scalars.size();
- } else if (!P2.isNull()) {
- const TreeEntry *E2 = P2.get<const TreeEntry *>();
- if (E->Scalars.size() == E2->Scalars.size())
- CommonVF = VF = E->Scalars.size();
- } else {
- // P2 is empty, check that we have same node + reshuffle (if any).
- if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
- VF = E->Scalars.size();
- SmallVector<int> CommonMask(Mask.begin(), Mask.end());
- ::addMask(CommonMask, E->getCommonMask());
- V1 = Constant::getNullValue(
- FixedVectorType::get(E->Scalars.front()->getType(), VF));
- return BaseShuffleAnalysis::createShuffle<InstructionCost>(
- V1, nullptr, CommonMask, Builder);
+ const TreeEntry *E2 = P2.get<const TreeEntry *>();
+ CommonVF = std::max(VF, E2->getVectorFactor());
+ assert(all_of(Mask,
+ [=](int Idx) {
+ return Idx < 2 * static_cast<int>(CommonVF);
+ }) &&
+ "All elements in mask must be less than 2 * CommonVF.");
+ if (E->Scalars.size() == E2->Scalars.size()) {
+ SmallVector<int> EMask = E->getCommonMask();
+ SmallVector<int> E2Mask = E2->getCommonMask();
+ if (!EMask.empty() || !E2Mask.empty()) {
+ for (int &Idx : CommonMask) {
+ if (Idx == PoisonMaskElem)
+ continue;
+ if (Idx < static_cast<int>(CommonVF) && !EMask.empty())
+ Idx = EMask[Idx];
+ else if (Idx >= static_cast<int>(CommonVF))
+ Idx = (E2Mask.empty() ? Idx - CommonVF : E2Mask[Idx - CommonVF]) +
+ E->Scalars.size();
+ }
}
+ CommonVF = E->Scalars.size();
}
V1 = Constant::getNullValue(
- FixedVectorType::get(E->Scalars.front()->getType(), VF));
- }
- if (!V2 && !P2.isNull()) {
- const TreeEntry *E = P2.get<const TreeEntry *>();
+ FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
+ V2 = getAllOnesValue(
+ *R.DL, FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
+ } else if (!V1 && P2.isNull()) {
+ // Shuffle single entry node.
+ const TreeEntry *E = P1.get<const TreeEntry *>();
unsigned VF = E->getVectorFactor();
- unsigned V1VF = cast<FixedVectorType>(V1->getType())->getNumElements();
- if (!CommonVF && V1VF == E->Scalars.size())
+ CommonVF = VF;
+ assert(
+ all_of(Mask,
+ [=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
+ "All elements in mask must be less than CommonVF.");
+ if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
+ SmallVector<int> EMask = E->getCommonMask();
+ assert(!EMask.empty() && "Expected non-empty common mask.");
+ for (int &Idx : CommonMask) {
+ if (Idx != PoisonMaskElem)
+ Idx = EMask[Idx];
+ }
CommonVF = E->Scalars.size();
- if (CommonVF)
- VF = CommonVF;
- V2 = Constant::getNullValue(
- FixedVectorType::get(E->Scalars.front()->getType(), VF));
- }
- return BaseShuffleAnalysis::createShuffle<InstructionCost>(V1, V2, Mask,
- Builder);
+ }
+ V1 = Constant::getNullValue(
+ FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
+ } else if (V1 && P2.isNull()) {
+ // Shuffle single vector.
+ CommonVF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ assert(
+ all_of(Mask,
+ [=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
+ "All elements in mask must be less than CommonVF.");
+ } else if (V1 && !V2) {
+ // Shuffle vector and tree node.
+ unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ const TreeEntry *E2 = P2.get<const TreeEntry *>();
+ CommonVF = std::max(VF, E2->getVectorFactor());
+ assert(all_of(Mask,
+ [=](int Idx) {
+ return Idx < 2 * static_cast<int>(CommonVF);
+ }) &&
+ "All elements in mask must be less than 2 * CommonVF.");
+ if (E2->Scalars.size() == VF && VF != CommonVF) {
+ SmallVector<int> E2Mask = E2->getCommonMask();
+ assert(!E2Mask.empty() && "Expected non-empty common mask.");
+ for (int &Idx : CommonMask) {
+ if (Idx == PoisonMaskElem)
+ continue;
+ if (Idx >= static_cast<int>(CommonVF))
+ Idx = E2Mask[Idx - CommonVF] + VF;
+ }
+ CommonVF = VF;
+ }
+ V1 = Constant::getNullValue(
+ FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF));
+ V2 = getAllOnesValue(
+ *R.DL,
+ FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF));
+ } else {
+ assert(V1 && V2 && "Expected both vectors.");
+ unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ CommonVF =
+ std::max(VF, cast<FixedVectorType>(V2->getType())->getNumElements());
+ assert(all_of(Mask,
+ [=](int Idx) {
+ return Idx < 2 * static_cast<int>(CommonVF);
+ }) &&
+ "All elements in mask must be less than 2 * CommonVF.");
+ if (V1->getType() != V2->getType()) {
+ V1 = Constant::getNullValue(FixedVectorType::get(
+ cast<FixedVectorType>(V1->getType())->getElementType(), CommonVF));
+ V2 = getAllOnesValue(
+ *R.DL, FixedVectorType::get(
+ cast<FixedVectorType>(V1->getType())->getElementType(),
+ CommonVF));
+ }
+ }
+ InVectors.front() = Constant::getNullValue(FixedVectorType::get(
+ cast<FixedVectorType>(V1->getType())->getElementType(),
+ CommonMask.size()));
+ if (InVectors.size() == 2)
+ InVectors.pop_back();
+ return BaseShuffleAnalysis::createShuffle<InstructionCost>(
+ V1, V2, CommonMask, Builder);
}
public:
@@ -7415,7 +7512,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
return ConstantVector::getSplat(
ElementCount::getFixed(VL.size()),
- Constant::getNullValue(VL.front()->getType()));
+ getAllOnesValue(*R.DL, VL.front()->getType()));
}
/// Finalize emission of the shuffles.
InstructionCost
@@ -7424,31 +7521,22 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
IsFinalized = true;
if (Action) {
const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
- if (InVectors.size() == 2) {
+ if (InVectors.size() == 2)
Cost += createShuffle(Vec, InVectors.back(), CommonMask);
- InVectors.pop_back();
- } else {
+ else
Cost += createShuffle(Vec, nullptr, CommonMask);
- }
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (CommonMask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx;
assert(VF > 0 &&
"Expected vector length for the final value before action.");
- Value *V = Vec.dyn_cast<Value *>();
- if (!Vec.isNull() && !V)
- V = Constant::getNullValue(FixedVectorType::get(
- Vec.get<const TreeEntry *>()->Scalars.front()->getType(),
- CommonMask.size()));
+ Value *V = Vec.get<Value *>();
Action(V, CommonMask);
+ InVectors.front() = V;
}
::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true);
if (CommonMask.empty())
return Cost;
- int Limit = CommonMask.size() * 2;
- if (all_of(CommonMask, [=](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(CommonMask))
- return Cost;
return Cost +
createShuffle(InVectors.front(),
InVectors.size() == 2 ? InVectors.back() : nullptr,
@@ -7624,7 +7712,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
if (NeedToShuffleReuses)
::addMask(Mask, E->ReuseShuffleIndices);
- if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask))
+ if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
CommonCost =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask);
assert((E->State == TreeEntry::Vectorize ||
@@ -7975,7 +8063,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
} else {
SmallVector<int> Mask;
inversePermutation(OpTE->ReorderIndices, Mask);
- if (ShuffleVectorInst::isReverseMask(Mask))
+ if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
CCH = TTI::CastContextHint::Reversed;
}
}
@@ -8834,9 +8922,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
unsigned VecVF = TE->getVectorFactor();
if (VF != VecVF &&
(any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); }) ||
- (all_of(Mask,
- [VF](int Idx) { return Idx < 2 * static_cast<int>(VF); }) &&
- !ShuffleVectorInst::isIdentityMask(Mask)))) {
+ !ShuffleVectorInst::isIdentityMask(Mask, VF))) {
SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
@@ -8865,9 +8951,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
assert((TEs.size() == 1 || TEs.size() == 2) &&
"Expected exactly 1 or 2 tree entries.");
if (TEs.size() == 1) {
- int Limit = 2 * Mask.size();
- if (!all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) ||
- !ShuffleVectorInst::isIdentityMask(Mask)) {
+ if (!ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
InstructionCost C =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FTy, Mask);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
@@ -9643,7 +9727,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
return V1;
unsigned VF = Mask.size();
unsigned LocalVF = cast<FixedVectorType>(V1->getType())->getNumElements();
- if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask))
+ if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF))
return V1;
Value *Vec = Builder.CreateShuffleVector(V1, Mask);
if (auto *I = dyn_cast<Instruction>(Vec)) {
@@ -10056,9 +10140,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
return false;
unsigned I =
*find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
- int Sz = Mask.size();
- if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) &&
- ShuffleVectorInst::isIdentityMask(Mask))
+ if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
std::iota(Mask.begin(), Mask.end(), 0);
else
std::fill(Mask.begin(), Mask.end(), I);
@@ -10302,11 +10384,11 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
(ExtractShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(ExtractMask, [&](int I) { return I >= EMSz; }) &&
- ShuffleVectorInst::isIdentityMask(ExtractMask)) ||
+ ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) ||
(GatherShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(Mask, [&](int I) { return I >= MSz; }) &&
- ShuffleVectorInst::isIdentityMask(Mask));
+ ShuffleVectorInst::isIdentityMask(Mask, MSz));
bool EnoughConstsForShuffle =
IsSingleShuffle &&
(none_of(GatheredScalars,
@@ -10599,7 +10681,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) &&
NumElts != NumScalars) {
if (IsFirstUndef.all()) {
- if (!ShuffleVectorInst::isIdentityMask(InsertMask)) {
+ if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
@@ -11422,7 +11504,7 @@ Value *BoUpSLP::vectorizeTree(
// non-resizing mask.
if (Mask.size() != cast<FixedVectorType>(Vals.front()->getType())
->getNumElements() ||
- !ShuffleVectorInst::isIdentityMask(Mask))
+ !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
return CreateShuffle(Vals.front(), nullptr, Mask);
return Vals.front();
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index d6196c84e8e2de2..99f03cdcf648c02 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -461,44 +461,21 @@ exit:
define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
; CHECK-LABEL: @combine_load_factor2_i64(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
-; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4
; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index c5981c44d96dd92..eb3d395f4c6a6f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -241,21 +241,10 @@ define void @tiny_vector_gather(ptr %a, ptr %v1, ptr %v2) {
; CHECK-LABEL: @tiny_vector_gather(
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[V1:%.*]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[V2:%.*]], align 4
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[A:%.*]], align 16
-; CHECK-NEXT: [[PTR1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR1]], align 4
-; CHECK-NEXT: [[PTR2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR2]], align 8
-; CHECK-NEXT: [[PTR3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 3
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR3]], align 4
-; CHECK-NEXT: [[PTR4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR4]], align 16
-; CHECK-NEXT: [[PTR5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 5
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR5]], align 4
-; CHECK-NEXT: [[PTR6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 6
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR6]], align 8
-; CHECK-NEXT: [[PTR7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 7
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR7]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[A:%.*]], align 16
; CHECK-NEXT: ret void
;
%1 = load i32, ptr %v1, align 4
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index 637c69204addfe2..20b8529b386324f 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1024,71 +1024,141 @@ TEST(InstructionsTest, ShuffleMaskQueries) {
Constant *C7 = ConstantInt::get(Int32Ty, 7);
Constant *Identity = ConstantVector::get({C0, CU, C2, C3, C4});
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(Identity));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Identity)); // identity is distinguished from select
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Identity));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Identity)); // identity is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Identity));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Identity));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Identity,
+ cast<FixedVectorType>(Identity->getType())
+ ->getNumElements())); // identity is distinguished from select
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ Identity, cast<FixedVectorType>(Identity->getType())
+ ->getNumElements())); // identity is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
Constant *Select = ConstantVector::get({CU, C1, C5});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Select));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Select));
-
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+
Constant *Reverse = ConstantVector::get({C3, C2, C1, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Reverse));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Reverse));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(Reverse));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Reverse)); // reverse is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Reverse));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Reverse));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())
+ ->getNumElements())); // reverse is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
Constant *SingleSource = ConstantVector::get({C2, C2, C0, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(SingleSource));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
Constant *ZeroEltSplat = ConstantVector::get({C0, C0, CU, C0});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(ZeroEltSplat));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ZeroEltSplat)); // 0-splat is always single source
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(ZeroEltSplat));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ZeroEltSplat, cast<FixedVectorType>(ZeroEltSplat->getType())
+ ->getNumElements())); // 0-splat is always single source
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
Constant *Transpose = ConstantVector::get({C0, C4, C2, C6});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Transpose));
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
// More tests to make sure the logic is/stays correct...
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({CU, C1, CU, C3})));
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({C4, CU, C6, CU})));
-
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({C4, C1, C6, CU})));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({CU, C1, C6, C3})));
-
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C7, C6, CU, C4})));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C3, CU, C1, CU})));
-
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C7, C5, CU, C7})));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C3, C0, CU, C3})));
-
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({C4, CU, CU, C4})));
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({CU, C0, CU, C0})));
-
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C5, C3, C7})));
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3})));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ ConstantVector::get({CU, C1, CU, C3}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ ConstantVector::get({C4, CU, C6, CU}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ ConstantVector::get({C4, C1, C6, CU}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ ConstantVector::get({CU, C1, C6, C3}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ ConstantVector::get({C7, C6, CU, C4}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ ConstantVector::get({C3, CU, C1, CU}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ConstantVector::get({C7, C5, CU, C7}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ConstantVector::get({C3, C0, CU, C3}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ConstantVector::get({C4, CU, CU, C4}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ConstantVector::get({CU, C0, CU, C0}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
+ ConstantVector::get({C1, C5, C3, C7}), 4));
+ EXPECT_TRUE(
+ ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}), 2));
// Nothing special about the values here - just re-using inputs to reduce code.
Constant *V0 = ConstantVector::get({C0, C1, C2, C3});
diff --git a/llvm/unittests/IR/ShuffleVectorInstTest.cpp b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
index ba1b9807cffb43b..5d840ec8d672047 100644
--- a/llvm/unittests/IR/ShuffleVectorInstTest.cpp
+++ b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
@@ -14,51 +14,84 @@ using namespace llvm;
namespace {
TEST(ShuffleVectorInst, isIdentityMask) {
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 5));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 3));
+
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 6));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}, 4));
}
TEST(ShuffleVectorInst, isSelectMask) {
- ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}));
+ ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 4));
+
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}));
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}, 4));
}
TEST(ShuffleVectorInst, isReverseMask) {
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}));
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}, 4));
}
TEST(ShuffleVectorInst, isZeroEltSplatMask) {
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}));
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 4));
+
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}, 4));
}
TEST(ShuffleVectorInst, isTransposeMask) {
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}));
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}, 4));
}
TEST(ShuffleVectorInst, isSpliceMask) {
int Index;
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 4, Index));
ASSERT_EQ(0, Index);
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 7, Index));
ASSERT_EQ(1, Index);
- ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, Index));
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 3, Index));
+ ASSERT_FALSE(
+ ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 6, Index));
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 5, Index));
+ ASSERT_FALSE(
+ ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 8, Index));
+
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, 4, Index));
}
TEST(ShuffleVectorInst, isExtractSubvectorMask) {
More information about the llvm-commits
mailing list