[llvm] 0738953 - Revert "[IR]Add NumSrcElts param to is..Mask static function in ShuffleVectorInst."
Arthur Eubanks via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 4 14:40:48 PDT 2023
Author: Arthur Eubanks
Date: 2023-10-04T14:37:16-07:00
New Revision: 07389535a702863248ee052bb406c93c814c3d03
URL: https://github.com/llvm/llvm-project/commit/07389535a702863248ee052bb406c93c814c3d03
DIFF: https://github.com/llvm/llvm-project/commit/07389535a702863248ee052bb406c93c814c3d03.diff
LOG: Revert "[IR]Add NumSrcElts param to is..Mask static function in ShuffleVectorInst."
This reverts commit b186f1f68be11630355afb0c08b80374a6d31782.
Causes crashes, see https://reviews.llvm.org/D158449.
Added:
Modified:
llvm/include/llvm/CodeGen/BasicTTIImpl.h
llvm/include/llvm/IR/Instructions.h
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
llvm/unittests/IR/InstructionsTest.cpp
llvm/unittests/IR/ShuffleVectorInstTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 8508086ea3a4ed6..c11d558a73e9d09 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -935,28 +935,31 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
ArrayRef<int> Mask,
VectorType *Ty, int &Index,
VectorType *&SubTy) const {
- if (Mask.empty())
+ int Limit = Mask.size() * 2;
+ if (Mask.empty() ||
+ // Extra check required by isSingleSourceMaskImpl function (called by
+ // ShuffleVectorInst::isSingleSourceMask).
+ any_of(Mask, [Limit](int I) { return I >= Limit; }))
return Kind;
- int NumSrcElts = Ty->getElementCount().getKnownMinValue();
switch (Kind) {
case TTI::SK_PermuteSingleSrc:
- if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts))
+ if (ShuffleVectorInst::isReverseMask(Mask))
return TTI::SK_Reverse;
- if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts))
+ if (ShuffleVectorInst::isZeroEltSplatMask(Mask))
return TTI::SK_Broadcast;
break;
case TTI::SK_PermuteTwoSrc: {
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
- Mask, NumSrcElts, NumSubElts, Index)) {
+ Mask, Mask.size(), NumSubElts, Index)) {
SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts);
return TTI::SK_InsertSubvector;
}
- if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts))
+ if (ShuffleVectorInst::isSelectMask(Mask))
return TTI::SK_Select;
- if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts))
+ if (ShuffleVectorInst::isTransposeMask(Mask))
return TTI::SK_Transpose;
- if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index))
+ if (ShuffleVectorInst::isSpliceMask(Mask, Index))
return TTI::SK_Splice;
break;
}
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index c85727ce30a94a4..a0c8406fe4ec1e5 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2068,14 +2068,13 @@ class ShuffleVectorInst : public Instruction {
/// Return true if this shuffle mask chooses elements from exactly one source
/// vector.
/// Example: <7,5,undef,7>
- /// This assumes that vector operands (of length \p NumSrcElts) are the same
- /// length as the mask.
- static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) {
+ /// This assumes that vector operands are the same length as the mask.
+ static bool isSingleSourceMask(ArrayRef<int> Mask);
+ static bool isSingleSourceMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSingleSourceMask(MaskAsInts, NumSrcElts);
+ return isSingleSourceMask(MaskAsInts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2083,8 +2082,7 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() &&
- isSingleSourceMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2092,8 +2090,8 @@ class ShuffleVectorInst : public Instruction {
/// necessarily a no-op because it may change the number of elements from its
/// input vectors or it may provide demanded bits knowledge via undef lanes.
/// Example: <undef,undef,2,3>
- static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isIdentityMask(const Constant *Mask, int NumSrcElts) {
+ static bool isIdentityMask(ArrayRef<int> Mask);
+ static bool isIdentityMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
// Not possible to express a shuffle mask for a scalable vector for this
@@ -2103,7 +2101,7 @@ class ShuffleVectorInst : public Instruction {
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isIdentityMask(MaskAsInts, NumSrcElts);
+ return isIdentityMask(MaskAsInts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2116,7 +2114,7 @@ class ShuffleVectorInst : public Instruction {
if (isa<ScalableVectorType>(getType()))
return false;
- return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2140,12 +2138,12 @@ class ShuffleVectorInst : public Instruction {
/// In that case, the shuffle is better classified as an identity shuffle.
/// This assumes that vector operands are the same length as the mask
/// (a length-changing shuffle can never be equivalent to a vector select).
- static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isSelectMask(const Constant *Mask, int NumSrcElts) {
+ static bool isSelectMask(ArrayRef<int> Mask);
+ static bool isSelectMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSelectMask(MaskAsInts, NumSrcElts);
+ return isSelectMask(MaskAsInts);
}
/// Return true if this shuffle chooses elements from its source vectors
@@ -2157,20 +2155,19 @@ class ShuffleVectorInst : public Instruction {
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
/// one source vector.
/// Example: <7,6,undef,4>
- /// This assumes that vector operands (of length \p NumSrcElts) are the same
- /// length as the mask.
- static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isReverseMask(const Constant *Mask, int NumSrcElts) {
+ /// This assumes that vector operands are the same length as the mask.
+ static bool isReverseMask(ArrayRef<int> Mask);
+ static bool isReverseMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isReverseMask(MaskAsInts, NumSrcElts);
+ return isReverseMask(MaskAsInts);
}
/// Return true if this shuffle swaps the order of elements from exactly
@@ -2178,20 +2175,19 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
/// as the first element of exactly one source vector.
/// Example: <4,undef,undef,4>
- /// This assumes that vector operands (of length \p NumSrcElts) are the same
- /// length as the mask.
- static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) {
+ /// This assumes that vector operands are the same length as the mask.
+ static bool isZeroEltSplatMask(ArrayRef<int> Mask);
+ static bool isZeroEltSplatMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isZeroEltSplatMask(MaskAsInts, NumSrcElts);
+ return isZeroEltSplatMask(MaskAsInts);
}
/// Return true if all elements of this shuffle are the same value as the
@@ -2201,8 +2197,7 @@ class ShuffleVectorInst : public Instruction {
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() &&
- isZeroEltSplatMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2237,12 +2232,12 @@ class ShuffleVectorInst : public Instruction {
/// ; Transposed matrix
/// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
/// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
- static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
- static bool isTransposeMask(const Constant *Mask, int NumSrcElts) {
+ static bool isTransposeMask(ArrayRef<int> Mask);
+ static bool isTransposeMask(const Constant *Mask) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isTransposeMask(MaskAsInts, NumSrcElts);
+ return isTransposeMask(MaskAsInts);
}
/// Return true if this shuffle transposes the elements of its inputs without
@@ -2251,21 +2246,19 @@ class ShuffleVectorInst : public Instruction {
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size());
+ return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is a splice mask, concatenating the two
/// inputs together and then extracts an original width vector starting from
/// the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
- /// This assumes that vector operands (of length \p NumSrcElts) are the same
- /// length as the mask.
- static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts, int &Index);
- static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) {
+ static bool isSpliceMask(ArrayRef<int> Mask, int &Index);
+ static bool isSpliceMask(const Constant *Mask, int &Index) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSpliceMask(MaskAsInts, NumSrcElts, Index);
+ return isSpliceMask(MaskAsInts, Index);
}
/// Return true if this shuffle splices two inputs without changing the length
@@ -2273,8 +2266,7 @@ class ShuffleVectorInst : public Instruction {
/// then extracts an original width vector starting from the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
bool isSplice(int &Index) const {
- return !changesLength() &&
- isSpliceMask(ShuffleMask, ShuffleMask.size(), Index);
+ return !changesLength() && isSpliceMask(ShuffleMask, Index);
}
/// Return true if this shuffle mask is an extract subvector mask.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f2bd5137bd20935..05a19ed989befdd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24151,7 +24151,7 @@ static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
// Profitability check: only deal with extractions from the first subvector
// unless the mask becomes an identity mask.
- if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) ||
+ if (!ShuffleVectorInst::isIdentityMask(NewMask) ||
any_of(NewMask, [](int M) { return M < 0; }))
for (auto &DemandedSubvector : DemandedSubvectors)
if (DemandedSubvector.second != 0)
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index ece3b58792dd1f8..b76e355996ea530 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2136,10 +2136,10 @@ static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return UsesLHS || UsesRHS;
}
-bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {
+bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask) {
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isSingleSourceMaskImpl(Mask, NumSrcElts);
+ return isSingleSourceMaskImpl(Mask, Mask.size());
}
static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
@@ -2154,75 +2154,65 @@ static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return true;
}
-bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
+bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask) {
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isIdentityMaskImpl(Mask, NumSrcElts);
+ return isIdentityMaskImpl(Mask, Mask.size());
}
-bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
- if (!isSingleSourceMask(Mask, NumSrcElts))
+bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask) {
+ if (!isSingleSourceMask(Mask))
return false;
// The number of elements in the mask must be at least 2.
- if (NumSrcElts < 2)
+ int NumElts = Mask.size();
+ if (NumElts < 2)
return false;
- for (int I = 0, E = Mask.size(); I < E; ++I) {
- if (Mask[I] == -1)
+ for (int i = 0; i < NumElts; ++i) {
+ if (Mask[i] == -1)
continue;
- if (Mask[I] != (NumSrcElts - 1 - I) &&
- Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))
+ if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i))
return false;
}
return true;
}
-bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
- if (!isSingleSourceMask(Mask, NumSrcElts))
+bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask) {
+ if (!isSingleSourceMask(Mask))
return false;
- for (int I = 0, E = Mask.size(); I < E; ++I) {
- if (Mask[I] == -1)
+ for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
+ if (Mask[i] == -1)
continue;
- if (Mask[I] != 0 && Mask[I] != NumSrcElts)
+ if (Mask[i] != 0 && Mask[i] != NumElts)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
+bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask) {
// Select is
diff erentiated from identity. It requires using both sources.
- if (isSingleSourceMask(Mask, NumSrcElts))
+ if (isSingleSourceMask(Mask))
return false;
- for (int I = 0, E = Mask.size(); I < E; ++I) {
- if (Mask[I] == -1)
+ for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
+ if (Mask[i] == -1)
continue;
- if (Mask[I] != I && Mask[I] != (NumSrcElts + I))
+ if (Mask[i] != i && Mask[i] != (NumElts + i))
return false;
}
return true;
}
-bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
+bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
// Example masks that will return true:
// v1 = <a, b, c, d>
// v2 = <e, f, g, h>
// trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>
// trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
// 1. The number of elements in the mask must be a power-of-2 and at least 2.
- int Sz = Mask.size();
- if (Sz < 2 || !isPowerOf2_32(Sz))
+ int NumElts = Mask.size();
+ if (NumElts < 2 || !isPowerOf2_32(NumElts))
return false;
// 2. The first element of the mask must be either a 0 or a 1.
@@ -2231,26 +2221,23 @@ bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
// 3. The
diff erence between the first 2 elements must be equal to the
// number of elements in the mask.
- if ((Mask[1] - Mask[0]) != NumSrcElts)
+ if ((Mask[1] - Mask[0]) != NumElts)
return false;
// 4. The
diff erence between consecutive even-numbered and odd-numbered
// elements must be equal to 2.
- for (int I = 2; I < Sz; ++I) {
- int MaskEltVal = Mask[I];
+ for (int i = 2; i < NumElts; ++i) {
+ int MaskEltVal = Mask[i];
if (MaskEltVal == -1)
return false;
- int MaskEltPrevVal = Mask[I - 2];
+ int MaskEltPrevVal = Mask[i - 2];
if (MaskEltVal - MaskEltPrevVal != 2)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
- int &Index) {
- if (Mask.size() != static_cast<unsigned>(NumSrcElts))
- return false;
+bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
int StartIndex = -1;
for (int I = 0, E = Mask.size(); I != E; ++I) {
@@ -2261,7 +2248,7 @@ bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
if (StartIndex == -1) {
// Don't support a StartIndex that begins in the second input, or if the
// first non-undef index would access below the StartIndex.
- if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))
+ if (MaskEltVal < I || E <= (MaskEltVal - I))
return false;
StartIndex = MaskEltVal - I;
@@ -2556,7 +2543,7 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {
// case.
if (isa<ScalableVectorType>(getType()))
return false;
- if (!isSingleSourceMask(ShuffleMask, VF))
+ if (!isSingleSourceMask(ShuffleMask))
return false;
return isOneUseSingleSourceMask(ShuffleMask, VF);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3ae7a893ca4e9e3..5848667728878df 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11813,7 +11813,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) ||
(VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) &&
- ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) {
+ ShuffleVectorInst::isReverseMask(ShuffleMask)) {
SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
DAG.getConstant(8, dl, MVT::i32));
@@ -25837,8 +25837,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SDLoc DL(Op);
unsigned MinSVESize = Subtarget.getMinSVEVectorSizeInBits();
unsigned MaxSVESize = Subtarget.getMaxSVEVectorSizeInBits();
- bool IsSingleOp =
- ShuffleVectorInst::isSingleSourceMask(ShuffleMask, ShuffleMask.size());
+ bool IsSingleOp = ShuffleVectorInst::isSingleSourceMask(ShuffleMask);
// Ignore two operands if no SVE2 or all index numbers couldn't
// be represented.
@@ -26022,8 +26021,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits();
unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits();
if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) {
- if (ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size()) &&
- Op2.isUndef()) {
+ if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) {
Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1);
return convertFromScalableVector(DAG, VT, Op);
}
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 8cca167e32ee791..52ce370bbacc642 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -8375,7 +8375,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
unsigned EltSize = VT.getScalarSizeInBits();
if (EltSize >= 32 ||
ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
- ShuffleVectorInst::isIdentityMask(M, M.size()) ||
+ ShuffleVectorInst::isIdentityMask(M) ||
isVREVMask(M, VT, 64) ||
isVREVMask(M, VT, 32) ||
isVREVMask(M, VT, 16))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index c06862d013f54ec..2ce5298c3c01510 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -4337,8 +4337,7 @@ static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
assert(VT.getVectorElementType() == MVT::i1);
- if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
- SVN->getMask().size()) ||
+ if (!ShuffleVectorInst::isReverseMask(SVN->getMask()) ||
!SVN->getOperand(1).isUndef())
return SDValue();
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index e947d4d1e8acdee..2f510a215d80bf2 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -322,10 +322,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
LT.second.getVectorElementType().getSizeInBits() ==
Tp->getElementType()->getPrimitiveSizeInBits() &&
LT.second.getVectorNumElements() <
- cast<FixedVectorType>(Tp)->getNumElements() &&
- divideCeil(Mask.size(),
- cast<FixedVectorType>(Tp)->getNumElements()) ==
- static_cast<unsigned>(*LT.first.getValue())) {
+ cast<FixedVectorType>(Tp)->getNumElements()) {
unsigned NumRegs = *LT.first.getValue();
unsigned VF = cast<FixedVectorType>(Tp)->getNumElements();
unsigned SubVF = PowerOf2Ceil(VF / NumRegs);
@@ -501,7 +498,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
InstructionCost Cost = MemCost;
for (unsigned Index : Indices) {
FixedVectorType *SubVecTy =
- FixedVectorType::get(FVTy->getElementType(), VF * Factor);
+ FixedVectorType::get(FVTy->getElementType(), VF);
auto Mask = createStrideMask(Index, Factor, VF);
InstructionCost ShuffleCost =
getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 5a6413653908ec8..d838d1f96e310d8 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1596,6 +1596,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
BaseTp->getElementType()->getPrimitiveSizeInBits() &&
LegalVT.getVectorNumElements() <
cast<FixedVectorType>(BaseTp)->getNumElements()) {
+
unsigned VecTySize = DL.getTypeStoreSize(BaseTp);
unsigned LegalVTSize = LegalVT.getStoreSize();
// Number of source vectors after legalization:
@@ -1620,10 +1621,6 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
// copy of the previous destination register (the cost is
// TTI::TCC_Basic). If the source register is just reused, the cost for
// this operation is 0.
- NumOfDests =
- getTypeLegalizationCost(
- FixedVectorType::get(BaseTp->getElementType(), Mask.size()))
- .first;
unsigned E = *NumOfDests.getValue();
unsigned NormalizedVF =
LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
@@ -1638,7 +1635,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
[this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
&Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
- if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) {
+ if (!ShuffleVectorInst::isIdentityMask(RegMask)) {
// Check if the previous register can be just copied to the next
// one.
if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index c8b58c51d4e6ecb..69d7593d5fc1e36 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2122,8 +2122,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) {
NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i];
// A select mask with undef elements might look like an identity mask.
- assert((ShuffleVectorInst::isSelectMask(NewMask, NumElts) ||
- ShuffleVectorInst::isIdentityMask(NewMask, NumElts)) &&
+ assert((ShuffleVectorInst::isSelectMask(NewMask) ||
+ ShuffleVectorInst::isIdentityMask(NewMask)) &&
"Unexpected shuffle mask");
return new ShuffleVectorInst(X, Y, NewMask);
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a010ffb3c9d7896..bd67b69e0b55d9e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3812,7 +3812,7 @@ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
inversePermutation(Order, MaskOrder);
}
reorderReuses(MaskOrder, Mask);
- if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) {
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder)) {
Order.clear();
return;
}
@@ -4333,7 +4333,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
static bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask,
unsigned Sz) {
ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
- if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
+ if (ShuffleVectorInst::isIdentityMask(FirstCluster))
return false;
for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
ArrayRef<int> Cluster = Mask.slice(I, Sz);
@@ -6646,24 +6646,9 @@ class BaseShuffleAnalysis {
bool IsStrict) {
int Limit = Mask.size();
int VF = VecTy->getNumElements();
- int Index = -1;
- if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit))
- return true;
- if (!IsStrict) {
- // Consider extract subvector starting from index 0.
- if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
- Index == 0)
- return true;
- // All VF-size submasks are identity (e.g.
- // <poison,poison,poison,poison,0,1,2,poison,poison,1,2,3> etc. for VF 4).
- if (Limit % VF == 0 && all_of(seq<int>(0, Limit / VF), [=](int Idx) {
- ArrayRef<int> Slice = Mask.slice(Idx * VF, VF);
- return all_of(Slice, [](int I) { return I == PoisonMaskElem; }) ||
- ShuffleVectorInst::isIdentityMask(Slice, VF);
- }))
- return true;
- }
- return false;
+ return (VF == Limit || !IsStrict) &&
+ all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
+ ShuffleVectorInst::isIdentityMask(Mask);
}
/// Tries to combine 2
diff erent masks into single one.
@@ -6733,8 +6718,7 @@ class BaseShuffleAnalysis {
if (isIdentityMask(Mask, SVTy, /*IsStrict=*/false)) {
if (!IdentityOp || !SinglePermute ||
(isIdentityMask(Mask, SVTy, /*IsStrict=*/true) &&
- !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask,
- IdentityMask.size()))) {
+ !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask))) {
IdentityOp = SV;
// Store current mask in the IdentityMask so later we did not lost
// this info if IdentityOp is selected as the best candidate for the
@@ -6804,7 +6788,7 @@ class BaseShuffleAnalysis {
}
if (auto *OpTy = dyn_cast<FixedVectorType>(Op->getType());
!OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) ||
- ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) {
+ ShuffleVectorInst::isZeroEltSplatMask(Mask)) {
if (IdentityOp) {
V = IdentityOp;
assert(Mask.size() == IdentityMask.size() &&
@@ -6820,7 +6804,7 @@ class BaseShuffleAnalysis {
/*IsStrict=*/true) ||
(Shuffle && Mask.size() == Shuffle->getShuffleMask().size() &&
Shuffle->isZeroEltSplat() &&
- ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())));
+ ShuffleVectorInst::isZeroEltSplatMask(Mask)));
}
V = Op;
return false;
@@ -6925,9 +6909,11 @@ class BaseShuffleAnalysis {
CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
}
}
- if (Op1 == Op2 &&
- (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) ||
- (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) &&
+ const int Limit = CombinedMask1.size() * 2;
+ if (Op1 == Op2 && Limit == 2 * VF &&
+ all_of(CombinedMask1, [=](int Idx) { return Idx < Limit; }) &&
+ (ShuffleVectorInst::isIdentityMask(CombinedMask1) ||
+ (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1) &&
isa<ShuffleVectorInst>(Op1) &&
cast<ShuffleVectorInst>(Op1)->getShuffleMask() ==
ArrayRef(CombinedMask1))))
@@ -6967,20 +6953,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
SmallPtrSetImpl<Value *> &CheckedExtracts;
constexpr static TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
- if (Ty->getScalarType()->isPointerTy()) {
- Constant *Res = ConstantExpr::getIntToPtr(
- ConstantInt::getAllOnesValue(
- IntegerType::get(Ty->getContext(),
- DL.getTypeStoreSizeInBits(Ty->getScalarType()))),
- Ty->getScalarType());
- if (auto *VTy = dyn_cast<VectorType>(Ty))
- Res = ConstantVector::getSplat(VTy->getElementCount(), Res);
- return Res;
- }
- return Constant::getAllOnesValue(Ty);
- }
-
InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) {
if ((!Root && allConstant(VL)) || all_of(VL, UndefValue::classof))
return TTI::TCC_Free;
@@ -7211,12 +7183,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const TargetTransformInfo &TTI;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
- int Index = -1;
+ int Limit = 2 * VF;
return Mask.empty() ||
(VF == Mask.size() &&
- ShuffleVectorInst::isIdentityMask(Mask, VF)) ||
- (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
- Index == 0);
+ all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
+ ShuffleVectorInst::isIdentityMask(Mask));
}
public:
@@ -7229,17 +7200,21 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return TTI.getShuffleCost(TTI::SK_PermuteTwoSrc,
- cast<VectorType>(V1->getType()), Mask);
+ return TTI.getShuffleCost(
+ TTI::SK_PermuteTwoSrc,
+ FixedVectorType::get(
+ cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
+ Mask);
}
InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask) const {
// Empty mask or identity mask are free.
- unsigned VF =
- cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
- if (isEmptyOrIdentity(Mask, VF))
+ if (isEmptyOrIdentity(Mask, Mask.size()))
return TTI::TCC_Free;
- return TTI.getShuffleCost(TTI::SK_PermuteSingleSrc,
- cast<VectorType>(V1->getType()), Mask);
+ return TTI.getShuffleCost(
+ TTI::SK_PermuteSingleSrc,
+ FixedVectorType::get(
+ cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
+ Mask);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
@@ -7256,119 +7231,47 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const PointerUnion<Value *, const TreeEntry *> &P2,
ArrayRef<int> Mask) {
ShuffleCostBuilder Builder(TTI);
- SmallVector<int> CommonMask(Mask.begin(), Mask.end());
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
- unsigned CommonVF = Mask.size();
- if (!V1 && !V2 && !P2.isNull()) {
- // Shuffle 2 entry nodes.
+ unsigned CommonVF = 0;
+ if (!V1) {
const TreeEntry *E = P1.get<const TreeEntry *>();
unsigned VF = E->getVectorFactor();
- const TreeEntry *E2 = P2.get<const TreeEntry *>();
- CommonVF = std::max(VF, E2->getVectorFactor());
- assert(all_of(Mask,
- [=](int Idx) {
- return Idx < 2 * static_cast<int>(CommonVF);
- }) &&
- "All elements in mask must be less than 2 * CommonVF.");
- if (E->Scalars.size() == E2->Scalars.size()) {
- SmallVector<int> EMask = E->getCommonMask();
- SmallVector<int> E2Mask = E2->getCommonMask();
- if (!EMask.empty() || !E2Mask.empty()) {
- for (int &Idx : CommonMask) {
- if (Idx == PoisonMaskElem)
- continue;
- if (Idx < static_cast<int>(CommonVF) && !EMask.empty())
- Idx = EMask[Idx];
- else if (Idx >= static_cast<int>(CommonVF))
- Idx = (E2Mask.empty() ? Idx - CommonVF : E2Mask[Idx - CommonVF]) +
- E->Scalars.size();
- }
+ if (V2) {
+ unsigned V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
+ if (V2VF != VF && V2VF == E->Scalars.size())
+ VF = E->Scalars.size();
+ } else if (!P2.isNull()) {
+ const TreeEntry *E2 = P2.get<const TreeEntry *>();
+ if (E->Scalars.size() == E2->Scalars.size())
+ CommonVF = VF = E->Scalars.size();
+ } else {
+ // P2 is empty, check that we have same node + reshuffle (if any).
+ if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
+ VF = E->Scalars.size();
+ SmallVector<int> CommonMask(Mask.begin(), Mask.end());
+ ::addMask(CommonMask, E->getCommonMask());
+ V1 = Constant::getNullValue(
+ FixedVectorType::get(E->Scalars.front()->getType(), VF));
+ return BaseShuffleAnalysis::createShuffle<InstructionCost>(
+ V1, nullptr, CommonMask, Builder);
}
- CommonVF = E->Scalars.size();
}
V1 = Constant::getNullValue(
- FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
- V2 = getAllOnesValue(
- *R.DL, FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
- } else if (!V1 && P2.isNull()) {
- // Shuffle single entry node.
- const TreeEntry *E = P1.get<const TreeEntry *>();
+ FixedVectorType::get(E->Scalars.front()->getType(), VF));
+ }
+ if (!V2 && !P2.isNull()) {
+ const TreeEntry *E = P2.get<const TreeEntry *>();
unsigned VF = E->getVectorFactor();
- CommonVF = VF;
- assert(
- all_of(Mask,
- [=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
- "All elements in mask must be less than CommonVF.");
- if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
- SmallVector<int> EMask = E->getCommonMask();
- assert(!EMask.empty() && "Expected non-empty common mask.");
- for (int &Idx : CommonMask) {
- if (Idx != PoisonMaskElem)
- Idx = EMask[Idx];
- }
+ unsigned V1VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ if (!CommonVF && V1VF == E->Scalars.size())
CommonVF = E->Scalars.size();
- }
- V1 = Constant::getNullValue(
- FixedVectorType::get(E->Scalars.front()->getType(), CommonVF));
- } else if (V1 && P2.isNull()) {
- // Shuffle single vector.
- CommonVF = cast<FixedVectorType>(V1->getType())->getNumElements();
- assert(
- all_of(Mask,
- [=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
- "All elements in mask must be less than CommonVF.");
- } else if (V1 && !V2) {
- // Shuffle vector and tree node.
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
- const TreeEntry *E2 = P2.get<const TreeEntry *>();
- CommonVF = std::max(VF, E2->getVectorFactor());
- assert(all_of(Mask,
- [=](int Idx) {
- return Idx < 2 * static_cast<int>(CommonVF);
- }) &&
- "All elements in mask must be less than 2 * CommonVF.");
- if (E2->Scalars.size() == VF && VF != CommonVF) {
- SmallVector<int> E2Mask = E2->getCommonMask();
- assert(!E2Mask.empty() && "Expected non-empty common mask.");
- for (int &Idx : CommonMask) {
- if (Idx == PoisonMaskElem)
- continue;
- if (Idx >= static_cast<int>(CommonVF))
- Idx = E2Mask[Idx - CommonVF] + VF;
- }
- CommonVF = VF;
- }
- V1 = Constant::getNullValue(
- FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF));
- V2 = getAllOnesValue(
- *R.DL,
- FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF));
- } else {
- assert(V1 && V2 && "Expected both vectors.");
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
- CommonVF =
- std::max(VF, cast<FixedVectorType>(V2->getType())->getNumElements());
- assert(all_of(Mask,
- [=](int Idx) {
- return Idx < 2 * static_cast<int>(CommonVF);
- }) &&
- "All elements in mask must be less than 2 * CommonVF.");
- if (V1->getType() != V2->getType()) {
- V1 = Constant::getNullValue(FixedVectorType::get(
- cast<FixedVectorType>(V1->getType())->getElementType(), CommonVF));
- V2 = getAllOnesValue(
- *R.DL, FixedVectorType::get(
- cast<FixedVectorType>(V1->getType())->getElementType(),
- CommonVF));
- }
- }
- InVectors.front() = Constant::getNullValue(FixedVectorType::get(
- cast<FixedVectorType>(V1->getType())->getElementType(),
- CommonMask.size()));
- if (InVectors.size() == 2)
- InVectors.pop_back();
- return BaseShuffleAnalysis::createShuffle<InstructionCost>(
- V1, V2, CommonMask, Builder);
+ if (CommonVF)
+ VF = CommonVF;
+ V2 = Constant::getNullValue(
+ FixedVectorType::get(E->Scalars.front()->getType(), VF));
+ }
+ return BaseShuffleAnalysis::createShuffle<InstructionCost>(V1, V2, Mask,
+ Builder);
}
public:
@@ -7512,7 +7415,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
return ConstantVector::getSplat(
ElementCount::getFixed(VL.size()),
- getAllOnesValue(*R.DL, VL.front()->getType()));
+ Constant::getNullValue(VL.front()->getType()));
}
/// Finalize emission of the shuffles.
InstructionCost
@@ -7521,22 +7424,31 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
IsFinalized = true;
if (Action) {
const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
- if (InVectors.size() == 2)
+ if (InVectors.size() == 2) {
Cost += createShuffle(Vec, InVectors.back(), CommonMask);
- else
+ InVectors.pop_back();
+ } else {
Cost += createShuffle(Vec, nullptr, CommonMask);
+ }
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (CommonMask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx;
assert(VF > 0 &&
"Expected vector length for the final value before action.");
- Value *V = Vec.get<Value *>();
+ Value *V = Vec.dyn_cast<Value *>();
+ if (!Vec.isNull() && !V)
+ V = Constant::getNullValue(FixedVectorType::get(
+ Vec.get<const TreeEntry *>()->Scalars.front()->getType(),
+ CommonMask.size()));
Action(V, CommonMask);
- InVectors.front() = V;
}
::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true);
if (CommonMask.empty())
return Cost;
+ int Limit = CommonMask.size() * 2;
+ if (all_of(CommonMask, [=](int Idx) { return Idx < Limit; }) &&
+ ShuffleVectorInst::isIdentityMask(CommonMask))
+ return Cost;
return Cost +
createShuffle(InVectors.front(),
InVectors.size() == 2 ? InVectors.back() : nullptr,
@@ -7712,7 +7624,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
if (NeedToShuffleReuses)
::addMask(Mask, E->ReuseShuffleIndices);
- if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
+ if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask))
CommonCost =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask);
assert((E->State == TreeEntry::Vectorize ||
@@ -8063,7 +7975,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
} else {
SmallVector<int> Mask;
inversePermutation(OpTE->ReorderIndices, Mask);
- if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
+ if (ShuffleVectorInst::isReverseMask(Mask))
CCH = TTI::CastContextHint::Reversed;
}
}
@@ -8922,7 +8834,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
unsigned VecVF = TE->getVectorFactor();
if (VF != VecVF &&
(any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); }) ||
- !ShuffleVectorInst::isIdentityMask(Mask, VF))) {
+ (all_of(Mask,
+ [VF](int Idx) { return Idx < 2 * static_cast<int>(VF); }) &&
+ !ShuffleVectorInst::isIdentityMask(Mask)))) {
SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
@@ -8941,23 +8855,19 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
// Calculate the cost of the reshuffled vectors, if any.
for (int I = 0, E = FirstUsers.size(); I < E; ++I) {
Value *Base = cast<Instruction>(FirstUsers[I].first)->getOperand(0);
+ unsigned VF = ShuffleMasks[I].begin()->second.size();
+ auto *FTy = FixedVectorType::get(
+ cast<VectorType>(FirstUsers[I].first->getType())->getElementType(), VF);
auto Vector = ShuffleMasks[I].takeVector();
- unsigned VF = 0;
- auto EstimateShufflesCost = [&](ArrayRef<int> Mask,
- ArrayRef<const TreeEntry *> TEs) {
+ auto &&EstimateShufflesCost = [this, FTy,
+ &Cost](ArrayRef<int> Mask,
+ ArrayRef<const TreeEntry *> TEs) {
assert((TEs.size() == 1 || TEs.size() == 2) &&
"Expected exactly 1 or 2 tree entries.");
if (TEs.size() == 1) {
- if (VF == 0)
- VF = TEs.front()->getVectorFactor();
- auto *FTy =
- FixedVectorType::get(TEs.back()->Scalars.front()->getType(), VF);
- if (!ShuffleVectorInst::isIdentityMask(Mask, VF) &&
- !all_of(enumerate(Mask), [=](const auto &Data) {
- return Data.value() == PoisonMaskElem ||
- (Data.index() < VF &&
- static_cast<int>(Data.index()) == Data.value());
- })) {
+ int Limit = 2 * Mask.size();
+ if (!all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) ||
+ !ShuffleVectorInst::isIdentityMask(Mask)) {
InstructionCost C =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FTy, Mask);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
@@ -8968,15 +8878,6 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
Cost += C;
}
} else {
- if (VF == 0) {
- if (TEs.front() &&
- TEs.front()->getVectorFactor() == TEs.back()->getVectorFactor())
- VF = TEs.front()->getVectorFactor();
- else
- VF = Mask.size();
- }
- auto *FTy =
- FixedVectorType::get(TEs.back()->Scalars.front()->getType(), VF);
InstructionCost C =
TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, FTy, Mask);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
@@ -8986,7 +8887,6 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
dbgs() << "SLP: Current total cost = " << Cost << "\n");
Cost += C;
}
- VF = Mask.size();
return TEs.back();
};
(void)performExtractsShuffleAction<const TreeEntry>(
@@ -9743,7 +9643,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
return V1;
unsigned VF = Mask.size();
unsigned LocalVF = cast<FixedVectorType>(V1->getType())->getNumElements();
- if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF))
+ if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask))
return V1;
Value *Vec = Builder.CreateShuffleVector(V1, Mask);
if (auto *I = dyn_cast<Instruction>(Vec)) {
@@ -10156,7 +10056,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
return false;
unsigned I =
*find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
- if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
+ int Sz = Mask.size();
+ if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) &&
+ ShuffleVectorInst::isIdentityMask(Mask))
std::iota(Mask.begin(), Mask.end(), 0);
else
std::fill(Mask.begin(), Mask.end(), I);
@@ -10400,11 +10302,11 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
(ExtractShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(ExtractMask, [&](int I) { return I >= EMSz; }) &&
- ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) ||
+ ShuffleVectorInst::isIdentityMask(ExtractMask)) ||
(GatherShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(Mask, [&](int I) { return I >= MSz; }) &&
- ShuffleVectorInst::isIdentityMask(Mask, MSz));
+ ShuffleVectorInst::isIdentityMask(Mask));
bool EnoughConstsForShuffle =
IsSingleShuffle &&
(none_of(GatheredScalars,
@@ -10697,7 +10599,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) &&
NumElts != NumScalars) {
if (IsFirstUndef.all()) {
- if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) {
+ if (!ShuffleVectorInst::isIdentityMask(InsertMask)) {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
@@ -11520,7 +11422,7 @@ Value *BoUpSLP::vectorizeTree(
// non-resizing mask.
if (Mask.size() != cast<FixedVectorType>(Vals.front()->getType())
->getNumElements() ||
- !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
+ !ShuffleVectorInst::isIdentityMask(Mask))
return CreateShuffle(Vals.front(), nullptr, Mask);
return Vals.front();
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 99f03cdcf648c02..d6196c84e8e2de2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -461,21 +461,44 @@ exit:
define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
; CHECK-LABEL: @combine_load_factor2_i64(
; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4
; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index eb3d395f4c6a6f9..c5981c44d96dd92 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -241,10 +241,21 @@ define void @tiny_vector_gather(ptr %a, ptr %v1, ptr %v2) {
; CHECK-LABEL: @tiny_vector_gather(
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[V1:%.*]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[V2:%.*]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[A:%.*]], align 16
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR1]], align 4
+; CHECK-NEXT: [[PTR2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR2]], align 8
+; CHECK-NEXT: [[PTR3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 3
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR3]], align 4
+; CHECK-NEXT: [[PTR4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR4]], align 16
+; CHECK-NEXT: [[PTR5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 5
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR5]], align 4
+; CHECK-NEXT: [[PTR6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 6
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR6]], align 8
+; CHECK-NEXT: [[PTR7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 7
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR7]], align 4
; CHECK-NEXT: ret void
;
%1 = load i32, ptr %v1, align 4
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index 20b8529b386324f..637c69204addfe2 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1024,141 +1024,71 @@ TEST(InstructionsTest, ShuffleMaskQueries) {
Constant *C7 = ConstantInt::get(Int32Ty, 7);
Constant *Identity = ConstantVector::get({C0, CU, C2, C3, C4});
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
- Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
- Identity,
- cast<FixedVectorType>(Identity->getType())
- ->getNumElements())); // identity is distinguished from select
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
- Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- Identity, cast<FixedVectorType>(Identity->getType())
- ->getNumElements())); // identity is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
- Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
- Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(Identity));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Identity)); // identity is distinguished from select
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Identity));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Identity)); // identity is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Identity));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Identity));
Constant *Select = ConstantVector::get({CU, C1, C5});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
- Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
-
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Select));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(Select));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Select));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Select));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Select));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Select));
+
Constant *Reverse = ConstantVector::get({C3, C2, C1, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())
- ->getNumElements())); // reverse is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
- Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Reverse));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Reverse));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(Reverse));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Reverse)); // reverse is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Reverse));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Reverse));
Constant *SingleSource = ConstantVector::get({C2, C2, C0, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
- SingleSource,
- cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(SingleSource));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(SingleSource));
Constant *ZeroEltSplat = ConstantVector::get({C0, C0, CU, C0});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
- ZeroEltSplat,
- cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
- ZeroEltSplat,
- cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
- ZeroEltSplat,
- cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- ZeroEltSplat, cast<FixedVectorType>(ZeroEltSplat->getType())
- ->getNumElements())); // 0-splat is always single source
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
- ZeroEltSplat,
- cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
- ZeroEltSplat,
- cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(ZeroEltSplat));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(ZeroEltSplat));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(ZeroEltSplat));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ZeroEltSplat)); // 0-splat is always single source
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ZeroEltSplat));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(ZeroEltSplat));
Constant *Transpose = ConstantVector::get({C0, C4, C2, C6});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
- Transpose,
- cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Transpose));
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(Transpose));
// More tests to make sure the logic is/stays correct...
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
- ConstantVector::get({CU, C1, CU, C3}), 4));
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
- ConstantVector::get({C4, CU, C6, CU}), 4));
-
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
- ConstantVector::get({C4, C1, C6, CU}), 4));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
- ConstantVector::get({CU, C1, C6, C3}), 4));
-
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
- ConstantVector::get({C7, C6, CU, C4}), 4));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
- ConstantVector::get({C3, CU, C1, CU}), 4));
-
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- ConstantVector::get({C7, C5, CU, C7}), 4));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
- ConstantVector::get({C3, C0, CU, C3}), 4));
-
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
- ConstantVector::get({C4, CU, CU, C4}), 4));
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
- ConstantVector::get({CU, C0, CU, C0}), 4));
-
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
- ConstantVector::get({C1, C5, C3, C7}), 4));
- EXPECT_TRUE(
- ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}), 2));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({CU, C1, CU, C3})));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({C4, CU, C6, CU})));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({C4, C1, C6, CU})));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({CU, C1, C6, C3})));
+
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C7, C6, CU, C4})));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C3, CU, C1, CU})));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C7, C5, CU, C7})));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C3, C0, CU, C3})));
+
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({C4, CU, CU, C4})));
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({CU, C0, CU, C0})));
+
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C5, C3, C7})));
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3})));
// Nothing special about the values here - just re-using inputs to reduce code.
Constant *V0 = ConstantVector::get({C0, C1, C2, C3});
diff --git a/llvm/unittests/IR/ShuffleVectorInstTest.cpp b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
index 5d840ec8d672047..ba1b9807cffb43b 100644
--- a/llvm/unittests/IR/ShuffleVectorInstTest.cpp
+++ b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
@@ -14,84 +14,51 @@ using namespace llvm;
namespace {
TEST(ShuffleVectorInst, isIdentityMask) {
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 4));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 5));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 3));
-
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 6));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 5));
-
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}));
}
TEST(ShuffleVectorInst, isSelectMask) {
- ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 4));
-
- ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 5));
+ ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}));
- ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}));
}
TEST(ShuffleVectorInst, isReverseMask) {
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 4));
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 5));
-
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}));
}
TEST(ShuffleVectorInst, isZeroEltSplatMask) {
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 4));
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 4));
-
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 5));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}));
}
TEST(ShuffleVectorInst, isTransposeMask) {
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 4));
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 3));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 5));
-
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}));
}
TEST(ShuffleVectorInst, isSpliceMask) {
int Index;
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 4, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, Index));
ASSERT_EQ(0, Index);
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 7, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, Index));
ASSERT_EQ(1, Index);
- ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 3, Index));
- ASSERT_FALSE(
- ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 6, Index));
- ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 5, Index));
- ASSERT_FALSE(
- ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 8, Index));
-
- ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, 4, Index));
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, Index));
}
TEST(ShuffleVectorInst, isExtractSubvectorMask) {
More information about the llvm-commits
mailing list