[llvm] 1ee6ec2 - Remove "mask" operand from shufflevector.
Eli Friedman via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 31 13:09:33 PDT 2020
Author: Eli Friedman
Date: 2020-03-31T13:08:59-07:00
New Revision: 1ee6ec2bf370fbd1d93f34c8b56741a9d3f22ed2
URL: https://github.com/llvm/llvm-project/commit/1ee6ec2bf370fbd1d93f34c8b56741a9d3f22ed2
DIFF: https://github.com/llvm/llvm-project/commit/1ee6ec2bf370fbd1d93f34c8b56741a9d3f22ed2.diff
LOG: Remove "mask" operand from shufflevector.
Instead, represent the mask as out-of-line data in the instruction. This
should be more efficient in the places that currently use
getShuffleVector(), and paves the way for further changes to add new
shuffles for scalable vectors.
This doesn't change the syntax in textual IR. And I don't currently plan
to change the bitcode encoding in this patch, although we'll probably
need to do something once we extend shufflevector for scalable types.
I expect that once this is finished, we can then replace the raw "mask"
with something more appropriate for scalable vectors. Not sure exactly
what this looks like at the moment, but there are a few different ways
we could handle it. Maybe we could try to describe specific shuffles.
Or maybe we could define it in terms of a function to convert a fixed-length
array into an appropriate scalable vector, using a "step", or something
like that.
Differential Revision: https://reviews.llvm.org/D72467
Added:
Modified:
clang/lib/CodeGen/CGExpr.cpp
llvm/include/llvm/ADT/ArrayRef.h
llvm/include/llvm/Analysis/ConstantFolding.h
llvm/include/llvm/Analysis/InstructionSimplify.h
llvm/include/llvm/Analysis/TargetFolder.h
llvm/include/llvm/IR/ConstantFolder.h
llvm/include/llvm/IR/Constants.h
llvm/include/llvm/IR/IRBuilder.h
llvm/include/llvm/IR/IRBuilderFolder.h
llvm/include/llvm/IR/Instructions.h
llvm/include/llvm/IR/NoFolder.h
llvm/include/llvm/IR/PatternMatch.h
llvm/lib/Analysis/ConstantFolding.cpp
llvm/lib/Analysis/InstructionSimplify.cpp
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Analysis/ValueTracking.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
llvm/lib/IR/AsmWriter.cpp
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/ConstantFold.cpp
llvm/lib/IR/ConstantFold.h
llvm/lib/IR/Constants.cpp
llvm/lib/IR/ConstantsContext.h
llvm/lib/IR/Core.cpp
llvm/lib/IR/Instruction.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/ARM/MVETailPredication.cpp
llvm/lib/Target/X86/X86PartialReduction.cpp
llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
llvm/lib/Transforms/Scalar/GVN.cpp
llvm/lib/Transforms/Scalar/GVNSink.cpp
llvm/lib/Transforms/Scalar/NewGVN.cpp
llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/unittests/IR/PatternMatch.cpp
Removed:
################################################################################
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 671ada019cec..b53b77a819b4 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1675,7 +1675,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
// Shuffle vector to get vec3.
V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
- {0, 1, 2}, "extractVec");
+ ArrayRef<int>{0, 1, 2}, "extractVec");
return EmitFromMemory(V, Ty);
}
}
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 5cba73e67f04..d11f8b5b44e9 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -532,11 +532,21 @@ namespace llvm {
return LHS.equals(RHS);
}
- template<typename T>
+ template <typename T>
+ inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return ArrayRef<T>(LHS).equals(RHS);
+ }
+
+ template <typename T>
inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
+ template <typename T>
+ inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return !(LHS == RHS);
+ }
+
/// @}
template <typename T> hash_code hash_value(ArrayRef<T> S) {
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 90dc14b0ed9e..68aa1948ab88 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -119,10 +119,11 @@ Constant *ConstantFoldInsertElementInstruction(Constant *Val,
Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
/// Attempt to constant fold a shufflevector instruction with the
-/// specified operands and indices. The constant result is returned if
-/// successful; if not, null is returned.
+/// specified operands and mask. See class ShuffleVectorInst for a description
+/// of the mask representation. The constant result is returned if successful;
+/// if not, null is returned.
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
/// produce if it is constant and determinable. If this is not determinable,
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index b661caee6848..7a9a1a81555b 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -230,7 +230,8 @@ Value *SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
const SimplifyQuery &Q);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+/// See class ShuffleVectorInst for a description of the mask representation.
+Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q);
//=== Helper functions for higher up the class hierarchy.
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index 7277a1230e28..b23316ac3d9b 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -259,7 +259,7 @@ class TargetFolder final : public IRBuilderFolder {
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
}
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index 32939e79cd1d..da4a18e3c181 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -265,7 +265,7 @@ class ConstantFolder final : public IRBuilderFolder {
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return ConstantExpr::getShuffleVector(V1, V2, Mask);
}
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index a345795ff54a..868b038b055d 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -1206,7 +1206,8 @@ class ConstantExpr : public Constant {
Type *OnlyIfReducedTy = nullptr);
static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
Type *OnlyIfReducedTy = nullptr);
- static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
+ static Constant *getShuffleVector(Constant *V1, Constant *V2,
+ ArrayRef<int> Mask,
Type *OnlyIfReducedTy = nullptr);
static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
Type *OnlyIfReducedTy = nullptr);
@@ -1225,6 +1226,16 @@ class ConstantExpr : public Constant {
/// expression and return the list of indices.
ArrayRef<unsigned> getIndices() const;
+ /// Assert that this is a shufflevector and return the mask. See class
+ /// ShuffleVectorInst for a description of the mask representation.
+ ArrayRef<int> getShuffleMask() const;
+
+ /// Assert that this is a shufflevector and return the mask.
+ ///
+ /// TODO: This is a temporary hack until we update the bitcode format for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const;
+
/// Return a string representation for an opcode.
const char *getOpcodeName() const;
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 3b0ca0c1ae5c..d4449e196000 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2617,17 +2617,25 @@ class IRBuilderBase {
Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
const Twine &Name = "") {
- if (auto *V1C = dyn_cast<Constant>(V1))
- if (auto *V2C = dyn_cast<Constant>(V2))
- if (auto *MC = dyn_cast<Constant>(Mask))
- return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
- return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
+ return CreateShuffleVector(V1, V2, IntMask, Name);
}
- Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
const Twine &Name = "") {
- Value *Mask = ConstantDataVector::get(Context, IntMask);
- return CreateShuffleVector(V1, V2, Mask, Name);
+ SmallVector<int, 16> IntMask;
+ IntMask.assign(Mask.begin(), Mask.end());
+ return CreateShuffleVector(V1, V2, IntMask, Name);
+ }
+
+ /// See class ShuffleVectorInst for a description of the mask representation.
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name = "") {
+ if (auto *V1C = dyn_cast<Constant>(V1))
+ if (auto *V2C = dyn_cast<Constant>(V2))
+ return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
+ return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}
Value *CreateExtractValue(Value *Agg,
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index a77a5213dcea..e781e8e094af 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -129,7 +129,7 @@ class IRBuilderFolder {
virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt,
Constant *Idx) const = 0;
virtual Value *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const = 0;
+ ArrayRef<int> Mask) const = 0;
virtual Value *CreateExtractValue(Constant *Agg,
ArrayRef<unsigned> IdxList) const = 0;
virtual Value *CreateInsertValue(Constant *Agg, Constant *Val,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 713624d13bef..857b7374ac47 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -1988,10 +1988,22 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
// ShuffleVectorInst Class
//===----------------------------------------------------------------------===//
+constexpr int UndefMaskElem = -1;
+
/// This instruction constructs a fixed permutation of two
/// input vectors.
///
+/// For each element of the result vector, the shuffle mask selects an element
+/// from one of the input vectors to copy to the result. Non-negative elements
+/// in the mask represent an index into the concatenated pair of input vectors.
+/// UndefMaskElem (-1) specifies that the result element is undefined.
+///
+/// For scalable vectors, all the elements of the mask must be 0 or -1. This
+/// requirement may be relaxed in the future.
class ShuffleVectorInst : public Instruction {
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
@@ -2004,13 +2016,15 @@ class ShuffleVectorInst : public Instruction {
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr = "",
+ Instruction *InsertBefor = nullptr);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr, BasicBlock *InsertAtEnd);
- // allocate space for exactly three operands
- void *operator new(size_t s) {
- return User::operator new(s, 3);
- }
+ void *operator new(size_t s) { return User::operator new(s, 2); }
- /// Swap the first 2 operands and adjust the mask to preserve the semantics
+ /// Swap the operands and adjust the mask to preserve the semantics
/// of the instruction.
void commute();
@@ -2018,6 +2032,8 @@ class ShuffleVectorInst : public Instruction {
/// formed with the specified operands.
static bool isValidOperands(const Value *V1, const Value *V2,
const Value *Mask);
+ static bool isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask);
/// Overload to return most specific vector type.
///
@@ -2028,36 +2044,33 @@ class ShuffleVectorInst : public Instruction {
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
- Constant *getMask() const {
- return cast<Constant>(getOperand(2));
- }
-
- /// Return the shuffle mask value for the specified element of the mask.
- /// Return -1 if the element is undef.
- static int getMaskValue(const Constant *Mask, unsigned Elt);
-
/// Return the shuffle mask value of this instruction for the given element
- /// index. Return -1 if the element is undef.
- int getMaskValue(unsigned Elt) const {
- return getMaskValue(getMask(), Elt);
- }
+ /// index. Return UndefMaskElem if the element is undef.
+ int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
/// Convert the input shuffle mask operand to a vector of integers. Undefined
- /// elements of the mask are returned as -1.
+ /// elements of the mask are returned as UndefMaskElem.
static void getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result);
/// Return the mask for this instruction as a vector of integers. Undefined
- /// elements of the mask are returned as -1.
+ /// elements of the mask are returned as UndefMaskElem.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
- return getShuffleMask(getMask(), Result);
+ Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
- SmallVector<int, 16> getShuffleMask() const {
- SmallVector<int, 16> Mask;
- getShuffleMask(Mask);
- return Mask;
- }
+ /// Return the mask for this instruction, for use in bitcode.
+ ///
+ /// TODO: This is temporary until we decide a new bitcode encoding for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
+
+ static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+ Type *ResultTy);
+
+ void setShuffleMask(ArrayRef<int> Mask);
+
+ ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
/// Return true if this shuffle returns a vector with a
diff erent number of
/// elements than its source vectors.
@@ -2065,7 +2078,7 @@ class ShuffleVectorInst : public Instruction {
/// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
bool changesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts != NumMaskElts;
}
@@ -2074,7 +2087,7 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
bool increasesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts < NumMaskElts;
}
@@ -2095,7 +2108,7 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(getMask());
+ return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2116,7 +2129,7 @@ class ShuffleVectorInst : public Instruction {
/// from its input vectors.
/// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
bool isIdentity() const {
- return !changesLength() && isIdentityMask(getShuffleMask());
+ return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2157,7 +2170,7 @@ class ShuffleVectorInst : public Instruction {
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(getMask());
+ return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
@@ -2177,7 +2190,7 @@ class ShuffleVectorInst : public Instruction {
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(getMask());
+ return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
@@ -2199,7 +2212,7 @@ class ShuffleVectorInst : public Instruction {
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(getMask());
+ return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2248,7 +2261,7 @@ class ShuffleVectorInst : public Instruction {
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(getMask());
+ return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is an extract subvector mask.
@@ -2267,7 +2280,7 @@ class ShuffleVectorInst : public Instruction {
/// Return true if this shuffle mask is an extract subvector mask.
bool isExtractSubvectorMask(int &Index) const {
int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
- return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
+ return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
}
/// Change values in a shuffle permute mask assuming the two vector operands
@@ -2293,9 +2306,8 @@ class ShuffleVectorInst : public Instruction {
};
template <>
-struct OperandTraits<ShuffleVectorInst> :
- public FixedNumOperandTraits<ShuffleVectorInst, 3> {
-};
+struct OperandTraits<ShuffleVectorInst>
+ : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 1e35cfe096eb..dcffa6b2f9da 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -300,7 +300,7 @@ class NoFolder final : public IRBuilderFolder {
}
Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return new ShuffleVectorInst(V1, V2, Mask);
}
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 03b92e524ad5..30c203dedf67 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -50,6 +50,10 @@ template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
return const_cast<Pattern &>(P).match(V);
}
+template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
+ return const_cast<Pattern &>(P).match(Mask);
+}
+
template <typename SubPattern_t> struct OneUse_match {
SubPattern_t SubPattern;
@@ -1350,12 +1354,69 @@ m_ExtractElement(const Val_t &Val, const Idx_t &Idx) {
return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
}
-/// Matches ShuffleVectorInst.
+/// Matches shuffle.
+template <typename T0, typename T1, typename T2> struct Shuffle_match {
+ T0 Op1;
+ T1 Op2;
+ T2 Mask;
+
+ Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
+ : Op1(Op1), Op2(Op2), Mask(Mask) {}
+
+ template <typename OpTy> bool match(OpTy *V) {
+ if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
+ return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
+ Mask.match(I->getShuffleMask());
+ }
+ return false;
+ }
+};
+
+struct m_Mask {
+ ArrayRef<int> &MaskRef;
+ m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ MaskRef = Mask;
+ return true;
+ }
+};
+
+struct m_ZeroMask {
+ bool match(ArrayRef<int> Mask) {
+ return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
+ }
+};
+
+struct m_SpecificMask {
+ ArrayRef<int> &MaskRef;
+ m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
+};
+
+struct m_SplatOrUndefMask {
+ int &SplatIndex;
+ m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
+ bool match(ArrayRef<int> Mask) {
+ auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
+ if (First == Mask.end())
+ return false;
+ SplatIndex = *First;
+ return all_of(Mask,
+ [First](int Elem) { return Elem == *First || Elem == -1; });
+ }
+};
+
+/// Matches ShuffleVectorInst independently of mask value.
+template <typename V1_t, typename V2_t>
+inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
+ return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
+}
+
template <typename V1_t, typename V2_t, typename Mask_t>
-inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
-m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
- return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
- m);
+inline Shuffle_match<V1_t, V2_t, Mask_t>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
+ return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
}
/// Matches LoadInst.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 5efebe2937ed..721ba735e9db 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1066,7 +1066,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
case Instruction::InsertElement:
return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
+ return ConstantExpr::getShuffleVector(
+ Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
}
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index e62ddf793a13..086ff7433544 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4439,36 +4439,31 @@ static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,
return RootVec;
}
-static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
- Type *RetTy, const SimplifyQuery &Q,
+static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+ ArrayRef<int> Mask, Type *RetTy,
+ const SimplifyQuery &Q,
unsigned MaxRecurse) {
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
return UndefValue::get(RetTy);
Type *InVecTy = Op0->getType();
- ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+ unsigned MaskNumElts = Mask.size();
ElementCount InVecEltCount = InVecTy->getVectorElementCount();
- assert(MaskEltCount.Scalable == InVecEltCount.Scalable &&
- "vscale mismatch between input vector and mask");
-
- bool Scalable = MaskEltCount.Scalable;
+ bool Scalable = InVecEltCount.Scalable;
SmallVector<int, 32> Indices;
- if (!Scalable) {
- ShuffleVectorInst::getShuffleMask(Mask, Indices);
- assert(MaskEltCount.Min == Indices.size() &&
- "Size of Indices not same as number of mask elements?");
- }
+ Indices.assign(Mask.begin(), Mask.end());
+ // Canonicalization: If mask does not select elements from an input vector,
+ // replace that input vector with undef.
if (!Scalable) {
- // Canonicalization: If mask does not select elements from an input vector,
- // replace that input vector with undef.
bool MaskSelects0 = false, MaskSelects1 = false;
- for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+ unsigned InVecNumElts = InVecEltCount.Min;
+ for (unsigned i = 0; i != MaskNumElts; ++i) {
if (Indices[i] == -1)
continue;
- if ((unsigned)Indices[i] < InVecEltCount.Min)
+ if ((unsigned)Indices[i] < InVecNumElts)
MaskSelects0 = true;
else
MaskSelects1 = true;
@@ -4514,8 +4509,8 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
// Shuffle mask undefs become undefined constant result elements.
- SmallVector<Constant *, 16> VecC(MaskEltCount.Min, C);
- for (unsigned i = 0; i != MaskEltCount.Min; ++i)
+ SmallVector<Constant *, 16> VecC(MaskNumElts, C);
+ for (unsigned i = 0; i != MaskNumElts; ++i)
if (Indices[i] == -1)
VecC[i] = UndefValue::get(C->getType());
return ConstantVector::get(VecC);
@@ -4526,7 +4521,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
// value type is same as the input vectors' type.
if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
- OpShuf->getMask()->getSplatValue())
+ is_splat(OpShuf->getShuffleMask()))
return Op0;
// All remaining transformation depend on the value of the mask, which is
@@ -4545,7 +4540,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
// shuffle. This handles simple identity shuffles as well as chains of
// shuffles that may widen/narrow and/or move elements across lanes and back.
Value *RootVec = nullptr;
- for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+ for (unsigned i = 0; i != MaskNumElts; ++i) {
// Note that recursion is limited for each vector element, so if any element
// exceeds the limit, this will fail to simplify.
RootVec =
@@ -4559,8 +4554,9 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
}
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
- Type *RetTy, const SimplifyQuery &Q) {
+Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+ ArrayRef<int> Mask, Type *RetTy,
+ const SimplifyQuery &Q) {
return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
}
@@ -5523,8 +5519,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ,
}
case Instruction::ShuffleVector: {
auto *SVI = cast<ShuffleVectorInst>(I);
- Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getMask(), SVI->getType(), Q);
+ Result =
+ SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
+ SVI->getShuffleMask(), SVI->getType(), Q);
break;
}
case Instruction::PHI:
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 89b1783a9a00..2ae44caaaa32 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -888,7 +888,7 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
- SmallVector<int, 16> ActualMask = SI->getShuffleMask();
+ ArrayRef<int> ActualMask = SI->getShuffleMask();
return Mask == ActualMask;
}
@@ -1153,7 +1153,7 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
// Fill the rest of the mask with -1 for undef.
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
- SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
+ ArrayRef<int> Mask = Shuffle->getShuffleMask();
if (ShuffleMask != Mask)
return RK_None;
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 007513020769..6a4d596a3b76 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -168,16 +168,16 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
APInt &DemandedLHS, APInt &DemandedRHS) {
// The length of scalable vectors is unknown at compile time, thus we
// cannot check their values
- if (Shuf->getMask()->getType()->getVectorElementCount().Scalable)
+ if (Shuf->getType()->getVectorElementCount().Scalable)
return false;
int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
- int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+ int NumMaskElts = Shuf->getType()->getVectorNumElements();
DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts);
if (DemandedElts.isNullValue())
return true;
// Simple case of a shuffle with zeroinitializer.
- if (isa<ConstantAggregateZero>(Shuf->getMask())) {
+ if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) {
DemandedLHS.setBit(0);
return true;
}
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 532518acc6ad..c8ca2052919c 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -339,9 +339,9 @@ const llvm::Value *llvm::getSplatValue(const Value *V) {
// shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...>
Value *Splat;
- if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
- m_ZeroInt()),
- m_Value(), m_ZeroInt())))
+ if (match(V, m_ShuffleVector(
+ m_InsertElement(m_Value(), m_Value(Splat), m_ZeroInt()),
+ m_Value(), m_ZeroMask())))
return Splat;
return nullptr;
@@ -366,7 +366,7 @@ bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) {
if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
// FIXME: We can safely allow undefs here. If Index was specified, we will
// check that the mask elt is defined at the required index.
- if (!Shuf->getMask()->getSplatValue())
+ if (!is_splat(Shuf->getShuffleMask()))
return false;
// Match any index.
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index d219f607410b..ebb172048646 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -3636,8 +3636,9 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
return Error(ID.Loc, "expected three operands to shufflevector");
if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2]))
return Error(ID.Loc, "invalid operands to shufflevector");
- ID.ConstantVal =
- ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]);
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Elts[2]), Mask);
+ ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask);
} else if (Opc == Instruction::ExtractElement) {
if (Elts.size() != 2)
return Error(ID.Loc, "expected two operands to extractelement");
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index de2a6b47e6e2..16d3b79598f1 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2337,6 +2337,15 @@ Error BitcodeReader::parseConstants() {
Type *CurFullTy = Type::getInt32Ty(Context);
unsigned NextCstNo = ValueList.size();
+ struct DelayedShufTy {
+ VectorType *OpTy;
+ VectorType *RTy;
+ Type *CurFullTy;
+ uint64_t Op0Idx;
+ uint64_t Op1Idx;
+ uint64_t Op2Idx;
+ };
+ std::vector<DelayedShufTy> DelayedShuffles;
while (true) {
Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
if (!MaybeEntry)
@@ -2353,6 +2362,29 @@ Error BitcodeReader::parseConstants() {
// Once all the constants have been read, go through and resolve forward
// references.
+ //
+ // We have to treat shuffles specially because they don't have three
+ // operands anymore. We need to convert the shuffle mask into an array,
+ // and we can't convert a forward reference.
+ for (auto &DelayedShuffle : DelayedShuffles) {
+ VectorType *OpTy = DelayedShuffle.OpTy;
+ VectorType *RTy = DelayedShuffle.RTy;
+ uint64_t Op0Idx = DelayedShuffle.Op0Idx;
+ uint64_t Op1Idx = DelayedShuffle.Op1Idx;
+ uint64_t Op2Idx = DelayedShuffle.Op2Idx;
+ Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
+ Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
+ Type *ShufTy =
+ VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount());
+ Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy);
+ if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2))
+ return error("Invalid shufflevector operands");
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(Op2, Mask);
+ Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
+ ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy);
+ ++NextCstNo;
+ }
ValueList.resolveConstantForwardRefs();
return Error::success();
case BitstreamEntry::Record:
@@ -2694,13 +2726,9 @@ Error BitcodeReader::parseConstants() {
VectorType *OpTy = dyn_cast<VectorType>(CurTy);
if (Record.size() < 3 || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- OpTy->getElementCount());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back(
+ {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]});
+ continue;
}
case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
VectorType *RTy = dyn_cast<VectorType>(CurTy);
@@ -2708,13 +2736,9 @@ Error BitcodeReader::parseConstants() {
dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
if (Record.size() < 4 || !RTy || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- RTy->getElementCount());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back(
+ {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]});
+ continue;
}
case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred]
if (Record.size() < 4)
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index a27eada55d7e..e9ad00d6d4ba 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2515,7 +2515,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
}
Record.push_back(VE.getValueID(C->getOperand(0)));
Record.push_back(VE.getValueID(C->getOperand(1)));
- Record.push_back(VE.getValueID(C->getOperand(2)));
+ Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
break;
case Instruction::ICmp:
case Instruction::FCmp:
@@ -2702,7 +2702,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
pushValueAndType(I.getOperand(0), InstID, Vals);
pushValue(I.getOperand(1), InstID, Vals);
- pushValue(I.getOperand(2), InstID, Vals);
+ pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
+ Vals);
break;
case Instruction::ICmp:
case Instruction::FCmp: {
diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
index f59c906c7b75..7419c4687fac 100644
--- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -88,11 +88,16 @@ static void orderValue(const Value *V, OrderMap &OM) {
if (OM.lookup(V).first)
return;
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands() && !isa<GlobalValue>(C))
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands() && !isa<GlobalValue>(C)) {
for (const Value *Op : C->operands())
if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op))
orderValue(Op, OM);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ orderValue(CE->getShuffleMaskForBitcode(), OM);
+ }
+ }
// Note: we cannot cache this lookup above, since inserting into the map
// changes the map's size, and thus affects the other IDs.
@@ -155,11 +160,14 @@ static OrderMap orderModule(const Module &M) {
for (const Argument &A : F.args())
orderValue(&A, OM);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
isa<InlineAsm>(*Op))
orderValue(Op, OM);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ orderValue(SVI->getShuffleMaskForBitcode(), OM);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
orderValue(&I, OM);
@@ -250,11 +258,17 @@ static void predictValueUseListOrder(const Value *V, const Function *F,
predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack);
// Recursive descent into constants.
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands()) // Visit GlobalValues.
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands()) { // Visit GlobalValues.
for (const Value *Op : C->operands())
if (isa<Constant>(Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, F, OM, Stack);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM,
+ Stack);
+ }
+ }
}
static UseListOrderStack predictUseListOrder(const Module &M) {
@@ -279,10 +293,14 @@ static UseListOrderStack predictUseListOrder(const Module &M) {
for (const Argument &A : F.args())
predictValueUseListOrder(&A, &F, OM, Stack);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, &F, OM, Stack);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM,
+ Stack);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
predictValueUseListOrder(&I, &F, OM, Stack);
@@ -413,6 +431,8 @@ ValueEnumerator::ValueEnumerator(const Module &M,
EnumerateMetadata(&F, MD->getMetadata());
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
EnumerateType(I.getType());
if (const auto *Call = dyn_cast<CallBase>(&I))
EnumerateAttributes(Call->getAttributes());
@@ -836,6 +856,9 @@ void ValueEnumerator::EnumerateValue(const Value *V) {
I != E; ++I)
if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
EnumerateValue(*I);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateValue(CE->getShuffleMaskForBitcode());
// Finally, add the value. Doing this could make the ValueID reference be
// dangling, don't reuse it.
@@ -913,6 +936,9 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) {
EnumerateOperandType(Op);
}
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateOperandType(CE->getShuffleMaskForBitcode());
}
void ValueEnumerator::EnumerateAttributes(AttributeList PAL) {
@@ -958,11 +984,14 @@ void ValueEnumerator::incorporateFunction(const Function &F) {
// Add all function-level constants to the value table.
for (const BasicBlock &BB : F) {
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Use &OI : I.operands()) {
if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI))
EnumerateValue(OI);
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateValue(SVI->getShuffleMaskForBitcode());
+ }
BasicBlocks.push_back(&BB);
ValueMap[&BB] = BasicBlocks.size();
}
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 4a416c3c8233..1654eba6bc57 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6257,7 +6257,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
}
static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
- SmallVector<int, 16> Mask(SVI->getShuffleMask());
+ ArrayRef<int> Mask(SVI->getShuffleMask());
int SplatElem = -1;
for (unsigned i = 0; i < Mask.size(); ++i) {
if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
@@ -6307,7 +6307,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
assert(InsertPt != UserBB->end());
InsertedShuffle =
new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getOperand(2), "", &*InsertPt);
+ SVI->getShuffleMask(), "", &*InsertPt);
InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
}
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 58888c4dadc7..8135ce86cd22 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1973,8 +1973,11 @@ bool IRTranslator::translateExtractElement(const User &U,
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
+ ArrayRef<int> Mask;
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(U).getShuffleMask();
ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
MIRBuilder
.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 26ed6e22af94..ed24e004f908 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3437,14 +3437,19 @@ void SelectionDAGBuilder::visitExtractElement(const User &I) {
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
- Constant *MaskV = cast<Constant>(I.getOperand(2));
+ ArrayRef<int> Mask;
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();
unsigned SrcNumElts = SrcVT.getVectorNumElements();
- if (MaskV->isNullValue() && VT.isScalableVector()) {
+ if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
+ VT.isScalableVector()) {
// Canonical splat form of first element of first input vector.
SDValue FirstElt =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
@@ -3458,8 +3463,6 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MaskV, Mask);
unsigned MaskNumElts = Mask.size();
if (SrcNumElts == MaskNumElts) {
diff --git a/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
index 51f31d3d5d8f..4a5fb1feb18f 100644
--- a/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ b/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -1868,7 +1868,6 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
- GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
GenericValue Dest;
// There is no need to check types of src1 and src2, because the compiled
@@ -1878,7 +1877,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
Type *TyContained = Ty->getElementType();
unsigned src1Size = (unsigned)Src1.AggregateVal.size();
unsigned src2Size = (unsigned)Src2.AggregateVal.size();
- unsigned src3Size = (unsigned)Src3.AggregateVal.size();
+ unsigned src3Size = I.getShuffleMask().size();
Dest.AggregateVal.resize(src3Size);
@@ -1888,7 +1887,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::IntegerTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal;
else if(j < src1Size + src2Size)
@@ -1904,7 +1903,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::FloatTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal;
else if(j < src1Size + src2Size)
@@ -1915,7 +1914,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::DoubleTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal;
else if(j < src1Size + src2Size)
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 733fc10c2ee3..45f393f8b8e7 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -462,6 +462,33 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) {
isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
}
+static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
+ Out << ", <";
+ if (Ty->getVectorIsScalable())
+ Out << "vscale x ";
+ Out << Mask.size() << " x i32> ";
+ bool FirstElt = true;
+ if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
+ Out << "zeroinitializer";
+ } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ Out << "undef";
+ } else {
+ Out << "<";
+ for (int Elt : Mask) {
+ if (FirstElt)
+ FirstElt = false;
+ else
+ Out << ", ";
+ Out << "i32 ";
+ if (Elt == UndefMaskElem)
+ Out << "undef";
+ else
+ Out << Elt;
+ }
+ Out << ">";
+ }
+}
+
namespace {
class TypePrinting {
@@ -1547,6 +1574,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
TypePrinter.print(CE->getType(), Out);
}
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
+
Out << ')';
return;
}
@@ -4093,6 +4123,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
RMWI->getSyncScopeID());
} else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
+ } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
+ PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
}
// Print Metadata info.
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index de5b94403bd7..96021dc1fbd5 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -2335,11 +2335,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
if (NumSrcElts == 2)
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 0, 1 });
+ Rep = Builder.CreateShuffleVector(
+ Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
else
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 2, 3, 0, 1, 2, 3 });
+ Rep =
+ Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
+ ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 07292e50fc8d..c4465d9bfa02 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -876,22 +876,22 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
return ConstantVector::get(Result);
}
-Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
- Constant *V2,
- Constant *Mask) {
- ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
+ ArrayRef<int> Mask) {
+ unsigned MaskNumElts = Mask.size();
+ ElementCount MaskEltCount = {MaskNumElts,
+ V1->getType()->getVectorIsScalable()};
Type *EltTy = V1->getType()->getVectorElementType();
// Undefined shuffle mask -> undefined value.
- if (isa<UndefValue>(Mask))
- return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
-
- // Don't break the bitcode reader hack.
- if (isa<ConstantExpr>(Mask)) return nullptr;
+ if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
+ }
// If the mask is all zeros this is a splat, no need to go through all
// elements.
- if (isa<ConstantAggregateZero>(Mask) && !MaskEltCount.Scalable) {
+ if (all_of(Mask, [](int Elt) { return Elt == 0; }) &&
+ !MaskEltCount.Scalable) {
Type *Ty = IntegerType::get(V1->getContext(), 32);
Constant *Elt =
ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
@@ -903,13 +903,12 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
if (ValTy->isScalable())
return nullptr;
- unsigned MaskNumElts = MaskEltCount.Min;
unsigned SrcNumElts = V1->getType()->getVectorNumElements();
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
for (unsigned i = 0; i != MaskNumElts; ++i) {
- int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
+ int Elt = Mask[i];
if (Elt == -1) {
Result.push_back(UndefValue::get(EltTy));
continue;
diff --git a/llvm/lib/IR/ConstantFold.h b/llvm/lib/IR/ConstantFold.h
index 9ad6e14e9e40..0cdd5cf3cbce 100644
--- a/llvm/lib/IR/ConstantFold.h
+++ b/llvm/lib/IR/ConstantFold.h
@@ -38,7 +38,7 @@ template <typename T> class ArrayRef;
Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
Constant *Idx);
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
ArrayRef<unsigned> Idxs);
Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index e001b5230751..60ab1a595c34 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -1230,8 +1230,7 @@ Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
Constant *UndefV = UndefValue::get(VTy);
V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
// Build shuffle mask to perform the splat.
- Type *MaskTy = VectorType::get(I32Ty, EC);
- Constant *Zeros = ConstantAggregateZero::get(MaskTy);
+ SmallVector<int, 8> Zeros(EC.Min, 0);
// Splat.
return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
}
@@ -1298,6 +1297,14 @@ unsigned ConstantExpr::getPredicate() const {
return cast<CompareConstantExpr>(this)->predicate;
}
+ArrayRef<int> ConstantExpr::getShuffleMask() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
+}
+
+Constant *ConstantExpr::getShuffleMaskForBitcode() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
+}
+
Constant *
ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
assert(Op->getType() == getOperand(OpNo)->getType() &&
@@ -1349,7 +1356,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
case Instruction::ExtractValue:
return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2],
+ return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
OnlyIfReducedTy);
case Instruction::GetElementPtr: {
auto *GEPO = cast<GEPOperator>(this);
@@ -2152,7 +2159,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
if (InRangeIndex && *InRangeIndex < 63)
SubClassOptionalData |= (*InRangeIndex + 1) << 1;
const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
- SubClassOptionalData, None, Ty);
+ SubClassOptionalData, None, None, Ty);
LLVMContextImpl *pImpl = C->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ReqTy, Key);
@@ -2254,23 +2261,25 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt,
}
Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask, Type *OnlyIfReducedTy) {
+ ArrayRef<int> Mask,
+ Type *OnlyIfReducedTy) {
assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector constant expr operands!");
if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
return FC; // Fold a few common cases.
- ElementCount NElts = Mask->getType()->getVectorElementCount();
+ unsigned NElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
- Type *ShufTy = VectorType::get(EltTy, NElts);
+ bool TypeIsScalable = V1->getType()->getVectorIsScalable();
+ Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
if (OnlyIfReducedTy == ShufTy)
return nullptr;
// Look up the constant in the table first to ensure uniqueness
- Constant *ArgVec[] = { V1, V2, Mask };
- const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
+ Constant *ArgVec[] = {V1, V2};
+ ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask);
LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
@@ -3117,7 +3126,7 @@ Instruction *ConstantExpr::getAsInstruction() const {
case Instruction::ExtractValue:
return ExtractValueInst::Create(Ops[0], getIndices());
case Instruction::ShuffleVector:
- return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
case Instruction::GetElementPtr: {
const auto *GO = cast<GEPOperator>(this);
diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h
index 9ecbb6137836..ad521c8a1bdf 100644
--- a/llvm/lib/IR/ConstantsContext.h
+++ b/llvm/lib/IR/ConstantsContext.h
@@ -26,6 +26,7 @@
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@@ -146,21 +147,24 @@ class InsertElementConstantExpr : public ConstantExpr {
/// shufflevector constant exprs.
class ShuffleVectorConstantExpr : public ConstantExpr {
public:
- ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
- : ConstantExpr(VectorType::get(
- cast<VectorType>(C1->getType())->getElementType(),
- cast<VectorType>(C3->getType())->getElementCount()),
- Instruction::ShuffleVector,
- &Op<0>(), 3) {
+ ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask)
+ : ConstantExpr(
+ VectorType::get(cast<VectorType>(C1->getType())->getElementType(),
+ Mask.size(), C1->getType()->getVectorIsScalable()),
+ Instruction::ShuffleVector, &Op<0>(), 2) {
+ assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
Op<0>() = C1;
Op<1>() = C2;
- Op<2>() = C3;
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ ShuffleMaskForBitcode =
+ ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType());
}
- // allocate space for exactly three operands
- void *operator new(size_t s) {
- return User::operator new(s, 3);
- }
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
+ void *operator new(size_t s) { return User::operator new(s, 2); }
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -319,7 +323,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value)
template <>
struct OperandTraits<ShuffleVectorConstantExpr>
- : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
+ : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
template <>
@@ -460,36 +464,58 @@ struct InlineAsmKeyType {
};
struct ConstantExprKeyType {
+private:
uint8_t Opcode;
uint8_t SubclassOptionalData;
uint16_t SubclassData;
ArrayRef<Constant *> Ops;
ArrayRef<unsigned> Indexes;
+ ArrayRef<int> ShuffleMask;
Type *ExplicitTy;
+ static ArrayRef<int> getShuffleMaskIfValid(const ConstantExpr *CE) {
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ return CE->getShuffleMask();
+ return None;
+ }
+
+ static ArrayRef<unsigned> getIndicesIfValid(const ConstantExpr *CE) {
+ if (CE->hasIndices())
+ return CE->getIndices();
+ return None;
+ }
+
+ static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) {
+ if (auto *GEPCE = dyn_cast<GetElementPtrConstantExpr>(CE))
+ return GEPCE->getSourceElementType();
+ return nullptr;
+ }
+
+public:
ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops,
unsigned short SubclassData = 0,
unsigned short SubclassOptionalData = 0,
ArrayRef<unsigned> Indexes = None,
+ ArrayRef<int> ShuffleMask = None,
Type *ExplicitTy = nullptr)
: Opcode(Opcode), SubclassOptionalData(SubclassOptionalData),
SubclassData(SubclassData), Ops(Ops), Indexes(Indexes),
- ExplicitTy(ExplicitTy) {}
+ ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {}
ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands),
- Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
- ExplicitTy(nullptr) {}
+ Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+ ExplicitTy(getSourceElementTypeIfValid(CE)) {}
ConstantExprKeyType(const ConstantExpr *CE,
SmallVectorImpl<Constant *> &Storage)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0),
- Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
- ExplicitTy(nullptr) {
+ Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+ ExplicitTy(getSourceElementTypeIfValid(CE)) {
assert(Storage.empty() && "Expected empty storage");
for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
Storage.push_back(CE->getOperand(I));
@@ -499,7 +525,8 @@ struct ConstantExprKeyType {
bool operator==(const ConstantExprKeyType &X) const {
return Opcode == X.Opcode && SubclassData == X.SubclassData &&
SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops &&
- Indexes == X.Indexes;
+ Indexes == X.Indexes && ShuffleMask == X.ShuffleMask &&
+ ExplicitTy == X.ExplicitTy;
}
bool operator==(const ConstantExpr *CE) const {
@@ -514,15 +541,21 @@ struct ConstantExprKeyType {
for (unsigned I = 0, E = Ops.size(); I != E; ++I)
if (Ops[I] != CE->getOperand(I))
return false;
- if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()))
+ if (Indexes != getIndicesIfValid(CE))
+ return false;
+ if (ShuffleMask != getShuffleMaskIfValid(CE))
+ return false;
+ if (ExplicitTy != getSourceElementTypeIfValid(CE))
return false;
return true;
}
unsigned getHash() const {
- return hash_combine(Opcode, SubclassOptionalData, SubclassData,
- hash_combine_range(Ops.begin(), Ops.end()),
- hash_combine_range(Indexes.begin(), Indexes.end()));
+ return hash_combine(
+ Opcode, SubclassOptionalData, SubclassData,
+ hash_combine_range(Ops.begin(), Ops.end()),
+ hash_combine_range(Indexes.begin(), Indexes.end()),
+ hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy);
}
using TypeClass = ConstantInfo<ConstantExpr>::TypeClass;
@@ -546,17 +579,14 @@ struct ConstantExprKeyType {
case Instruction::InsertElement:
return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask);
case Instruction::InsertValue:
return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
case Instruction::ExtractValue:
return new ExtractValueConstantExpr(Ops[0], Indexes, Ty);
case Instruction::GetElementPtr:
- return GetElementPtrConstantExpr::Create(
- ExplicitTy ? ExplicitTy
- : cast<PointerType>(Ops[0]->getType()->getScalarType())
- ->getElementType(),
- Ops[0], Ops.slice(1), Ty, SubclassOptionalData);
+ return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1),
+ Ty, SubclassOptionalData);
case Instruction::ICmp:
return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData,
Ops[0], Ops[1]);
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 313bc65e775c..da2e25e77978 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -1781,9 +1781,11 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant,
LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
LLVMValueRef VectorBConstant,
LLVMValueRef MaskConstant) {
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask);
return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant),
unwrap<Constant>(VectorBConstant),
- unwrap<Constant>(MaskConstant)));
+ IntMask));
}
LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 2ef109067dd6..cd476a360155 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -443,6 +443,9 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2,
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
+ return SVI->getShuffleMask() ==
+ cast<ShuffleVectorInst>(I2)->getShuffleMask();
return true;
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 3807752eae0d..37e194a0feab 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1843,57 +1843,109 @@ bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt,
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &Name,
Instruction *InsertBefore)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
- cast<VectorType>(Mask->getType())->getElementCount()),
- ShuffleVector,
- OperandTraits<ShuffleVectorInst>::op_begin(this),
- OperandTraits<ShuffleVectorInst>::operands(this),
- InsertBefore) {
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ cast<VectorType>(Mask->getType())->getElementCount()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
+
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ setShuffleMask(MaskArr);
setName(Name);
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
+ const Twine &Name, BasicBlock *InsertAtEnd)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ cast<VectorType>(Mask->getType())->getElementCount()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+
+ Op<0>() = V1;
+ Op<1>() = V2;
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ setShuffleMask(MaskArr);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &Name,
- BasicBlock *InsertAtEnd)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
- cast<VectorType>(Mask->getType())->getElementCount()),
- ShuffleVector,
- OperandTraits<ShuffleVectorInst>::op_begin(this),
- OperandTraits<ShuffleVectorInst>::operands(this),
- InsertAtEnd) {
+ Instruction *InsertBefore)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(Mask);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name, BasicBlock *InsertAtEnd)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(Mask);
setName(Name);
}
void ShuffleVectorInst::commute() {
int NumOpElts = Op<0>()->getType()->getVectorNumElements();
- int NumMaskElts = getMask()->getType()->getVectorNumElements();
- SmallVector<Constant*, 16> NewMask(NumMaskElts);
- Type *Int32Ty = Type::getInt32Ty(getContext());
+ int NumMaskElts = ShuffleMask.size();
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
- if (MaskElt == -1) {
- NewMask[i] = UndefValue::get(Int32Ty);
+ if (MaskElt == UndefMaskElem) {
+ NewMask[i] = UndefMaskElem;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
- NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
+ NewMask[i] = MaskElt;
}
- Op<2>() = ConstantVector::get(NewMask);
+ setShuffleMask(NewMask);
Op<0>().swap(Op<1>());
}
+bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask) {
+ // V1 and V2 must be vectors of the same type.
+ if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
+ return false;
+
+ // Make sure the mask elements make sense.
+ int V1Size = cast<VectorType>(V1->getType())->getNumElements();
+ for (int Elem : Mask)
+ if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+ return false;
+
+ if (V1->getType()->getVectorIsScalable())
+ if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask))
+ return false;
+
+ return true;
+}
+
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
const Value *Mask) {
// V1 and V2 must be vectors of the same type.
@@ -1902,7 +1954,8 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
// Mask must be vector of i32.
auto *MaskTy = dyn_cast<VectorType>(Mask->getType());
- if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32))
+ if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) ||
+ MaskTy->isScalable() != V1->getType()->getVectorIsScalable())
return false;
// Check to see if Mask is valid.
@@ -1930,34 +1983,12 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
return true;
}
- // The bitcode reader can create a place holder for a forward reference
- // used as the shuffle mask. When this occurs, the shuffle mask will
- // fall into this case and fail. To avoid this error, do this bit of
- // ugliness to allow such a mask pass.
- if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
- if (CE->getOpcode() == Instruction::UserOp1)
- return true;
-
return false;
}
-int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) {
- assert(i < Mask->getType()->getVectorNumElements() && "Index out of range");
- assert(!Mask->getType()->getVectorElementCount().Scalable &&
- "Length of scalable vectors unknown at compile time");
- if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask))
- return CDS->getElementAsInteger(i);
- Constant *C = Mask->getAggregateElement(i);
- if (isa<UndefValue>(C))
- return -1;
- return cast<ConstantInt>(C)->getZExtValue();
-}
-
void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result) {
- assert(!Mask->getType()->getVectorElementCount().Scalable &&
- "Length of scalable vectors unknown at compile time");
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ unsigned NumElts = Mask->getType()->getVectorElementCount().Min;
if (isa<ConstantAggregateZero>(Mask)) {
Result.resize(NumElts, 0);
return;
@@ -1975,6 +2006,30 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
}
}
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType());
+}
+Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+ Type *ResultTy) {
+ Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext());
+ if (ResultTy->getVectorIsScalable()) {
+ assert(is_splat(Mask) && "Unexpected shuffle");
+ Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true);
+ if (Mask[0] == 0)
+ return Constant::getNullValue(VecTy);
+ return UndefValue::get(VecTy);
+ }
+ SmallVector<Constant *, 16> MaskConst;
+ for (int Elem : Mask) {
+ if (Elem == UndefMaskElem)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ return ConstantVector::get(MaskConst);
+}
+
static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
assert(!Mask.empty() && "Shuffle mask must contain elements");
bool UsesLHS = false;
@@ -2124,7 +2179,7 @@ bool ShuffleVectorInst::isIdentityWithPadding() const {
return false;
// The first part of the mask must choose elements from exactly 1 source op.
- SmallVector<int, 16> Mask = getShuffleMask();
+ ArrayRef<int> Mask = getShuffleMask();
if (!isIdentityMaskImpl(Mask, NumOpElts))
return false;
@@ -4298,7 +4353,7 @@ InsertElementInst *InsertElementInst::cloneImpl() const {
}
ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
- return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
+ return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
}
PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 84364dbb230b..a31a91766ebe 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3310,7 +3310,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) {
void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
- SV.getOperand(2)),
+ SV.getShuffleMask()),
"Invalid shufflevector operands!", &SV);
visitInstruction(SV);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 149de47e3df8..523ffe993efc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9207,10 +9207,10 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
};
- Constant *M1, *M2;
+ ArrayRef<int> M1, M2;
Value *S1Op1, *S2Op1;
- if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
- !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+ if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
+ !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
return false;
// Check that the operands are half as wide as the result and we extract
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 395c04161ea7..b7baecdec6b6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -210,7 +210,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
Arg.replaceAllUsesWith(NewVal);
} else if (IsV3) {
Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
- {0, 1, 2},
+ ArrayRef<int>{0, 1, 2},
Arg.getName() + ".load");
Arg.replaceAllUsesWith(Shuf);
} else {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index cf641e3948ac..5a2cd48a81e9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -409,7 +409,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
DL->getTypeSizeInBits(Val->getType())) {
assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
- { 0, 1, 2 });
+ ArrayRef<int>{0, 1, 2});
}
Val = B.CreateBitCast(Val, EffectiveEltTy);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index a2de0adb34d7..6d03126b2db0 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -15547,7 +15547,7 @@ bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
return false;
if (!match(I->getOperand(Op),
m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
- m_Undef(), m_Zero()))) {
+ m_Undef(), m_ZeroMask()))) {
return false;
}
Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index dd7867397b45..26b71ba5b600 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -286,7 +286,7 @@ bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
Instruction *Insert = nullptr;
// The shuffle which broadcasts the index iv into a vector.
if (!match(BroadcastSplat,
- m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+ m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// The insert element which initialises a vector with the index iv.
@@ -409,7 +409,7 @@ static bool MatchElemCountLoopSetup(Loop *L, Instruction *Shuffle,
Instruction *Insert = nullptr;
if (!match(Shuffle,
- m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+ m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// Insert the limit into a vector.
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index daa293d1e740..26c05eab45b2 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -384,7 +384,7 @@ bool X86PartialReduction::trySADReplacement(Value *Op, BinaryOperator *Add) {
NumElts = OpTy->getVectorNumElements();
if (NumElts == 2) {
// Extract down to 2 elements.
- Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], {0, 1});
+ Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{0, 1});
} else if (NumElts >= 8) {
SmallVector<uint32_t, 32> ConcatMask(NumElts);
unsigned SubElts = Ops[0]->getType()->getVectorNumElements();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 12fb8e9e96f5..6b1824048f75 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -642,12 +642,12 @@ static Instruction *shrinkSplatShuffle(TruncInst &Trunc,
InstCombiner::BuilderTy &Builder) {
auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
- Shuf->getMask()->getSplatValue() &&
+ is_splat(Shuf->getShuffleMask()) &&
Shuf->getType() == Shuf->getOperand(0)->getType()) {
// trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
Constant *NarrowUndef = UndefValue::get(Trunc.getType());
Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
- return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
+ return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
}
return nullptr;
@@ -2553,7 +2553,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
// Return a new shuffle vector. Use the same element ID's, as we
// know the vector types match #elts.
- return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
+ return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index c8276c759180..e2c53ae0b31d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2814,11 +2814,10 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
return nullptr;
Value *Vec;
- Constant *Mask;
- if (match(BCSrcOp,
- m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
+ ArrayRef<int> Mask;
+ if (match(BCSrcOp, m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
// Check whether every element of Mask is the same constant
- if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
+ if (is_splat(Mask)) {
auto *VecTy = cast<VectorType>(BCSrcOp->getType());
auto *EltTy = cast<IntegerType>(VecTy->getElementType());
if (C->isSplat(EltTy->getBitWidth())) {
@@ -2827,6 +2826,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
// then:
// => %E = extractelement <N x iK> %vec, i32 Elem
// icmp <pred> iK %SplatVal, <pattern>
+ Value *Elem = Builder.getInt32(Mask[0]);
Value *Extract = Builder.CreateExtractElement(Vec, Elem);
Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
return new ICmpInst(Pred, Extract, NewC);
@@ -5380,15 +5380,15 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
bool IsFP = isa<FCmpInst>(Cmp);
Value *V1, *V2;
- Constant *M;
- if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))))
+ ArrayRef<int> M;
+ if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))))
return nullptr;
// If both arguments of the cmp are shuffles that use the same mask and
// shuffle within a single vector, move the shuffle after the cmp:
// cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
Type *V1Ty = V1->getType();
- if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
+ if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2)
: Builder.CreateICmp(Pred, V1, V2);
@@ -5405,16 +5405,16 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
// Length-changing splats are ok, so adjust the constants as needed:
// cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
- Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true);
- if (ScalarC && ScalarM) {
+ int MaskSplatIndex;
+ if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
// We allow undefs in matching, but this transform removes those for safety.
// Demanded elements analysis should be able to recover some/all of that.
C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
- M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
- ScalarM);
+ SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
: Builder.CreateICmp(Pred, V1, C);
- return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);
+ return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()),
+ NewM);
}
return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 90b00536b471..6edf11ae6655 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1389,7 +1389,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
Shuffle->getOperand(0)->getType()->getVectorNumElements();
// Handle trivial case of a splat. Only check the first element of LHS
// operand.
- if (isa<ConstantAggregateZero>(Shuffle->getMask()) &&
+ if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) &&
DemandedElts.isAllOnesValue()) {
if (!isa<UndefValue>(I->getOperand(1))) {
I->setOperand(1, UndefValue::get(I->getOperand(1)->getType()));
@@ -1518,15 +1518,14 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
}
if (NewUndefElts) {
// Add additional discovered undefs.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
- Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
- Shuffle->getMaskValue(i)));
+ Elts.push_back(Shuffle->getMaskValue(i));
}
- I->setOperand(2, ConstantVector::get(Elts));
+ Shuffle->setShuffleMask(Elts);
MadeChange = true;
}
break;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index b866f44779eb..2d72696ee531 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -730,7 +730,7 @@ Instruction *InstCombiner::visitInsertValueInst(InsertValueInst &I) {
}
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
- int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
+ int MaskSize = Shuf.getShuffleMask().size();
int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
// A vector select does not change the size of the operands.
@@ -842,13 +842,10 @@ static Instruction *foldInsEltIntoSplat(InsertElementInst &InsElt) {
// inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
// --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *Zero = ConstantInt::getNullValue(I32Ty);
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (unsigned i = 0; i != NumMaskElts; ++i)
- NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
+ NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
}
@@ -878,26 +875,23 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
// For example:
// inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
- Constant *OldMask = Shuf->getMask();
+ SmallVector<int, 16> NewMask(NumMaskElts);
+ ArrayRef<int> OldMask = Shuf->getShuffleMask();
for (unsigned i = 0; i != NumMaskElts; ++i) {
if (i != IdxC) {
// All mask elements besides the inserted element remain the same.
- NewMaskVec[i] = OldMask->getAggregateElement(i);
- } else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
+ NewMask[i] = OldMask[i];
+ } else if (OldMask[i] == (int)IdxC) {
// If the mask element was already set, there's nothing to do
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
+ assert(OldMask[i] == UndefMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
- NewMaskVec[i] = NewMaskEltC;
+ NewMask[i] = IdxC;
}
}
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
}
@@ -965,27 +959,25 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) {
// mask vector with the insertelt index plus the length of the vector
// (because the constant vector operand of a shuffle is always the 2nd
// operand).
- Constant *Mask = Shuf->getMask();
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask = Shuf->getShuffleMask();
+ unsigned NumElts = Mask.size();
SmallVector<Constant *, 16> NewShufElts(NumElts);
- SmallVector<Constant *, 16> NewMaskElts(NumElts);
+ SmallVector<int, 16> NewMaskElts(NumElts);
for (unsigned I = 0; I != NumElts; ++I) {
if (I == InsEltIndex) {
NewShufElts[I] = InsEltScalar;
- Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
- NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
+ NewMaskElts[I] = InsEltIndex + NumElts;
} else {
// Copy over the existing values.
NewShufElts[I] = ShufConstVec->getAggregateElement(I);
- NewMaskElts[I] = Mask->getAggregateElement(I);
+ NewMaskElts[I] = Mask[I];
}
}
// Create new operands for a shuffle that includes the constant of the
// original insertelt. The old shuffle will be dead now.
return new ShuffleVectorInst(Shuf->getOperand(0),
- ConstantVector::get(NewShufElts),
- ConstantVector::get(NewMaskElts));
+ ConstantVector::get(NewShufElts), NewMaskElts);
} else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
// Transform sequences of insertelements ops with constant data/indexes into
// a single shuffle op.
@@ -1305,17 +1297,9 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
if (isa<ConstantAggregateZero>(V))
return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size()));
- if (Constant *C = dyn_cast<Constant>(V)) {
- SmallVector<Constant *, 16> MaskValues;
- for (int i = 0, e = Mask.size(); i != e; ++i) {
- if (Mask[i] == -1)
- MaskValues.push_back(UndefValue::get(I32Ty));
- else
- MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i]));
- }
+ if (Constant *C = dyn_cast<Constant>(V))
return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()),
- ConstantVector::get(MaskValues));
- }
+ Mask);
Instruction *I = cast<Instruction>(V);
switch (I->getOpcode()) {
@@ -1404,7 +1388,7 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
// Shuffles to: |EE|FF|GG|HH|
// +--+--+--+--+
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
- SmallVector<int, 16> &Mask) {
+ ArrayRef<int> Mask) {
unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
unsigned MaskElems = Mask.size();
unsigned BegIdx = Mask.front();
@@ -1487,12 +1471,12 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
// Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
// The existing binop constant vector remains in the same operand position.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -1506,7 +1490,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1518,14 +1502,14 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
InstCombiner::BuilderTy &Builder) {
Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Value *X;
uint64_t IndexC;
// Match a shuffle that is a splat to a non-zero element.
if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
m_ConstantInt(IndexC)))) ||
- !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
+ !match(Op1, m_Undef()) || match(Mask, m_ZeroMask()) || IndexC == 0)
return nullptr;
// Insert into element 0 of an undef vector.
@@ -1538,12 +1522,12 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
// shuf (inselt undef, X, 2), undef, <2,2,undef>
// --> shuf (inselt undef, X, 0), undef, <0,0,undef>
unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
+ SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (isa<UndefValue>(Mask->getAggregateElement(i)))
- NewMask[i] = Mask->getAggregateElement(i);
+ if (Mask[i] == UndefMaskElem)
+ NewMask[i] = Mask[i];
- return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
}
/// Try to fold shuffles that are the equivalent of a vector select.
@@ -1612,14 +1596,14 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
BinaryOperator::BinaryOps BOpc = Opc0;
// Select the constant elements needed for the single binop.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
// We are moving a binop after a shuffle. When a shuffle has an undefined
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
@@ -1668,7 +1652,7 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
NewBO->andIRFlags(B1);
if (DropNSW)
NewBO->setHasNoSignedWrap(false);
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1694,8 +1678,7 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
// and have the same number of elements as this shuffle.
unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
Value *NarrowCond;
- if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
- m_Constant()))) ||
+ if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
!cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
return nullptr;
@@ -1703,8 +1686,8 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
// shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
// sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
Value *Undef = UndefValue::get(X->getType());
- Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
- Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
+ Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
+ Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
@@ -1715,8 +1698,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
return nullptr;
Value *X, *Y;
- Constant *Mask;
- if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
+ ArrayRef<int> Mask;
+ if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Mask(Mask))))
return nullptr;
// Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
@@ -1736,16 +1719,16 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
// shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
// shuf X, Y, <C0, undef, C2, undef>
unsigned NumElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumElts);
- assert(NumElts < Mask->getType()->getVectorNumElements() &&
+ SmallVector<int, 16> NewMask(NumElts);
+ assert(NumElts < Mask.size() &&
"Identity with extract must have less elements than its inputs");
for (unsigned i = 0; i != NumElts; ++i) {
- Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
- Constant *MaskElt = Mask->getAggregateElement(i);
- NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
+ int ExtractMaskElt = Shuf.getMaskValue(i);
+ int MaskElt = Mask[i];
+ NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
}
- return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(X, Y, NewMask);
}
/// Try to replace a shuffle with an insertelement or try to replace a shuffle
@@ -1753,7 +1736,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
InstCombiner &IC) {
Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ SmallVector<int, 16> Mask;
+ Shuf.getShuffleMask(Mask);
// The shuffle must not change vector sizes.
// TODO: This restriction could be removed if the insert has only one use
@@ -1872,7 +1856,7 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
@@ -1905,28 +1889,28 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
Value *LHS = SVI.getOperand(0);
Value *RHS = SVI.getOperand(1);
- if (auto *V = SimplifyShuffleVectorInst(
- LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
+ if (auto *V =
+ SimplifyShuffleVectorInst(LHS, RHS, SVI.getShuffleMask(),
+ SVI.getType(), SQ.getWithInstruction(&SVI)))
return replaceInstUsesWith(SVI, V);
// shuffle x, x, mask --> shuffle x, undef, mask'
unsigned VWidth = SVI.getType()->getVectorNumElements();
unsigned LHSWidth = LHS->getType()->getVectorNumElements();
- SmallVector<int, 16> Mask = SVI.getShuffleMask();
+ ArrayRef<int> Mask = SVI.getShuffleMask();
Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
if (LHS == RHS) {
assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
// Remap any references to RHS to use LHS.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) {
// Propagate undef elements or force mask to LHS.
if (Mask[i] < 0)
- Elts.push_back(UndefValue::get(Int32Ty));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
+ Elts.push_back(Mask[i] % LHSWidth);
}
- return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()),
- ConstantVector::get(Elts));
+ return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), Elts);
}
// shuffle undef, x, mask --> shuffle x, undef, mask'
@@ -2151,8 +2135,8 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
if (newLHS == LHS && newRHS == RHS)
return MadeChange ? &SVI : nullptr;
- SmallVector<int, 16> LHSMask;
- SmallVector<int, 16> RHSMask;
+ ArrayRef<int> LHSMask;
+ ArrayRef<int> RHSMask;
if (newLHS != LHS)
LHSMask = LHSShuffle->getShuffleMask();
if (RHSShuffle && newRHS != RHS)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 68797662f45e..23370cf21f07 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1534,9 +1534,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// narrow binop on each pair of the source operands followed by concatenation
// of the results.
Value *L0, *L1, *R0, *R1;
- Constant *Mask;
- if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
+ ArrayRef<int> Mask;
+ if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
+ match(RHS,
+ m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
LHS->hasOneUse() && RHS->hasOneUse() &&
cast<ShuffleVectorInst>(LHS)->isConcat() &&
cast<ShuffleVectorInst>(RHS)->isConcat()) {
@@ -1560,7 +1561,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
- auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
+ auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
Value *XY = Builder.CreateBinOp(Opcode, X, Y);
if (auto *BO = dyn_cast<BinaryOperator>(XY))
BO->copyIRFlags(&Inst);
@@ -1570,8 +1571,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// If both arguments of the binary operation are shuffles that use the same
// mask and shuffle within a single vector, move the shuffle after the binop.
Value *V1, *V2;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
+ match(RHS,
+ m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
// Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
@@ -1581,17 +1583,19 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// If both arguments of a commutative binop are select-shuffles that use the
// same mask with commuted operands, the shuffles are unnecessary.
if (Inst.isCommutative() &&
- match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
+ match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
- m_Specific(Mask)))) {
+ m_SpecificMask(Mask)))) {
auto *LShuf = cast<ShuffleVectorInst>(LHS);
auto *RShuf = cast<ShuffleVectorInst>(RHS);
// TODO: Allow shuffles that contain undefs in the mask?
// That is legal, but it reduces undef knowledge.
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
- if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
- RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
+ if (LShuf->isSelect() &&
+ !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+ RShuf->isSelect() &&
+ !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
@@ -1608,9 +1612,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// other binops, so they can be folded. It may also enable demanded elements
// transforms.
Constant *C;
- if (match(&Inst, m_c_BinOp(
- m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
- m_Constant(C))) &&
+ if (match(&Inst, m_c_BinOp(m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(),
+ m_Mask(Mask))),
+ m_Constant(C))) &&
V1->getType()->getVectorNumElements() <= NumElts) {
assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
"Shuffle should not change scalar type");
@@ -1621,8 +1625,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// reorder is not possible. A 1-to-1 mapping is not required. Example:
// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
bool ConstOp1 = isa<Constant>(RHS);
- SmallVector<int, 16> ShMask;
- ShuffleVectorInst::getShuffleMask(Mask, ShMask);
+ ArrayRef<int> ShMask = Mask;
unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
@@ -1687,12 +1690,12 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
std::swap(LHS, RHS);
Value *X;
- Constant *MaskC;
- const APInt *SplatIndex;
+ ArrayRef<int> MaskC;
+ int SplatIndex;
BinaryOperator *BO;
if (!match(LHS, m_OneUse(m_ShuffleVector(m_Value(X), m_Undef(),
- m_Constant(MaskC)))) ||
- !match(MaskC, m_APIntAllowUndef(SplatIndex)) ||
+ m_Mask(MaskC)))) ||
+ !match(MaskC, m_SplatOrUndefMask(SplatIndex)) ||
X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) ||
BO->getOpcode() != Opcode)
return nullptr;
@@ -1701,10 +1704,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// moving 'Y' before the splat shuffle, we are implicitly assuming
// that it is not undef/poison at the splat index.
Value *Y, *OtherOp;
- if (isSplatValue(BO->getOperand(0), SplatIndex->getZExtValue())) {
+ if (isSplatValue(BO->getOperand(0), SplatIndex)) {
Y = BO->getOperand(0);
OtherOp = BO->getOperand(1);
- } else if (isSplatValue(BO->getOperand(1), SplatIndex->getZExtValue())) {
+ } else if (isSplatValue(BO->getOperand(1), SplatIndex)) {
Y = BO->getOperand(1);
OtherOp = BO->getOperand(0);
} else {
@@ -1716,7 +1719,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
UndefValue *Undef = UndefValue::get(Inst.getType());
- Constant *NewMask = ConstantInt::get(MaskC->getType(), *SplatIndex);
+ SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
Value *NewSplat = Builder.CreateShuffleVector(NewBO, Undef, NewMask);
Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index dd892921b62b..ea6156f4c56d 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1867,10 +1867,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
void visitShuffleVectorInst(ShuffleVectorInst &I) {
- insertShadowCheck(I.getOperand(2), &I);
IRBuilder<> IRB(&I);
setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
- I.getOperand(2), "_msprop"));
+ I.getShuffleMask(), "_msprop"));
setOriginForNaryOp(I);
}
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 6b9792aa77b0..d831b0da37a5 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -289,7 +289,7 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
e.commutative = true;
}
- if (CmpInst *C = dyn_cast<CmpInst>(I)) {
+ if (auto *C = dyn_cast<CmpInst>(I)) {
// Sort the operand value numbers so x<y and y>x get the same value number.
CmpInst::Predicate Predicate = C->getPredicate();
if (e.varargs[0] > e.varargs[1]) {
@@ -298,10 +298,11 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
}
e.opcode = (C->getOpcode() << 8) | Predicate;
e.commutative = true;
- } else if (InsertValueInst *E = dyn_cast<InsertValueInst>(I)) {
- for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
- II != IE; ++II)
- e.varargs.push_back(*II);
+ } else if (auto *E = dyn_cast<InsertValueInst>(I)) {
+ e.varargs.append(E->idx_begin(), E->idx_end());
+ } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
+ ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
+ e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
}
return e;
@@ -1732,7 +1733,8 @@ uint32_t GVN::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
// instead of value numbers. Those index numbers should not be
// translated.
if ((i > 1 && Exp.opcode == Instruction::InsertValue) ||
- (i > 0 && Exp.opcode == Instruction::ExtractValue))
+ (i > 0 && Exp.opcode == Instruction::ExtractValue) ||
+ (i > 1 && Exp.opcode == Instruction::ShuffleVector))
continue;
Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn);
}
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 6b9a88d04eda..d0b96218137c 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -350,6 +350,7 @@ using ModelledPHISet = DenseSet<ModelledPHI, DenseMapInfo<ModelledPHI>>;
class InstructionUseExpr : public GVNExpression::BasicExpression {
unsigned MemoryUseOrder = -1;
bool Volatile = false;
+ std::vector<int> ShuffleMask;
public:
InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
@@ -366,15 +367,18 @@ class InstructionUseExpr : public GVNExpression::BasicExpression {
void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; }
void setVolatile(bool V) { Volatile = V; }
+ void setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ }
hash_code getHashValue() const override {
return hash_combine(GVNExpression::BasicExpression::getHashValue(),
- MemoryUseOrder, Volatile);
+ MemoryUseOrder, Volatile, ArrayRef<int>(ShuffleMask));
}
template <typename Function> hash_code getHashValue(Function MapFn) {
- hash_code H =
- hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
+ hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile,
+ ArrayRef<int>(ShuffleMask));
for (auto *V : operands())
H = hash_combine(H, MapFn(V));
return H;
@@ -402,6 +406,8 @@ class ValueTable {
CmpInst::Predicate Predicate = C->getPredicate();
E->setOpcode((C->getOpcode() << 8) | Predicate);
}
+ if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
+ E->setShuffleMask(SVI->getShuffleMask());
return E;
}
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 6a643480f312..17b2c534513d 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -2030,10 +2030,12 @@ NewGVN::performSymbolicEvaluation(Value *V,
case Instruction::Select:
case Instruction::ExtractElement:
case Instruction::InsertElement:
- case Instruction::ShuffleVector:
case Instruction::GetElementPtr:
E = createExpression(I);
break;
+ case Instruction::ShuffleVector:
+ // FIXME: Add support for shufflevector to createExpression.
+ return nullptr;
default:
return nullptr;
}
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index fe344ce5f8db..b8b98d673334 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1009,7 +1009,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
auto *SV = cast<ShuffleVectorInst>(I);
UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
std::string Name = suffixed_name_or(I, ".base", "base_sv");
- return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
+ return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
Name, SV);
}
};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3a89fd829384..af51dea002da 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3405,7 +3405,7 @@ void InnerLoopVectorizer::truncateToMinimalBitwidths() {
auto *O1 = B.CreateZExtOrTrunc(
SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
- NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
+ NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
} else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
// Don't do anything with the operands, just extend the result.
continue;
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index cb11d1ec71f2..186746bca791 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -991,31 +991,32 @@ TEST_F(PatternMatchTest, VectorOps) {
EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
// Test matching shufflevector
- EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
- EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
+ ArrayRef<int> Mask;
+ EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
+ EXPECT_TRUE(
+ match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
EXPECT_TRUE(A == VI3);
EXPECT_TRUE(B == VI4);
- EXPECT_TRUE(C == IdxVec);
A = B = C = nullptr; // reset
// Test matching the vector splat pattern
EXPECT_TRUE(match(
SI1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(A == Val);
}
More information about the llvm-commits
mailing list