[llvm] d22edb9 - [IR][NFC] Change UndefMaskElem to PoisonMaskElem
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 27 10:02:05 PDT 2023
Author: ManuelJBrito
Date: 2023-04-27T18:01:54+01:00
New Revision: d22edb9794245b7b37490b9f40bd66130d8cf3b7
URL: https://github.com/llvm/llvm-project/commit/d22edb9794245b7b37490b9f40bd66130d8cf3b7
DIFF: https://github.com/llvm/llvm-project/commit/d22edb9794245b7b37490b9f40bd66130d8cf3b7.diff
LOG: [IR][NFC] Change UndefMaskElem to PoisonMaskElem
Following the change in shufflevector semantics,
poison will be used to represent undefined elements in shufflevector masks.
Differential Revision: https://reviews.llvm.org/D149256
Added:
Modified:
llvm/include/llvm-c/Core.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/include/llvm/IR/Instructions.h
llvm/lib/Analysis/InstructionSimplify.cpp
llvm/lib/Analysis/ValueTracking.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
llvm/lib/IR/AsmWriter.cpp
llvm/lib/IR/ConstantFold.cpp
llvm/lib/IR/Core.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/VectorCombine.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 0eaf949c61f48..78dff8a8c274f 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -4093,8 +4093,8 @@ int LLVMGetUndefMaskElem(void);
* Get the mask value at position Elt in the mask of a ShuffleVector
* instruction.
*
- * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is undef
- * at that position.
+ * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is
+ * poison at that position.
*/
int LLVMGetMaskValue(LLVMValueRef ShuffleVectorInst, unsigned Elt);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index db5ebb7c30b2f..131e1e3f4b91b 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1269,7 +1269,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
APInt DemandedDstElts =
APInt::getZero(Shuffle->getShuffleMask().size());
for (auto I : enumerate(Shuffle->getShuffleMask())) {
- if (I.value() != UndefMaskElem)
+ if (I.value() != PoisonMaskElem)
DemandedDstElts.setBit(I.index());
}
return TargetTTI->getReplicationShuffleCost(
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 46202e4570ff4..d7b1568ea2c01 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2002,7 +2002,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
// ShuffleVectorInst Class
//===----------------------------------------------------------------------===//
-constexpr int UndefMaskElem = -1;
+constexpr int PoisonMaskElem = -1;
/// This instruction constructs a fixed permutation of two
/// input vectors.
@@ -2010,7 +2010,7 @@ constexpr int UndefMaskElem = -1;
/// For each element of the result vector, the shuffle mask selects an element
/// from one of the input vectors to copy to the result. Non-negative elements
/// in the mask represent an index into the concatenated pair of input vectors.
-/// UndefMaskElem (-1) specifies that the result element is undefined.
+/// PoisonMaskElem (-1) specifies that the result element is poison.
///
/// For scalable vectors, all the elements of the mask must be 0 or -1. This
/// requirement may be relaxed in the future.
@@ -2068,16 +2068,16 @@ class ShuffleVectorInst : public Instruction {
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
/// Return the shuffle mask value of this instruction for the given element
- /// index. Return UndefMaskElem if the element is undef.
+ /// index. Return PoisonMaskElem if the element is undef.
int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
/// Convert the input shuffle mask operand to a vector of integers. Undefined
- /// elements of the mask are returned as UndefMaskElem.
+ /// elements of the mask are returned as PoisonMaskElem.
static void getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result);
/// Return the mask for this instruction as a vector of integers. Undefined
- /// elements of the mask are returned as UndefMaskElem.
+ /// elements of the mask are returned as PoisonMaskElem.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 9a2b030aba5c7..5b3543a8856f1 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5235,7 +5235,7 @@ static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1,
ArrayRef<int> Mask, Type *RetTy,
const SimplifyQuery &Q,
unsigned MaxRecurse) {
- if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
+ if (all_of(Mask, [](int Elem) { return Elem == PoisonMaskElem; }))
return UndefValue::get(RetTy);
auto *InVecTy = cast<VectorType>(Op0->getType());
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 35f5bb7b51a01..028920ccd0bfb 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6438,7 +6438,7 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly,
ArrayRef<int> Mask = isa<ConstantExpr>(Op)
? cast<ConstantExpr>(Op)->getShuffleMask()
: cast<ShuffleVectorInst>(Op)->getShuffleMask();
- return is_contained(Mask, UndefMaskElem);
+ return is_contained(Mask, PoisonMaskElem);
}
case Instruction::FNeg:
case Instruction::PHI:
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index aadecfb9e8180..423ef670bb3c2 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -444,13 +444,13 @@ void llvm::processShuffleMasks(
int Idx = I * SzDest + K;
if (Idx == Sz)
break;
- if (Mask[Idx] >= Sz || Mask[Idx] == UndefMaskElem)
+ if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem)
continue;
int SrcRegIdx = Mask[Idx] / SzSrc;
// Add a cost of PermuteTwoSrc for each new source register permute,
// if we have more than one source registers.
if (RegMasks[SrcRegIdx].empty())
- RegMasks[SrcRegIdx].assign(SzDest, UndefMaskElem);
+ RegMasks[SrcRegIdx].assign(SzDest, PoisonMaskElem);
RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc;
}
}
@@ -482,8 +482,8 @@ void llvm::processShuffleMasks(
auto &&CombineMasks = [](MutableArrayRef<int> FirstMask,
ArrayRef<int> SecondMask) {
for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) {
- if (SecondMask[Idx] != UndefMaskElem) {
- assert(FirstMask[Idx] == UndefMaskElem &&
+ if (SecondMask[Idx] != PoisonMaskElem) {
+ assert(FirstMask[Idx] == PoisonMaskElem &&
"Expected undefined mask element.");
FirstMask[Idx] = SecondMask[Idx] + VF;
}
@@ -491,7 +491,7 @@ void llvm::processShuffleMasks(
};
auto &&NormalizeMask = [](MutableArrayRef<int> Mask) {
for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
- if (Mask[Idx] != UndefMaskElem)
+ if (Mask[Idx] != PoisonMaskElem)
Mask[Idx] = Idx;
}
};
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 46795fd8b627e..7762aeef114c5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2383,7 +2383,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
EVT EltVT = NewVT.getVectorElementType();
SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
for (unsigned I = 0; I < NewElts; ++I) {
- if (Mask[I] == UndefMaskElem)
+ if (Mask[I] == PoisonMaskElem)
continue;
unsigned Idx = Mask[I];
if (Idx >= NewElts)
@@ -2423,11 +2423,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
// Use shuffles operands instead of shuffles themselves.
// 1. Adjust mask.
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (Inputs[SrcRegIdx].isUndef()) {
- Idx = UndefMaskElem;
+ Idx = PoisonMaskElem;
continue;
}
auto *Shuffle =
@@ -2435,8 +2435,8 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
if (!Shuffle || !is_contained(P.second, SrcRegIdx))
continue;
int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
- if (MaskElt == UndefMaskElem) {
- Idx = UndefMaskElem;
+ if (MaskElt == PoisonMaskElem) {
+ Idx = PoisonMaskElem;
continue;
}
Idx = MaskElt % NewElts +
@@ -2455,11 +2455,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
// Check if any concat_vectors can be simplified.
SmallBitVector UsedSubVector(2 * std::size(Inputs));
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (Inputs[SrcRegIdx].isUndef()) {
- Idx = UndefMaskElem;
+ Idx = PoisonMaskElem;
continue;
}
TargetLowering::LegalizeTypeAction TypeAction =
@@ -2489,7 +2489,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
if (!Pairs.empty() && Pairs.front().size() > 1) {
// Adjust mask.
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
auto *It = find_if(
@@ -2531,14 +2531,14 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
!Shuffle->getOperand(1).isUndef()) {
// Find the only used operand, if possible.
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (SrcRegIdx != I)
continue;
int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
- if (MaskElt == UndefMaskElem) {
- Idx = UndefMaskElem;
+ if (MaskElt == PoisonMaskElem) {
+ Idx = PoisonMaskElem;
continue;
}
int OpIdx = MaskElt / NewElts;
@@ -2564,14 +2564,14 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
// Found that operand is used already.
// 1. Fix the mask for the reused operand.
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (SrcRegIdx != I)
continue;
int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
- if (MaskElt == UndefMaskElem) {
- Idx = UndefMaskElem;
+ if (MaskElt == PoisonMaskElem) {
+ Idx = PoisonMaskElem;
continue;
}
int MaskIdx = MaskElt / NewElts;
@@ -2588,7 +2588,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
Inputs[I] = Shuffle->getOperand(Op);
// Adjust mask.
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (SrcRegIdx != I)
@@ -2622,11 +2622,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
unsigned ConstNum = UniqueConstantVec.size();
for (int &Idx : Mask) {
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
unsigned SrcRegIdx = Idx / NewElts;
if (Inputs[SrcRegIdx].isUndef()) {
- Idx = UndefMaskElem;
+ Idx = PoisonMaskElem;
continue;
}
const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
@@ -2655,7 +2655,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
// Build a shuffle mask for the output, discovering on the fly which
// input vectors to use as shuffle operands.
unsigned FirstMaskIdx = High * NewElts;
- SmallVector<int> Mask(NewElts * std::size(Inputs), UndefMaskElem);
+ SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
assert(!Output && "Expected default initialized initial value.");
TryPeekThroughShufflesInputs(Mask);
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 028c4efbc0b37..d9404bdb7b4af 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -425,7 +425,7 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
bool FirstElt = true;
if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
Out << "zeroinitializer";
- } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ } else if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) {
Out << "poison";
} else {
Out << "<";
@@ -435,7 +435,7 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
else
Out << ", ";
Out << "i32 ";
- if (Elt == UndefMaskElem)
+ if (Elt == PoisonMaskElem)
Out << "poison";
else
Out << Elt;
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 59131a4264fc5..101e1de31b2b1 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -711,7 +711,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
Type *EltTy = V1VTy->getElementType();
// Undefined shuffle mask -> undefined value.
- if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) {
return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
}
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 4821ff56fe457..4aee5ce481fe0 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -3949,7 +3949,7 @@ int LLVMGetMaskValue(LLVMValueRef SVInst, unsigned Elt) {
return I->getMaskValue(Elt);
}
-int LLVMGetUndefMaskElem(void) { return UndefMaskElem; }
+int LLVMGetUndefMaskElem(void) { return PoisonMaskElem; }
LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) {
Value *P = unwrap(AtomicInst);
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index c72fdc8ba75e3..3983fd8ed7ff8 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2164,8 +2164,8 @@ void ShuffleVectorInst::commute() {
SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
- if (MaskElt == UndefMaskElem) {
- NewMask[i] = UndefMaskElem;
+ if (MaskElt == PoisonMaskElem) {
+ NewMask[i] = PoisonMaskElem;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
@@ -2186,11 +2186,11 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
int V1Size =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
for (int Elem : Mask)
- if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+ if (Elem != PoisonMaskElem && Elem >= V1Size * 2)
return false;
if (isa<ScalableVectorType>(V1->getType()))
- if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !all_equal(Mask))
+ if ((Mask[0] != 0 && Mask[0] != PoisonMaskElem) || !all_equal(Mask))
return false;
return true;
@@ -2289,7 +2289,7 @@ Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
}
SmallVector<Constant *, 16> MaskConst;
for (int Elem : Mask) {
- if (Elem == UndefMaskElem)
+ if (Elem == PoisonMaskElem)
MaskConst.push_back(UndefValue::get(Int32Ty));
else
MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
@@ -2627,7 +2627,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask,
"Run out of mask?");
Mask = Mask.drop_front(ReplicationFactor);
if (!all_of(CurrSubMask, [CurrElt](int MaskElt) {
- return MaskElt == UndefMaskElem || MaskElt == CurrElt;
+ return MaskElt == PoisonMaskElem || MaskElt == CurrElt;
}))
return false;
}
@@ -2639,7 +2639,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask,
bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,
int &ReplicationFactor, int &VF) {
// undef-less case is trivial.
- if (!llvm::is_contained(Mask, UndefMaskElem)) {
+ if (!llvm::is_contained(Mask, PoisonMaskElem)) {
ReplicationFactor =
Mask.take_while([](int MaskElt) { return MaskElt == 0; }).size();
if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0)
@@ -2657,7 +2657,7 @@ bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,
// Before doing that, let's perform basic correctness checking first.
int Largest = -1;
for (int MaskElt : Mask) {
- if (MaskElt == UndefMaskElem)
+ if (MaskElt == PoisonMaskElem)
continue;
// Elements must be in non-decreasing order.
if (MaskElt < Largest)
@@ -2703,11 +2703,11 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF) {
return false;
for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) {
ArrayRef<int> SubMask = Mask.slice(K, VF);
- if (all_of(SubMask, [](int Idx) { return Idx == UndefMaskElem; }))
+ if (all_of(SubMask, [](int Idx) { return Idx == PoisonMaskElem; }))
continue;
SmallBitVector Used(VF, false);
for_each(SubMask, [&Used, VF](int Idx) {
- if (Idx != UndefMaskElem && Idx < VF)
+ if (Idx != PoisonMaskElem && Idx < VF)
Used.set(Idx);
});
if (!Used.all())
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1a7adf62047db..1b51140fe1d16 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14744,9 +14744,9 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
auto Mask = SVI->getShuffleMask();
// Sanity check if all the indices are NOT in range.
- // If mask is `undef` or `poison`, `Mask` may be a vector of -1s.
- // If all of them are `undef`, OOB read will happen later.
- if (llvm::all_of(Mask, [](int Idx) { return Idx == UndefMaskElem; })) {
+ // If mask is `poison`, `Mask` may be a vector of -1s.
+ // If all of them are `poison`, OOB read will happen later.
+ if (llvm::all_of(Mask, [](int Idx) { return Idx == PoisonMaskElem; })) {
return false;
}
// A 64bit st2 which does not start at element 0 will involved adding extra
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 6e3fd7358bb3d..80a3f13f304b0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3265,9 +3265,9 @@ InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
unsigned NumSources = 0;
for (unsigned E = 0; E < LTNumElts; E++) {
int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E]
- : UndefMaskElem;
+ : PoisonMaskElem;
if (MaskElt < 0) {
- NMask.push_back(UndefMaskElem);
+ NMask.push_back(PoisonMaskElem);
continue;
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 784c80301991c..e2de1db698d6a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1612,7 +1612,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
- SmallVector<int> NormalizedMask(NormalizedVF, UndefMaskElem);
+ SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
copy(Mask, NormalizedMask.begin());
unsigned PrevSrcReg = 0;
ArrayRef<int> PrevRegMask;
@@ -1634,7 +1634,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
return;
}
if (SrcReg != DestReg &&
- any_of(RegMask, [](int I) { return I != UndefMaskElem; })) {
+ any_of(RegMask, [](int I) { return I != PoisonMaskElem; })) {
// Just a copy of the source register.
Cost += TTI::TCC_Basic;
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 771d375a2b889..fb67e9f260b92 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2728,7 +2728,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
for (i = 0; i != SubVecNumElts; ++i)
WidenMask.push_back(i);
for (; i != VecNumElts; ++i)
- WidenMask.push_back(UndefMaskElem);
+ WidenMask.push_back(PoisonMaskElem);
Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);
@@ -3023,7 +3023,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
int Sz = Mask.size();
SmallBitVector UsedIndices(Sz);
for (int Idx : Mask) {
- if (Idx == UndefMaskElem || UsedIndices.test(Idx))
+ if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
break;
UsedIndices.set(Idx);
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 0a746e25b31a4..c76f9baabd72c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -2496,7 +2496,7 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
// in the case of a shuffle with no undefined mask elements.
ArrayRef<int> Mask;
if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
- !is_contained(Mask, UndefMaskElem) &&
+ !is_contained(Mask, PoisonMaskElem) &&
cast<ShuffleVectorInst>(TVal)->isSelect()) {
if (X == FVal) {
// select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X)
@@ -2510,7 +2510,7 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
}
}
if (match(FVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
- !is_contained(Mask, UndefMaskElem) &&
+ !is_contained(Mask, PoisonMaskElem) &&
cast<ShuffleVectorInst>(FVal)->isSelect()) {
if (X == TVal) {
// select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 7195edc5d282a..29243dc89e34e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1553,7 +1553,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
- Elts.push_back(UndefMaskElem);
+ Elts.push_back(PoisonMaskElem);
else
Elts.push_back(Shuffle->getMaskValue(i));
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 5851e76454334..db452331c1011 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -1344,7 +1344,7 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(OldMask[i] == UndefMaskElem &&
+ assert(OldMask[i] == PoisonMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
NewMask[i] = IdxC;
}
@@ -2145,7 +2145,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- is_contained(Mask, UndefMaskElem) &&
+ is_contained(Mask, PoisonMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = InstCombiner::getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -2159,7 +2159,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -2194,7 +2194,7 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
cast<FixedVectorType>(Shuf.getType())->getNumElements();
SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (Mask[i] == UndefMaskElem)
+ if (Mask[i] == PoisonMaskElem)
NewMask[i] = Mask[i];
return new ShuffleVectorInst(NewIns, NewMask);
@@ -2279,7 +2279,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- is_contained(Mask, UndefMaskElem) &&
+ is_contained(Mask, PoisonMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = InstCombiner::getSafeVectorConstantForBinop(BOpc, NewC,
@@ -2330,7 +2330,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
NewI->andIRFlags(B1);
if (DropNSW)
NewI->setHasNoSignedWrap(false);
- if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
NewI->dropPoisonGeneratingFlags();
}
return replaceInstUsesWith(Shuf, NewBO);
@@ -2366,7 +2366,7 @@ static Instruction *foldTruncShuffle(ShuffleVectorInst &Shuf,
SrcType->getScalarSizeInBits() / DestType->getScalarSizeInBits();
ArrayRef<int> Mask = Shuf.getShuffleMask();
for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
- if (Mask[i] == UndefMaskElem)
+ if (Mask[i] == PoisonMaskElem)
continue;
uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;
assert(LSBIndex <= INT32_MAX && "Overflowed 32-bits");
@@ -2552,7 +2552,7 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
for (unsigned i = 0; i != NumElts; ++i) {
int ExtractMaskElt = Shuf.getMaskValue(i);
int MaskElt = Mask[i];
- NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
+ NewMask[i] = ExtractMaskElt == PoisonMaskElem ? ExtractMaskElt : MaskElt;
}
return new ShuffleVectorInst(X, Y, NewMask);
}
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 3757f4d06b007..762928b8dc0cb 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1707,9 +1707,9 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
if (LShuf->isSelect() &&
- !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+ !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) &&
RShuf->isSelect() &&
- !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
+ !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 1c8e4e3512dca..77a19567feef6 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -658,7 +658,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
if (isConstantIntVector(Mask)) {
unsigned MemIndex = 0;
VResult = PoisonValue::get(VecType);
- SmallVector<int, 16> ShuffleMask(VectorWidth, UndefMaskElem);
+ SmallVector<int, 16> ShuffleMask(VectorWidth, PoisonMaskElem);
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
Value *InsertElt;
if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue()) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5e7ac5fffabbe..0879de730c348 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -349,7 +349,7 @@ static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask,
UseMask MaskArg) {
SmallBitVector UseMask(VF, true);
for (auto [Idx, Value] : enumerate(Mask)) {
- if (Value == UndefMaskElem) {
+ if (Value == PoisonMaskElem) {
if (MaskArg == UseMask::UndefsAsMask)
UseMask.reset(Idx);
continue;
@@ -467,7 +467,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
Value *Vec2 = nullptr;
enum ShuffleMode { Unknown, Select, Permute };
ShuffleMode CommonShuffleMode = Unknown;
- Mask.assign(VL.size(), UndefMaskElem);
+ Mask.assign(VL.size(), PoisonMaskElem);
for (unsigned I = 0, E = VL.size(); I < E; ++I) {
// Undef can be represented as an undef element in a vector.
if (isa<UndefValue>(VL[I]))
@@ -644,7 +644,7 @@ tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
!isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
is_contained(UndefVectorExtracts, I))
continue;
- if (Mask[I] == UndefMaskElem && !isa<PoisonValue>(GatheredExtracts[I]))
+ if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]))
std::swap(VL[I], GatheredExtracts[I]);
}
return Res;
@@ -958,12 +958,12 @@ static void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
Mask.append(SubMask.begin(), SubMask.end());
return;
}
- SmallVector<int> NewMask(SubMask.size(), UndefMaskElem);
+ SmallVector<int> NewMask(SubMask.size(), PoisonMaskElem);
int TermValue = std::min(Mask.size(), SubMask.size());
for (int I = 0, E = SubMask.size(); I < E; ++I) {
if ((!ExtendingManyInputs &&
(SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue)) ||
- SubMask[I] == UndefMaskElem)
+ SubMask[I] == PoisonMaskElem)
continue;
NewMask[I] = Mask[SubMask[I]];
}
@@ -1010,7 +1010,7 @@ static void inversePermutation(ArrayRef<unsigned> Indices,
SmallVectorImpl<int> &Mask) {
Mask.clear();
const unsigned E = Indices.size();
- Mask.resize(E, UndefMaskElem);
+ Mask.resize(E, PoisonMaskElem);
for (unsigned I = 0; I < E; ++I)
Mask[Indices[I]] = I;
}
@@ -1023,7 +1023,7 @@ static void reorderScalars(SmallVectorImpl<Value *> &Scalars,
UndefValue::get(Scalars.front()->getType()));
Prev.swap(Scalars);
for (unsigned I = 0, E = Prev.size(); I < E; ++I)
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
Scalars[Mask[I]] = Prev[I];
}
@@ -2538,8 +2538,8 @@ class BoUpSLP {
std::equal(VL.begin(), VL.end(), Mask.begin(),
[Scalars](Value *V, int Idx) {
return (isa<UndefValue>(V) &&
- Idx == UndefMaskElem) ||
- (Idx != UndefMaskElem && V == Scalars[Idx]);
+ Idx == PoisonMaskElem) ||
+ (Idx != PoisonMaskElem && V == Scalars[Idx]);
});
};
if (!ReorderIndices.empty()) {
@@ -3719,7 +3719,7 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
SmallVector<int> Prev(Reuses.begin(), Reuses.end());
Prev.swap(Reuses);
for (unsigned I = 0, E = Prev.size(); I < E; ++I)
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
Reuses[Mask[I]] = Prev[I];
}
@@ -3743,7 +3743,7 @@ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
}
Order.assign(Mask.size(), Mask.size());
for (unsigned I = 0, E = Mask.size(); I < E; ++I)
- if (MaskOrder[I] != UndefMaskElem)
+ if (MaskOrder[I] != PoisonMaskElem)
Order[MaskOrder[I]] = I;
fixupOrderingIndices(Order);
}
@@ -4078,14 +4078,14 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
std::optional<unsigned> Idx = getExtractIndex(cast<Instruction>(V));
return Idx && *Idx < Sz;
})) {
- SmallVector<int> ReorderMask(Sz, UndefMaskElem);
+ SmallVector<int> ReorderMask(Sz, PoisonMaskElem);
if (TE.ReorderIndices.empty())
std::iota(ReorderMask.begin(), ReorderMask.end(), 0);
else
inversePermutation(TE.ReorderIndices, ReorderMask);
for (unsigned I = 0; I < VF; ++I) {
int &Idx = ReusedMask[I];
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
Value *V = TE.Scalars[ReorderMask[Idx]];
std::optional<unsigned> EI = getExtractIndex(cast<Instruction>(V));
@@ -4100,7 +4100,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
for (unsigned K = 0; K < VF; K += Sz) {
OrdersType CurrentOrder(TE.ReorderIndices);
SmallVector<int> SubMask{ArrayRef(ReusedMask).slice(K, Sz)};
- if (SubMask.front() == UndefMaskElem)
+ if (SubMask.front() == PoisonMaskElem)
std::iota(SubMask.begin(), SubMask.end(), 0);
reorderOrder(CurrentOrder, SubMask);
transform(CurrentOrder, It, [K](unsigned Pos) { return Pos + K; });
@@ -4437,7 +4437,7 @@ void BoUpSLP::reorderTopToBottom() {
unsigned E = Order.size();
OrdersType CurrentOrder(E, E);
transform(Mask, CurrentOrder.begin(), [E](int Idx) {
- return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
+ return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
});
fixupOrderingIndices(CurrentOrder);
++OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second;
@@ -4462,10 +4462,10 @@ void BoUpSLP::reorderTopToBottom() {
continue;
SmallVector<int> Mask;
inversePermutation(BestOrder, Mask);
- SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
+ SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
unsigned E = BestOrder.size();
transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
- return I < E ? static_cast<int>(I) : UndefMaskElem;
+ return I < E ? static_cast<int>(I) : PoisonMaskElem;
});
// Do an actual reordering, if profitable.
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
@@ -4676,7 +4676,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
unsigned E = Order.size();
OrdersType CurrentOrder(E, E);
transform(Mask, CurrentOrder.begin(), [E](int Idx) {
- return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
+ return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
});
fixupOrderingIndices(CurrentOrder);
OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second +=
@@ -4755,10 +4755,10 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
VisitedOps.clear();
SmallVector<int> Mask;
inversePermutation(BestOrder, Mask);
- SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
+ SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
unsigned E = BestOrder.size();
transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
- return I < E ? static_cast<int>(I) : UndefMaskElem;
+ return I < E ? static_cast<int>(I) : PoisonMaskElem;
});
for (const std::pair<unsigned, TreeEntry *> &Op : Data.second) {
TreeEntry *TE = Op.second;
@@ -5167,7 +5167,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (isConstant(V)) {
ReuseShuffleIndicies.emplace_back(
- isa<UndefValue>(V) ? UndefMaskElem : UniqueValues.size());
+ isa<UndefValue>(V) ? PoisonMaskElem : UniqueValues.size());
UniqueValues.emplace_back(V);
continue;
}
@@ -6298,7 +6298,7 @@ buildShuffleEntryMask(ArrayRef<Value *> VL, ArrayRef<unsigned> ReorderIndices,
SmallVectorImpl<Value *> *OpScalars = nullptr,
SmallVectorImpl<Value *> *AltScalars = nullptr) {
unsigned Sz = VL.size();
- Mask.assign(Sz, UndefMaskElem);
+ Mask.assign(Sz, PoisonMaskElem);
SmallVector<int> OrderMask;
if (!ReorderIndices.empty())
inversePermutation(ReorderIndices, OrderMask);
@@ -6318,9 +6318,9 @@ buildShuffleEntryMask(ArrayRef<Value *> VL, ArrayRef<unsigned> ReorderIndices,
}
}
if (!ReusesIndices.empty()) {
- SmallVector<int> NewMask(ReusesIndices.size(), UndefMaskElem);
+ SmallVector<int> NewMask(ReusesIndices.size(), PoisonMaskElem);
transform(ReusesIndices, NewMask.begin(), [&Mask](int Idx) {
- return Idx != UndefMaskElem ? Mask[Idx] : UndefMaskElem;
+ return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem;
});
Mask.swap(NewMask);
}
@@ -6440,13 +6440,13 @@ class BaseShuffleAnalysis {
static void combineMasks(unsigned LocalVF, SmallVectorImpl<int> &Mask,
ArrayRef<int> ExtMask) {
unsigned VF = Mask.size();
- SmallVector<int> NewMask(ExtMask.size(), UndefMaskElem);
+ SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
- if (ExtMask[I] == UndefMaskElem)
+ if (ExtMask[I] == PoisonMaskElem)
continue;
int MaskedIdx = Mask[ExtMask[I] % VF];
NewMask[I] =
- MaskedIdx == UndefMaskElem ? UndefMaskElem : MaskedIdx % LocalVF;
+ MaskedIdx == PoisonMaskElem ? PoisonMaskElem : MaskedIdx % LocalVF;
}
Mask.swap(NewMask);
}
@@ -6533,9 +6533,9 @@ class BaseShuffleAnalysis {
if (auto *SVOpTy =
dyn_cast<FixedVectorType>(SV->getOperand(0)->getType()))
LocalVF = SVOpTy->getNumElements();
- SmallVector<int> ExtMask(Mask.size(), UndefMaskElem);
+ SmallVector<int> ExtMask(Mask.size(), PoisonMaskElem);
for (auto [Idx, I] : enumerate(Mask)) {
- if (I == UndefMaskElem ||
+ if (I == PoisonMaskElem ||
static_cast<unsigned>(I) >= SV->getShuffleMask().size())
continue;
ExtMask[Idx] = SV->getMaskValue(I);
@@ -6551,11 +6551,11 @@ class BaseShuffleAnalysis {
if (!IsOp1Undef && !IsOp2Undef) {
// Update mask and mark undef elems.
for (int &I : Mask) {
- if (I == UndefMaskElem)
+ if (I == PoisonMaskElem)
continue;
if (SV->getMaskValue(I % SV->getShuffleMask().size()) ==
- UndefMaskElem)
- I = UndefMaskElem;
+ PoisonMaskElem)
+ I = PoisonMaskElem;
}
break;
}
@@ -6577,8 +6577,8 @@ class BaseShuffleAnalysis {
"Expected masks of same sizes.");
// Clear known poison elements.
for (auto [I, Idx] : enumerate(Mask))
- if (Idx == UndefMaskElem)
- IdentityMask[I] = UndefMaskElem;
+ if (Idx == PoisonMaskElem)
+ IdentityMask[I] = PoisonMaskElem;
Mask.swap(IdentityMask);
auto *Shuffle = dyn_cast<ShuffleVectorInst>(V);
return SinglePermute &&
@@ -6614,8 +6614,8 @@ class BaseShuffleAnalysis {
Value *Op2 = V2;
int VF =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
- SmallVector<int> CombinedMask1(Mask.size(), UndefMaskElem);
- SmallVector<int> CombinedMask2(Mask.size(), UndefMaskElem);
+ SmallVector<int> CombinedMask1(Mask.size(), PoisonMaskElem);
+ SmallVector<int> CombinedMask2(Mask.size(), PoisonMaskElem);
for (int I = 0, E = Mask.size(); I < E; ++I) {
if (Mask[I] < VF)
CombinedMask1[I] = Mask[I];
@@ -6633,9 +6633,9 @@ class BaseShuffleAnalysis {
// again.
if (auto *SV1 = dyn_cast<ShuffleVectorInst>(Op1))
if (auto *SV2 = dyn_cast<ShuffleVectorInst>(Op2)) {
- SmallVector<int> ExtMask1(Mask.size(), UndefMaskElem);
+ SmallVector<int> ExtMask1(Mask.size(), PoisonMaskElem);
for (auto [Idx, I] : enumerate(CombinedMask1)) {
- if (I == UndefMaskElem)
+ if (I == PoisonMaskElem)
continue;
ExtMask1[Idx] = SV1->getMaskValue(I);
}
@@ -6643,9 +6643,9 @@ class BaseShuffleAnalysis {
cast<FixedVectorType>(SV1->getOperand(1)->getType())
->getNumElements(),
ExtMask1, UseMask::SecondArg);
- SmallVector<int> ExtMask2(CombinedMask2.size(), UndefMaskElem);
+ SmallVector<int> ExtMask2(CombinedMask2.size(), PoisonMaskElem);
for (auto [Idx, I] : enumerate(CombinedMask2)) {
- if (I == UndefMaskElem)
+ if (I == PoisonMaskElem)
continue;
ExtMask2[Idx] = SV2->getMaskValue(I);
}
@@ -6685,8 +6685,8 @@ class BaseShuffleAnalysis {
->getElementCount()
.getKnownMinValue());
for (int I = 0, E = Mask.size(); I < E; ++I) {
- if (CombinedMask2[I] != UndefMaskElem) {
- assert(CombinedMask1[I] == UndefMaskElem &&
+ if (CombinedMask2[I] != PoisonMaskElem) {
+ assert(CombinedMask1[I] == PoisonMaskElem &&
"Expected undefined mask element");
CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
}
@@ -6880,25 +6880,25 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// Process extracts in blocks of EltsPerVector to check if the source vector
// operand can be re-used directly. If not, add the cost of creating a
// shuffle to extract the values into a vector register.
- SmallVector<int> RegMask(EltsPerVector, UndefMaskElem);
+ SmallVector<int> RegMask(EltsPerVector, PoisonMaskElem);
for (auto *V : VL) {
++Idx;
// Reached the start of a new vector registers.
if (Idx % EltsPerVector == 0) {
- RegMask.assign(EltsPerVector, UndefMaskElem);
+ RegMask.assign(EltsPerVector, PoisonMaskElem);
AllConsecutive = true;
continue;
}
// Need to exclude undefs from analysis.
- if (isa<UndefValue>(V) || Mask[Idx] == UndefMaskElem)
+ if (isa<UndefValue>(V) || Mask[Idx] == PoisonMaskElem)
continue;
// Check all extracts for a vector register on the target directly
// extract values in order.
unsigned CurrentIdx = *getExtractIndex(cast<Instruction>(V));
- if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != UndefMaskElem) {
+ if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != PoisonMaskElem) {
unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
CurrentIdx % EltsPerVector == Idx % EltsPerVector;
@@ -6945,7 +6945,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
DenseMap<Value *, int> ExtractVectorsTys;
for (auto [I, V] : enumerate(VL)) {
// Ignore non-extractelement scalars.
- if (isa<UndefValue>(V) || (!Mask.empty() && Mask[I] == UndefMaskElem))
+ if (isa<UndefValue>(V) || (!Mask.empty() && Mask[I] == PoisonMaskElem))
continue;
// If all users of instruction are going to be vectorized and this
// instruction itself is not going to be vectorized, consider this
@@ -7167,7 +7167,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
// Remove shuffled elements from list of gathers.
for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
GatheredScalars[I] = PoisonValue::get(ScalarTy);
}
LLVM_DEBUG(dbgs() << "SLP: shuffled " << Entries.size()
@@ -7398,7 +7398,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
unsigned NumOfParts = TTI->getNumberOfParts(SrcVecTy);
- SmallVector<int> InsertMask(NumElts, UndefMaskElem);
+ SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
unsigned OffsetBeg = *getInsertIndex(VL.front());
unsigned OffsetEnd = OffsetBeg;
InsertMask[OffsetBeg] = 0;
@@ -7435,13 +7435,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
SmallVector<int> Mask;
if (!E->ReorderIndices.empty()) {
inversePermutation(E->ReorderIndices, Mask);
- Mask.append(InsertVecSz - Mask.size(), UndefMaskElem);
+ Mask.append(InsertVecSz - Mask.size(), PoisonMaskElem);
} else {
- Mask.assign(VecSz, UndefMaskElem);
+ Mask.assign(VecSz, PoisonMaskElem);
std::iota(Mask.begin(), std::next(Mask.begin(), InsertVecSz), 0);
}
bool IsIdentity = true;
- SmallVector<int> PrevMask(InsertVecSz, UndefMaskElem);
+ SmallVector<int> PrevMask(InsertVecSz, PoisonMaskElem);
Mask.swap(PrevMask);
for (unsigned I = 0; I < NumScalars; ++I) {
unsigned InsertIdx = *getInsertIndex(VL[PrevMask[I]]);
@@ -7484,14 +7484,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InsertVecTy);
} else {
for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
- Mask[I] = InMask.test(I) ? UndefMaskElem : I;
+ Mask[I] = InMask.test(I) ? PoisonMaskElem : I;
for (unsigned I = OffsetBeg - Offset, End = OffsetEnd - Offset;
I <= End; ++I)
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
Mask[I] = I + VecSz;
for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I)
Mask[I] =
- ((I >= InMask.size()) || InMask.test(I)) ? UndefMaskElem : I;
+ ((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I;
Cost += TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
}
}
@@ -8133,8 +8133,8 @@ static T *performExtractsShuffleAction(
ResizeAction(ShuffleMask.begin()->first, Mask, /*ForSingleMask=*/false);
SmallBitVector IsBasePoison = isUndefVector<true>(Base, UseMask);
for (unsigned Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
- if (Mask[Idx] == UndefMaskElem)
- Mask[Idx] = IsBasePoison.test(Idx) ? UndefMaskElem : Idx;
+ if (Mask[Idx] == PoisonMaskElem)
+ Mask[Idx] = IsBasePoison.test(Idx) ? PoisonMaskElem : Idx;
else
Mask[Idx] = (Res.second ? Idx : Mask[Idx]) + VF;
}
@@ -8163,8 +8163,8 @@ static T *performExtractsShuffleAction(
// can shuffle them directly.
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
- if (SecMask[I] != UndefMaskElem) {
- assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars.");
+ if (SecMask[I] != PoisonMaskElem) {
+ assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
Mask[I] = SecMask[I] + Vec1VF;
}
}
@@ -8177,12 +8177,12 @@ static T *performExtractsShuffleAction(
ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
- if (Mask[I] != UndefMaskElem) {
- assert(SecMask[I] == UndefMaskElem && "Multiple uses of scalars.");
+ if (Mask[I] != PoisonMaskElem) {
+ assert(SecMask[I] == PoisonMaskElem && "Multiple uses of scalars.");
if (Res1.second)
Mask[I] = I;
- } else if (SecMask[I] != UndefMaskElem) {
- assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars.");
+ } else if (SecMask[I] != PoisonMaskElem) {
+ assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
Mask[I] = (Res2.second ? I : SecMask[I]) + VF;
}
}
@@ -8199,11 +8199,11 @@ static T *performExtractsShuffleAction(
ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
- if (SecMask[I] != UndefMaskElem) {
- assert((Mask[I] == UndefMaskElem || IsBaseNotUndef) &&
+ if (SecMask[I] != PoisonMaskElem) {
+ assert((Mask[I] == PoisonMaskElem || IsBaseNotUndef) &&
"Multiple uses of scalars.");
Mask[I] = (Res.second ? I : SecMask[I]) + VF;
- } else if (Mask[I] != UndefMaskElem) {
+ } else if (Mask[I] != PoisonMaskElem) {
Mask[I] = I;
}
}
@@ -8298,7 +8298,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
(void)ShuffleMasks.emplace_back();
SmallVectorImpl<int> &Mask = ShuffleMasks.back()[ScalarTE];
if (Mask.empty())
- Mask.assign(FTy->getNumElements(), UndefMaskElem);
+ Mask.assign(FTy->getNumElements(), PoisonMaskElem);
// Find the insertvector, vectorized in tree, if any.
Value *Base = VU;
while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
@@ -8312,7 +8312,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
do {
IEBase = cast<InsertElementInst>(Base);
int Idx = *getInsertIndex(IEBase);
- assert(Mask[Idx] == UndefMaskElem &&
+ assert(Mask[Idx] == PoisonMaskElem &&
"InsertElementInstruction used already.");
Mask[Idx] = Idx;
Base = IEBase->getOperand(0);
@@ -8332,7 +8332,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
int InIdx = *InsertIdx;
SmallVectorImpl<int> &Mask = ShuffleMasks[VecId][ScalarTE];
if (Mask.empty())
- Mask.assign(FTy->getNumElements(), UndefMaskElem);
+ Mask.assign(FTy->getNumElements(), PoisonMaskElem);
Mask[InIdx] = EU.Lane;
DemandedElts[VecId].setBit(InIdx);
continue;
@@ -8371,7 +8371,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
(all_of(Mask,
[VF](int Idx) { return Idx < 2 * static_cast<int>(VF); }) &&
!ShuffleVectorInst::isIdentityMask(Mask)))) {
- SmallVector<int> OrigMask(VecVF, UndefMaskElem);
+ SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
C = TTI->getShuffleCost(
@@ -8457,7 +8457,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
// No need to check for the topmost gather node.
if (TE == VectorizableTree.front().get())
return std::nullopt;
- Mask.assign(VL.size(), UndefMaskElem);
+ Mask.assign(VL.size(), PoisonMaskElem);
assert(TE->UserTreeIndices.size() == 1 &&
"Expected only single user of the gather node.");
// TODO: currently checking only for Scalars in the tree entry, need to count
@@ -8620,7 +8620,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
// Clear undef scalars.
for (int I = 0, Sz = VL.size(); I < Sz; ++I)
if (isa<PoisonValue>(VL[I]))
- Mask[I] = UndefMaskElem;
+ Mask[I] = PoisonMaskElem;
return TargetTransformInfo::SK_PermuteSingleSrc;
}
// No perfect match, just shuffle, so choose the first tree node from the
@@ -9069,7 +9069,7 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root) {
if (isa<PoisonValue>(VL[I]))
continue;
if (auto *SV = dyn_cast<ShuffleVectorInst>(Root)) {
- if (SV->getMaskValue(I) == UndefMaskElem)
+ if (SV->getMaskValue(I) == PoisonMaskElem)
continue;
}
}
@@ -9180,7 +9180,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
int V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
int VF = std::max(V1VF, V2VF);
int MinVF = std::min(V1VF, V2VF);
- SmallVector<int> IdentityMask(VF, UndefMaskElem);
+ SmallVector<int> IdentityMask(VF, PoisonMaskElem);
std::iota(IdentityMask.begin(), std::next(IdentityMask.begin(), MinVF),
0);
Value *&Op = MinVF == V1VF ? V1 : V2;
@@ -9211,7 +9211,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
static void transformMaskAfterShuffle(MutableArrayRef<int> CommonMask,
ArrayRef<int> Mask) {
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (Mask[Idx] != UndefMaskElem)
+ if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx;
}
@@ -9224,7 +9224,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
Value *VecBase = nullptr;
for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
int Idx = Mask[I];
- if (Idx == UndefMaskElem)
+ if (Idx == PoisonMaskElem)
continue;
auto *EI = cast<ExtractElementInst>(E->Scalars[I]);
VecBase = EI->getVectorOperand();
@@ -9272,7 +9272,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
}
V1 = createShuffle(V1, V2, Mask);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (Mask[Idx] != UndefMaskElem)
+ if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx + Sz;
InVectors.front() = Vec;
if (InVectors.size() == 2)
@@ -9285,7 +9285,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
if (InVectors.empty()) {
if (!isa<FixedVectorType>(V1->getType())) {
V1 = createShuffle(V1, nullptr, CommonMask);
- CommonMask.assign(Mask.size(), UndefMaskElem);
+ CommonMask.assign(Mask.size(), PoisonMaskElem);
transformMaskAfterShuffle(CommonMask, Mask);
}
InVectors.push_back(V1);
@@ -9307,7 +9307,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
transformMaskAfterShuffle(CommonMask, CommonMask);
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (CommonMask[Idx] == UndefMaskElem && Mask[Idx] != UndefMaskElem)
+ if (CommonMask[Idx] == PoisonMaskElem && Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] =
V->getType() != V1->getType()
? Idx + Sz
@@ -9325,7 +9325,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
// Check if second vector is required if the used elements are already
// used from the first one.
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem) {
+ if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) {
InVectors.push_back(V1);
break;
}
@@ -9334,7 +9334,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
if (auto *FTy = dyn_cast<FixedVectorType>(V1->getType()))
VF = FTy->getNumElements();
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem)
+ if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF);
}
/// Adds another one input vector and the mask for the shuffling.
@@ -9359,13 +9359,13 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
Vec = createShuffle(Vec, nullptr, CommonMask);
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
- if (CommonMask[Idx] != UndefMaskElem)
+ if (CommonMask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx;
assert(VF > 0 &&
"Expected vector length for the final value before action.");
unsigned VecVF = cast<FixedVectorType>(Vec->getType())->getNumElements();
if (VecVF < VF) {
- SmallVector<int> ResizeMask(VF, UndefMaskElem);
+ SmallVector<int> ResizeMask(VF, PoisonMaskElem);
std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0);
Vec = createShuffle(Vec, nullptr, ResizeMask);
}
@@ -9376,9 +9376,9 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
if (CommonMask.empty()) {
CommonMask.assign(ExtMask.begin(), ExtMask.end());
} else {
- SmallVector<int> NewMask(ExtMask.size(), UndefMaskElem);
+ SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
- if (ExtMask[I] == UndefMaskElem)
+ if (ExtMask[I] == PoisonMaskElem)
continue;
NewMask[I] = CommonMask[ExtMask[I]];
}
@@ -9449,18 +9449,18 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
// ... (use %2)
// %shuffle = shuffle <2 x> %2, poison, <2 x> {2, 0}
// br %block
- SmallVector<int> UniqueIdxs(VF, UndefMaskElem);
+ SmallVector<int> UniqueIdxs(VF, PoisonMaskElem);
SmallSet<int, 4> UsedIdxs;
int Pos = 0;
for (int Idx : VE->ReuseShuffleIndices) {
- if (Idx != static_cast<int>(VF) && Idx != UndefMaskElem &&
+ if (Idx != static_cast<int>(VF) && Idx != PoisonMaskElem &&
UsedIdxs.insert(Idx).second)
UniqueIdxs[Idx] = Pos;
++Pos;
}
assert(VF >= UsedIdxs.size() && "Expected vectorization factor "
"less than original vector size.");
- UniqueIdxs.append(VF - UsedIdxs.size(), UndefMaskElem);
+ UniqueIdxs.append(VF - UsedIdxs.size(), PoisonMaskElem);
V = FinalShuffle(V, UniqueIdxs);
} else {
assert(VF < cast<FixedVectorType>(V->getType())->getNumElements() &&
@@ -9543,7 +9543,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
if (It == VectorizableTree.end())
return false;
unsigned I =
- *find_if_not(Mask, [](int Idx) { return Idx == UndefMaskElem; });
+ *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
int Sz = Mask.size();
if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) &&
ShuffleVectorInst::isIdentityMask(Mask))
@@ -9603,10 +9603,10 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
// Restore the mask for previous partially matched values.
for (auto [I, V] : enumerate(E->Scalars)) {
if (isa<PoisonValue>(V)) {
- Mask[I] = UndefMaskElem;
+ Mask[I] = PoisonMaskElem;
continue;
}
- if (Mask[I] == UndefMaskElem)
+ if (Mask[I] == PoisonMaskElem)
Mask[I] = Entries.front()->findLaneForValue(V);
}
ShuffleBuilder.add(Entries.front()->VectorizedValue, Mask);
@@ -9622,7 +9622,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
}
// Remove shuffled elements from list of gathers.
for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
GatheredScalars[I] = PoisonValue::get(ScalarTy);
}
}
@@ -9669,7 +9669,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
if (NumNonConsts == 1) {
// Restore single insert element.
if (IsSplat) {
- ReuseMask.assign(VF, UndefMaskElem);
+ ReuseMask.assign(VF, PoisonMaskElem);
std::swap(Scalars.front(), Scalars[SinglePos]);
if (!UndefPos.empty() && UndefPos.front() == 0)
Scalars.front() = UndefValue::get(ScalarTy);
@@ -9708,7 +9708,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
// Replace undefs by the poisons, emit broadcast and then emit
// freeze.
for_each(UndefPos, [&](int I) {
- ReuseMask[I] = UndefMaskElem;
+ ReuseMask[I] = PoisonMaskElem;
if (isa<UndefValue>(Scalars[I]))
Scalars[I] = PoisonValue::get(ScalarTy);
});
@@ -9726,9 +9726,9 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
// Find input vectors.
Value *Vec2 = nullptr;
for (unsigned I = 0, Sz = ExtractMask.size(); I < Sz; ++I) {
- if (ExtractMask[I] == UndefMaskElem ||
- (!Mask.empty() && Mask[I] != UndefMaskElem)) {
- ExtractMask[I] = UndefMaskElem;
+ if (ExtractMask[I] == PoisonMaskElem ||
+ (!Mask.empty() && Mask[I] != PoisonMaskElem)) {
+ ExtractMask[I] = PoisonMaskElem;
continue;
}
if (isa<UndefValue>(E->Scalars[I]))
@@ -9815,7 +9815,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
}
// Generate constants for final shuffle and build a mask for them.
if (!all_of(GatheredScalars, PoisonValue::classof)) {
- SmallVector<int> BVMask(GatheredScalars.size(), UndefMaskElem);
+ SmallVector<int> BVMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, BVMask, /*IsRootPoison=*/true);
Value *BV = gather(GatheredScalars);
ShuffleBuilder.add(BV, BVMask);
@@ -9835,14 +9835,14 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
});
} else if (!allConstant(GatheredScalars)) {
// Gather unique scalars and all constants.
- SmallVector<int> ReuseMask(GatheredScalars.size(), UndefMaskElem);
+ SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
Vec = gather(GatheredScalars);
ShuffleBuilder.add(Vec, ReuseMask);
Vec = ShuffleBuilder.finalize(E->ReuseShuffleIndices);
} else {
// Gather all constants.
- SmallVector<int> Mask(E->Scalars.size(), UndefMaskElem);
+ SmallVector<int> Mask(E->Scalars.size(), PoisonMaskElem);
for (auto [I, V] : enumerate(E->Scalars)) {
if (!isa<PoisonValue>(V))
Mask[I] = I;
@@ -9986,14 +9986,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
SmallVector<int> Mask;
if (!E->ReorderIndices.empty()) {
inversePermutation(E->ReorderIndices, Mask);
- Mask.append(NumElts - NumScalars, UndefMaskElem);
+ Mask.append(NumElts - NumScalars, PoisonMaskElem);
} else {
- Mask.assign(NumElts, UndefMaskElem);
+ Mask.assign(NumElts, PoisonMaskElem);
std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0);
}
// Create InsertVector shuffle if necessary
bool IsIdentity = true;
- SmallVector<int> PrevMask(NumElts, UndefMaskElem);
+ SmallVector<int> PrevMask(NumElts, PoisonMaskElem);
Mask.swap(PrevMask);
for (unsigned I = 0; I < NumScalars; ++I) {
Value *Scalar = E->Scalars[PrevMask[I]];
@@ -10009,9 +10009,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
- SmallVector<int> InsertMask(NumElts, UndefMaskElem);
+ SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
for (unsigned I = 0; I < NumElts; I++) {
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
InsertMask[Offset + I] = I;
}
SmallBitVector UseMask =
@@ -10026,7 +10026,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
for (unsigned I = 0; I < NumElts; I++) {
- if (InsertMask[I] == UndefMaskElem && !IsFirstPoison.test(I))
+ if (InsertMask[I] == PoisonMaskElem && !IsFirstPoison.test(I))
InsertMask[I] = I + NumElts;
}
}
@@ -10044,8 +10044,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
for (unsigned I = 0; I < NumElts; I++) {
- if (InsertMask[I] == UndefMaskElem)
- InsertMask[I] = IsFirstPoison.test(I) ? UndefMaskElem : I;
+ if (InsertMask[I] == PoisonMaskElem)
+ InsertMask[I] = IsFirstPoison.test(I) ? PoisonMaskElem : I;
else
InsertMask[I] += NumElts;
}
@@ -10707,7 +10707,7 @@ Value *BoUpSLP::vectorizeTree(
ShuffledInserts.size() - 1);
SmallVectorImpl<int> &Mask = It->ValueMasks[Vec];
if (Mask.empty())
- Mask.assign(FTy->getNumElements(), UndefMaskElem);
+ Mask.assign(FTy->getNumElements(), PoisonMaskElem);
// Find the insertvector, vectorized in tree, if any.
Value *Base = VU;
while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
@@ -10720,7 +10720,7 @@ Value *BoUpSLP::vectorizeTree(
do {
IEBase = cast<InsertElementInst>(Base);
int IEIdx = *getInsertIndex(IEBase);
- assert(Mask[Idx] == UndefMaskElem &&
+ assert(Mask[Idx] == PoisonMaskElem &&
"InsertElementInstruction used already.");
Mask[IEIdx] = IEIdx;
Base = IEBase->getOperand(0);
@@ -10738,7 +10738,7 @@ Value *BoUpSLP::vectorizeTree(
}
SmallVectorImpl<int> &Mask = It->ValueMasks[Vec];
if (Mask.empty())
- Mask.assign(FTy->getNumElements(), UndefMaskElem);
+ Mask.assign(FTy->getNumElements(), PoisonMaskElem);
Mask[Idx] = ExternalUse.Lane;
It->InsertElements.push_back(cast<InsertElementInst>(User));
continue;
@@ -10780,8 +10780,8 @@ Value *BoUpSLP::vectorizeTree(
}
auto CreateShuffle = [&](Value *V1, Value *V2, ArrayRef<int> Mask) {
- SmallVector<int> CombinedMask1(Mask.size(), UndefMaskElem);
- SmallVector<int> CombinedMask2(Mask.size(), UndefMaskElem);
+ SmallVector<int> CombinedMask1(Mask.size(), PoisonMaskElem);
+ SmallVector<int> CombinedMask2(Mask.size(), PoisonMaskElem);
int VF = cast<FixedVectorType>(V1->getType())->getNumElements();
for (int I = 0, E = Mask.size(); I < E; ++I) {
if (Mask[I] < VF)
@@ -10806,9 +10806,9 @@ Value *BoUpSLP::vectorizeTree(
return std::make_pair(Vec, true);
}
if (!ForSingleMask) {
- SmallVector<int> ResizeMask(VF, UndefMaskElem);
+ SmallVector<int> ResizeMask(VF, PoisonMaskElem);
for (unsigned I = 0; I < VF; ++I) {
- if (Mask[I] != UndefMaskElem)
+ if (Mask[I] != PoisonMaskElem)
ResizeMask[Mask[I]] = Mask[I];
}
Vec = CreateShuffle(Vec, nullptr, ResizeMask);
@@ -11011,14 +11011,14 @@ void BoUpSLP::optimizeGatherSequence() {
// registers.
unsigned LastUndefsCnt = 0;
for (int I = 0, E = NewMask.size(); I < E; ++I) {
- if (SM1[I] == UndefMaskElem)
+ if (SM1[I] == PoisonMaskElem)
++LastUndefsCnt;
else
LastUndefsCnt = 0;
- if (NewMask[I] != UndefMaskElem && SM1[I] != UndefMaskElem &&
+ if (NewMask[I] != PoisonMaskElem && SM1[I] != PoisonMaskElem &&
NewMask[I] != SM1[I])
return false;
- if (NewMask[I] == UndefMaskElem)
+ if (NewMask[I] == PoisonMaskElem)
NewMask[I] = SM1[I];
}
// Check if the last undefs actually change the final number of used vector
@@ -13804,7 +13804,7 @@ class HorizontalReduction {
// 7>, if elements 4th and 6th elements have even number of repeats.
SmallVector<int> Mask(
cast<FixedVectorType>(VectorizedValue->getType())->getNumElements(),
- UndefMaskElem);
+ PoisonMaskElem);
std::iota(Mask.begin(), Mask.end(), 0);
bool NeedShuffle = false;
for (unsigned I = 0, VF = VL.size(); I < VF; ++I) {
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index f7bd0b1bec2d8..0f9ac2b8e5409 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -244,7 +244,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
// still need a shuffle to change the vector size.
auto *Ty = cast<FixedVectorType>(I.getType());
unsigned OutputNumElts = Ty->getNumElements();
- SmallVector<int, 16> Mask(OutputNumElts, UndefMaskElem);
+ SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
Mask[0] = OffsetEltIndex;
if (OffsetEltIndex)
@@ -480,7 +480,7 @@ static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
// to the new element index. Example for OldIndex == 2 and NewIndex == 0:
// ShufMask = { 2, undef, undef, undef }
auto *VecTy = cast<FixedVectorType>(Vec->getType());
- SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+ SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
ShufMask[NewIndex] = OldIndex;
return Builder.CreateShuffleVector(Vec, ShufMask, "shift");
}
@@ -914,7 +914,7 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
auto *CmpTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(X->getType()));
InstructionCost NewCost = TTI.getCmpSelInstrCost(
CmpOpcode, X->getType(), CmpInst::makeCmpResultType(X->getType()), Pred);
- SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+ SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
ShufMask[CheapIndex] = ExpensiveIndex;
NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy,
ShufMask);
@@ -1593,12 +1593,12 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
V2B.push_back(GetBaseMaskValue(SVI1B, V2[I].first));
}
while (V1A.size() < NumElts) {
- V1A.push_back(UndefMaskElem);
- V1B.push_back(UndefMaskElem);
+ V1A.push_back(PoisonMaskElem);
+ V1B.push_back(PoisonMaskElem);
}
while (V2A.size() < NumElts) {
- V2A.push_back(UndefMaskElem);
- V2B.push_back(UndefMaskElem);
+ V2A.push_back(PoisonMaskElem);
+ V2B.push_back(PoisonMaskElem);
}
auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
More information about the llvm-commits
mailing list