[llvm] [SLP] Vectorize full insertvalue buildvector sequences (PR #200274)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 11:58:27 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/200274
>From fa12c63583fbaacacfa009f9e6076d12a3db53a2 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 28 May 2026 13:57:42 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 219 +++++++++++++++---
llvm/test/Transforms/PhaseOrdering/X86/avg.ll | 28 +--
.../Transforms/SLPVectorizer/X86/PR35777.ll | 10 +-
.../SLPVectorizer/X86/insertvalue.ll | 10 +-
4 files changed, 209 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9d7821d2760f2..07435a20f9fc9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -322,6 +322,8 @@ static Type *getValueType(Value *V, bool LookThroughCmp = false) {
if (!SLPReVec)
if (auto *IE = dyn_cast<InsertElementInst>(V))
return IE->getOperand(1)->getType();
+ if (auto *IV = dyn_cast<InsertValueInst>(V))
+ return IV->getOperand(1)->getType();
return V->getType();
}
@@ -508,18 +510,21 @@ static bool isConstant(Value *V) {
/// insertelement/extractelement with constant indices for fixed vector type or
/// extractvalue instruction.
static bool isVectorLikeInstWithConstOps(Value *V) {
- if (!isa<InsertElementInst, ExtractElementInst>(V) &&
+ if (!isa<InsertElementInst, InsertValueInst, ExtractElementInst>(V) &&
!isa<ExtractValueInst, UndefValue>(V))
return false;
auto *I = dyn_cast<Instruction>(V);
if (!I || isa<ExtractValueInst>(I))
return true;
- if (!isa<FixedVectorType>(I->getOperand(0)->getType()))
- return false;
if (isa<ExtractElementInst>(I))
return isConstant(I->getOperand(1));
- assert(isa<InsertElementInst>(V) && "Expected only insertelement.");
- return isConstant(I->getOperand(2));
+ if (isa<InsertElementInst>(V)) {
+ if (!isa<FixedVectorType>(I->getOperand(0)->getType()))
+ return false;
+ return isConstant(I->getOperand(2));
+ }
+ assert(isa<InsertValueInst>(I) && "Expected InsertValueInst");
+ return true;
}
/// Returns power-of-2 number of elements in a single register (part), given the
@@ -2419,6 +2424,11 @@ class slpvectorizer::BoUpSLP {
/// \returns number of elements in vector if isomorphism exists, 0 otherwise.
unsigned canMapToVector(Type *T) const;
+ /// \returns true if the vectorized insertvalue result can be stored directly
+ /// as a vector, i.e. every insertvalue with an external user is consumed by a
+ /// single store only.
+ bool canVectorStoreInsertValue(const TreeEntry *E) const;
+
/// \returns True if the VectorizableTree is both tiny and not fully
/// vectorizable. We do not vectorize such trees.
bool isTreeTinyAndNotFullyVectorizable(bool ForReduction = false) const;
@@ -8385,7 +8395,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
TE.State == TreeEntry::StridedVectorize ||
TE.State == TreeEntry::CompressVectorize) &&
(isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
- (TopToBottom && isa<StoreInst, InsertElementInst>(TE.getMainOp()))))) {
+ (TopToBottom && isa<StoreInst, InsertElementInst, InsertValueInst>(
+ TE.getMainOp()))))) {
assert((TE.State == TreeEntry::SplitVectorize || !TE.isAltShuffle()) &&
"Alternate instructions are only supported by "
"BinaryOperator and CastInst.");
@@ -8826,6 +8837,7 @@ void BoUpSLP::reorderTopToBottom() {
const bool IgnoreReorder =
!UserIgnoreList && VectorizableTree.front()->hasState() &&
(VectorizableTree.front()->getOpcode() == Instruction::InsertElement ||
+ VectorizableTree.front()->getOpcode() == Instruction::InsertValue ||
VectorizableTree.front()->getOpcode() == Instruction::Store);
// Find all reorderable nodes with the given VF.
// Currently the are vectorized stores,loads,extracts + some gathering of
@@ -9062,7 +9074,7 @@ void BoUpSLP::reorderTopToBottom() {
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::CompressVectorize) &&
(isa<ExtractElementInst, ExtractValueInst, LoadInst, StoreInst,
- InsertElementInst>(TE->getMainOp()) ||
+ InsertElementInst, InsertValueInst>(TE->getMainOp()) ||
(SLPReVec && isa<ShuffleVectorInst>(TE->getMainOp()))))) {
assert(
(!TE->isAltShuffle() || (TE->State == TreeEntry::SplitVectorize &&
@@ -9072,7 +9084,7 @@ void BoUpSLP::reorderTopToBottom() {
// Build correct orders for extract{element,value}, loads,
// stores and alternate (split) nodes.
reorderOrder(TE->ReorderIndices, Mask);
- if (isa<InsertElementInst, StoreInst>(TE->getMainOp()))
+ if (isa<InsertElementInst, InsertValueInst, StoreInst>(TE->getMainOp()))
TE->reorderOperands(Mask);
} else {
// Reorder the node and its operands.
@@ -9117,7 +9129,9 @@ void BoUpSLP::buildReorderableOperands(
if (UserTE->getOpcode() == Instruction::ExtractElement ||
UserTE->getOpcode() == Instruction::ExtractValue)
continue;
- if (UserTE->getOpcode() == Instruction::InsertElement && I == 0)
+ if ((UserTE->getOpcode() == Instruction::InsertElement ||
+ UserTE->getOpcode() == Instruction::InsertValue) &&
+ I == 0)
continue;
if (UserTE->getOpcode() == Instruction::Store && I == 1 &&
(UserTE->State == TreeEntry::Vectorize ||
@@ -9541,7 +9555,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
Data.first->getMainOp()) ||
IsNotProfitableAltCodeNode(*Data.first))
Data.first->reorderOperands(Mask);
- if (!isa<InsertElementInst, StoreInst>(Data.first->getMainOp()) ||
+ if (!isa<InsertElementInst, InsertValueInst, StoreInst>(
+ Data.first->getMainOp()) ||
IsNotProfitableAltCodeNode(*Data.first) ||
Data.first->State == TreeEntry::CompressVectorize) {
reorderScalars(Data.first->Scalars, Mask);
@@ -10780,13 +10795,30 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
return TreeEntry::NeedToGather;
}
+ case Instruction::InsertValue:
+ // Handle only simple insertvalue with one index and non-vector.
+ // TODO: Support more complex insertvalues.
+ if (any_of(VL,
+ [](Value *V) {
+ auto *IV = dyn_cast<InsertValueInst>(V);
+ return IV && (IV->getNumIndices() != 1 ||
+ IV->getOperand(1)->getType()->isVectorTy());
+ }) ||
+ none_of(VL, [](Value *V) {
+ auto *IV = dyn_cast<InsertValueInst>(V);
+ return IV && isa<UndefValue>(IV->getAggregateOperand());
+ }))
+ return TreeEntry::NeedToGather;
+ [[fallthrough]];
case Instruction::InsertElement: {
// Check that we have a buildvector and not a shuffle of 2 or more
// different vectors.
ValueSet SourceVectors;
for (Value *V : VL) {
if (isa<PoisonValue>(V)) {
- LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement/poison vector.\n");
+ LLVM_DEBUG(
+ dbgs()
+ << "SLP: Gather of insertelement/insertvalue/poison vector.\n");
return TreeEntry::NeedToGather;
}
SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
@@ -10798,18 +10830,21 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
return !SourceVectors.contains(V);
}) >= 2) {
// Found 2nd source vector - cancel.
- LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
- "different source vectors.\n");
+ LLVM_DEBUG(
+ dbgs() << "SLP: Gather of insertelement/insertvalue vectors with "
+ "different source vectors.\n");
return TreeEntry::NeedToGather;
}
if (any_of(VL, [&SourceVectors](Value *V) {
- // The last InsertElement can have multiple uses.
+ // The last InsertElement/InsertValue can have multiple uses.
return SourceVectors.contains(V) && !V->hasOneUse();
})) {
- assert(SLPReVec && "Only supported by REVEC.");
- LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
- "multiple uses.\n");
+ assert((SLPReVec || ShuffleOrOp == Instruction::InsertValue) &&
+ "Only supported by REVEC or InsertValue.");
+ LLVM_DEBUG(
+ dbgs() << "SLP: Gather of insertelement/insertvalue vectors with "
+ "multiple uses.\n");
return TreeEntry::NeedToGather;
}
@@ -11856,6 +11891,14 @@ class InstructionsCompatibilityAnalysis {
// we are not extending buildTree_rec() towards the operands.
Operands.assign(1, {VL.size(), VL0->getOperand(0)});
return;
+ case Instruction::InsertValue:
+ Operands.assign(2, {VL.size(), nullptr});
+ for (auto [Idx, V] : enumerate(VL)) {
+ auto *IE = cast<InsertValueInst>(V);
+ for (auto [OpIdx, Ops] : enumerate(Operands))
+ Ops[Idx] = IE->getOperand(OpIdx);
+ }
+ return;
case Instruction::InsertElement:
Operands.assign(2, {VL.size(), nullptr});
for (auto [Idx, V] : enumerate(VL)) {
@@ -13034,6 +13077,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
}
return;
}
+ case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(ReuseShuffleIndices.empty() && "All inserts should be unique");
@@ -13423,6 +13467,38 @@ unsigned BoUpSLP::canMapToVector(Type *T) const {
return N;
}
+bool BoUpSLP::canVectorStoreInsertValue(const TreeEntry *E) const {
+ if (E->getOpcode() != Instruction::InsertValue ||
+ E->State != TreeEntry::Vectorize)
+ return false;
+
+ bool HasStore = false;
+ for (Value *V : E->Scalars) {
+ auto *IV = dyn_cast<InsertValueInst>(V);
+ if (!IV || isDeleted(IV))
+ return false;
+
+ bool HasExternalUser = false;
+ for (User *U : IV->users()) {
+ if (auto *UI = dyn_cast<Instruction>(U); UI && isDeleted(UI))
+ continue;
+ if (is_contained(E->Scalars, U))
+ continue;
+ HasExternalUser = true;
+ // Only a plain (non-volatile, non-atomic) store may consume the vector
+ // directly. Volatile/atomic stores must keep their original aggregate
+ // type and access semantics, so fall back to the store + load roundtrip.
+ auto *SI = dyn_cast<StoreInst>(U);
+ if (!SI || !SI->isSimple())
+ return false;
+ HasStore = true;
+ }
+ if (HasExternalUser && !IV->hasOneUse())
+ return false;
+ }
+ return HasStore;
+}
+
bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL,
SmallVectorImpl<unsigned> &CurrentOrder,
bool ResizeAllowed) const {
@@ -13651,6 +13727,15 @@ unsigned BoUpSLP::getNumVectorInsts() const {
if (TE.getOpcode() == Instruction::InsertElement ||
TE.getOpcode() == Instruction::ExtractElement)
continue;
+ if (TE.getOpcode() == Instruction::InsertValue) {
+ // InsertValue materializes via store + load unless it is stored directly
+ // as a vector.
+ if (!canVectorStoreInsertValue(&TE))
+ Count += 2;
+ if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
+ ++Count;
+ continue;
+ }
if (TE.State == TreeEntry::SplitVectorize)
Count += 2;
else
@@ -16581,7 +16666,9 @@ BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
// InsertElement operand 0 is the vector being inserted into, which is
// built incrementally and does not occupy an extra register.
- if (E->getOpcode() == Instruction::InsertElement && Idx == 0)
+ if ((E->getOpcode() == Instruction::InsertElement ||
+ E->getOpcode() == Instruction::InsertValue) &&
+ Idx == 0)
continue;
ArrayRef<Value *> Ops = E->getOperand(Idx);
if (Ops.empty() || allConstant(Ops) || isSplat(Ops))
@@ -16698,7 +16785,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (E->isGather() || TransformedToGatherNodes.contains(E)) {
if (allConstant(VL))
return 0;
- if (isa<InsertElementInst>(VL[0]))
+ if (isa<InsertElementInst, InsertValueInst>(VL[0]))
return InstructionCost::getInvalid();
return SpillsReloads +
processBuildVector<ShuffleCostEstimator, InstructionCost>(
@@ -16989,11 +17076,25 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
+ case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(E->ReuseShuffleIndices.empty() &&
"Unique insertelements only are expected.");
- auto *SrcVecTy = cast<FixedVectorType>(VL0->getType());
- unsigned const NumElts = SrcVecTy->getNumElements();
+ FixedVectorType *SrcVecTy = nullptr;
+ if (ShuffleOrOp == Instruction::InsertElement) {
+ SrcVecTy = cast<FixedVectorType>(VL0->getType());
+ } else {
+ unsigned MaxIdx = VL.size() - 1;
+ for (Value *V : VL) {
+ auto *I = dyn_cast<InsertValueInst>(V);
+ if (!I)
+ continue;
+ MaxIdx = std::max(MaxIdx, I->getIndices().front());
+ }
+ SrcVecTy =
+ cast<FixedVectorType>(getWidenedType(getValueType(VL0), MaxIdx + 1));
+ }
+ unsigned const NumElts = getNumElements(SrcVecTy);
unsigned const NumScalars = VL.size();
unsigned NumOfParts =
@@ -17096,6 +17197,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
}
}
+ if (ShuffleOrOp == Instruction::InsertValue &&
+ !canVectorStoreInsertValue(E)) {
+ Align VecAlign = DL->getPrefTypeAlign(VecTy);
+ Cost += TTI->getMemoryOpCost(Instruction::Store, SrcVecTy, VecAlign,
+ /*AddressSpace=*/0, CostKind) +
+ TTI->getMemoryOpCost(Instruction::Load, VL0->getType(), VecAlign,
+ /*AddressSpace=*/0, CostKind);
+ }
return Cost + SpillsReloads;
}
case Instruction::ZExt:
@@ -19023,6 +19132,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
GatheredLoadsNodes.insert(&TE);
if (!TE.isGather() && TE.State != TreeEntry::SplitVectorize &&
!(TE.Idx == 0 && (TE.getOpcode() == Instruction::InsertElement ||
+ TE.getOpcode() == Instruction::InsertValue ||
TE.getOpcode() == Instruction::Store)) &&
!isa<StructType>(getValueType(TE.Scalars.front()))) {
// Calculate costs of external uses.
@@ -19492,6 +19602,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
if (any_of(ExternalUses, [](const ExternalUser &EU) {
return isa<StructType>(EU.Scalar->getType()) &&
+ (EU.E.Idx != 0 || EU.E.State != TreeEntry::Vectorize ||
+ EU.E.getOpcode() != Instruction::InsertValue) &&
!isa_and_nonnull<ExtractValueInst>(EU.User);
}))
return InstructionCost::getInvalid();
@@ -19584,6 +19696,9 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
if (!SLPReVec && isa<FixedVectorType>(EU.Scalar->getType()))
continue;
+ if (isa<InsertValueInst>(EU.Scalar))
+ continue;
+
// If found user is an insertelement, do not calculate extract cost but try
// to detect it as a final shuffled/identity match.
// TODO: what if a user is insertvalue when REVEC is enabled?
@@ -23087,6 +23202,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
E->VectorizedValue = NewV;
return NewV;
}
+ case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(E->ReuseShuffleIndices.empty() && "All inserts should be unique");
if (const TreeEntry *OpE = getOperandEntry(E, 1);
@@ -23114,9 +23230,22 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
auto *FirstInsert = cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
}));
- const unsigned NumElts =
- cast<FixedVectorType>(FirstInsert->getType())->getNumElements();
const unsigned NumScalars = E->Scalars.size();
+ FixedVectorType *SrcVecTy = nullptr;
+ if (ShuffleOrOp == Instruction::InsertElement) {
+ SrcVecTy = cast<FixedVectorType>(VL0->getType());
+ } else {
+ unsigned MaxIdx = NumScalars - 1;
+ for (Value *V : E->Scalars) {
+ auto *I = dyn_cast<InsertValueInst>(V);
+ if (!I)
+ continue;
+ MaxIdx = std::max(MaxIdx, I->getIndices().front());
+ }
+ SrcVecTy = cast<FixedVectorType>(
+ getWidenedType(getValueType(VL0), MaxIdx + 1));
+ }
+ const unsigned NumElts = getNumElements(SrcVecTy);
unsigned Offset = *getElementIndex(VL0);
assert(Offset < NumElts && "Failed to find vector index offset");
@@ -23148,7 +23277,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if (NumElts != NumScalars && Offset == 0) {
// Follow all insert element instructions from the current buildvector
// sequence.
- InsertElementInst *Ins = cast<InsertElementInst>(VL0);
+ Instruction *Ins = VL0;
do {
std::optional<unsigned> InsertIdx = getElementIndex(Ins);
if (!InsertIdx)
@@ -23157,8 +23286,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
InsertMask[*InsertIdx] = *InsertIdx;
if (!Ins->hasOneUse())
break;
- Ins = dyn_cast_or_null<InsertElementInst>(
- Ins->getUniqueUndroppableUser());
+ Ins =
+ dyn_cast_or_null<Instruction>(Ins->getUniqueUndroppableUser());
} while (Ins);
SmallBitVector UseMask =
buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask);
@@ -23250,6 +23379,25 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
++NumVectorInstructions;
+ if (ShuffleOrOp == Instruction::InsertValue &&
+ !canVectorStoreInsertValue(E)) {
+ Type *AggTy = E->getMainOp()->getType();
+ Align SlotAlign = std::max(DL->getPrefTypeAlign(V->getType()),
+ DL->getPrefTypeAlign(AggTy));
+ AllocaInst *Slot;
+ {
+ // Keep the alloca in the function entry block so it is not executed
+ // (and does not grow the stack) on every iteration of a loop.
+ IRBuilderBase::InsertPointGuard Guard(Builder);
+ Builder.SetInsertPointPastAllocas(F);
+ Slot = Builder.CreateAlloca(AggTy, /*ArraySize=*/nullptr,
+ "vec2struct.slot");
+ Slot->setAlignment(SlotAlign);
+ }
+ (void)Builder.CreateAlignedStore(V, Slot, SlotAlign);
+ V = Builder.CreateAlignedLoad(AggTy, Slot, SlotAlign, "vec2struct");
+ NumVectorInstructions += 2;
+ }
E->VectorizedValue = V;
return V;
}
@@ -24255,6 +24403,8 @@ Value *BoUpSLP::vectorizeTree(
Value *Lane = Builder.getInt32(ExternalUse.Lane);
auto ExtractAndExtendIfNeeded = [&](Value *Vec) {
+ if (isa<InsertValueInst>(Scalar))
+ return Vec;
if (Scalar->getType() != Vec->getType()) {
Value *Ex = nullptr;
Value *ExV = nullptr;
@@ -26738,7 +26888,8 @@ void BoUpSLP::computeMinimumValueSizes() {
bool IsStoreOrInsertElt =
VectorizableTree.front()->hasState() &&
(VectorizableTree.front()->getOpcode() == Instruction::Store ||
- VectorizableTree.front()->getOpcode() == Instruction::InsertElement);
+ VectorizableTree.front()->getOpcode() == Instruction::InsertElement ||
+ VectorizableTree.front()->getOpcode() == Instruction::InsertValue);
if ((IsStoreOrInsertElt || UserIgnoreList) &&
ExtraBitWidthNodes.size() <= 1 &&
(!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 ||
@@ -28114,7 +28265,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
// determining vectorization factor for scalar instructions.
for (Value *V : VL) {
Type *Ty = V->getType();
- if (!isa<InsertElementInst>(V) && !isValidElementType(Ty)) {
+ if (!isa<InsertElementInst, InsertValueInst>(V) &&
+ !isValidElementType(Ty)) {
// NOTE: the following will give user internal llvm type name, which may
// not be useful.
R.getORE()->emit([&]() {
@@ -28193,7 +28345,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
continue;
if (R.isProfitableToReorder()) {
R.reorderTopToBottom();
- R.reorderBottomToTop(!isa<InsertElementInst>(Ops.front()));
+ R.reorderBottomToTop(
+ !isa<InsertElementInst, InsertValueInst>(Ops.front()));
}
R.transformNodes();
R.computeMinimumValueSizes();
@@ -31150,6 +31303,14 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
}
LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
// Aggregate value is unlikely to be processed in vector register.
+ Type *BVType = BuildVectorInsts.front()->getType();
+ ArrayRef<Type *> ContainedTypes = getContainedTypes(BVType);
+ if (ContainedTypes.size() == BuildVectorInsts.size() &&
+ all_of(ContainedTypes, [&](Type *Ty) {
+ return Ty == ContainedTypes.front() && isValidElementType(Ty);
+ }))
+ if (tryToVectorizeList(BuildVectorInsts, R, MaxVFOnly))
+ return true;
return tryToVectorizeList(BuildVectorOpds, R, MaxVFOnly);
}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 0b0ade31da145..270b17cd18085 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -17,42 +17,43 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-LABEL: @avgr_16_u8(
; SSE2-NEXT: entry:
; SSE2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
+; SSE2-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
+; SSE2-NEXT: [[TMP15:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i16
+; SSE2-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP15]], i64 1
; SSE2-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
-; SSE2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1:%.*]], i64 1
+; SSE2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1]], i64 1
; SSE2-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 16)
; SSE2-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE2-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
; SSE2-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
; SSE2-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
-; SSE2-NEXT: [[TMP7:%.*]] = trunc i64 [[A_COERCE1]] to i16
; SSE2-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
; SSE2-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
+; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
+; SSE2-NEXT: [[TMP21:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
+; SSE2-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP19]], i16 [[TMP21]], i64 1
; SSE2-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
-; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1:%.*]], i64 1
+; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1]], i64 1
; SSE2-NEXT: [[TMP11:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 16)
; SSE2-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE2-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE2-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
; SSE2-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE2-NEXT: [[TMP15:%.*]] = trunc i64 [[B_COERCE1]] to i16
; SSE2-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
; SSE2-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE2-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
-; SSE2-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1
-; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8)
+; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
+; SSE2-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE2-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
; SSE2-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56
-; SSE2-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
-; SSE2-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1
-; SSE2-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
; SSE2-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE2-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE2-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE2-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE2-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
+; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP18]], splat (i16 8)
+; SSE2-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
; SSE2-NEXT: [[TMP27:%.*]] = add nuw nsw <2 x i16> [[TMP20]], splat (i16 1)
; SSE2-NEXT: [[TMP28:%.*]] = add nuw nsw <2 x i16> [[TMP27]], [[TMP23]]
; SSE2-NEXT: [[TMP29:%.*]] = and <2 x i64> [[TMP3]], splat (i64 255)
@@ -71,7 +72,6 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE2-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE2-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE2-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE2-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE2-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
; SSE2-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
@@ -141,6 +141,8 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
; SSE4-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE4-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
+; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
+; SSE4-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; SSE4-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1
; SSE4-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8)
@@ -150,7 +152,6 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1
; SSE4-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE4-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
@@ -173,7 +174,6 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE4-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE4-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE4-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
; SSE4-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
index 05511f843a68f..bc0022253c23b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
@@ -6,6 +6,7 @@
define { i64, i64 } @patatino(double %arg) {
; CHECK-LABEL: @patatino(
; CHECK-NEXT: bb:
+; CHECK-NEXT: [[VEC2STRUCT_SLOT:%.*]] = alloca { i64, i64 }, align 16
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr @global, align 16
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 2), align 16
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[ARG:%.*]], i32 0
@@ -15,12 +16,9 @@ define { i64, i64 } @patatino(double %arg) {
; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16
; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32>
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0
-; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
-; CHECK-NEXT: [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64>
+; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[VEC2STRUCT_SLOT]], align 16
+; CHECK-NEXT: [[T17:%.*]] = load { i64, i64 }, ptr [[VEC2STRUCT_SLOT]], align 16
; CHECK-NEXT: ret { i64, i64 } [[T17]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
index 4c5815c0bd144..7609d2b3d4a40 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
@@ -230,15 +230,7 @@ define void @julia_load_struct_of_float(ptr %a, ptr %b, ptr %c) {
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A:%.*]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[B:%.*]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i32 0
-; CHECK-NEXT: [[C_STRUCT0:%.*]] = insertvalue [[PSEUDOVEC:%.*]] undef, float [[TMP5]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP4]], i32 1
-; CHECK-NEXT: [[C_STRUCT1:%.*]] = insertvalue [[PSEUDOVEC]] [[C_STRUCT0]], float [[TMP6]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
-; CHECK-NEXT: [[C_STRUCT2:%.*]] = insertvalue [[PSEUDOVEC]] [[C_STRUCT1]], float [[TMP7]], 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x float> [[TMP4]], i32 3
-; CHECK-NEXT: [[C_STRUCT3:%.*]] = insertvalue [[PSEUDOVEC]] [[C_STRUCT2]], float [[TMP8]], 3
-; CHECK-NEXT: store [[PSEUDOVEC]] [[C_STRUCT3]], ptr [[C:%.*]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[C:%.*]], align 4
; CHECK-NEXT: ret void
;
top:
More information about the llvm-commits
mailing list