[llvm] [SLP]Add extractelement as a main opcode for copyables (PR #216503)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 15 12:06:22 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/216503
Model a bundle of extractelements from a common vector plus a foreign scalar
as a copyable node: matching lanes reuse the source vector and copyable lanes
are inserted into it. Limited to the identity extract order without a reuse
shuffle, other cases fall back to gather.
Fixes #192849
>From 13b45f58265404eb36d4e029183682ecc9d0b907 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 15 Aug 2026 12:06:10 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 115 ++++++++++++++----
.../AArch64/externally-used-copyables.ll | 2 +-
.../SLPVectorizer/RISCV/load-store.ll | 14 +--
.../X86/copyable-extractelement-in-stores.ll | 18 +--
.../extractelement-single-use-many-nodes.ll | 2 +-
.../SLPVectorizer/X86/reduction-logical.ll | 4 +-
.../X86/reduction-with-removed-extracts.ll | 6 +-
.../X86/same-values-sub-node-with-poisons.ll | 37 +++---
.../X86/select-copyable-cmp-poison.ll | 18 +--
...split-node-reused-and-reordered-operand.ll | 20 +--
.../X86/split-node-reused-in-later-vector.ll | 14 +--
...reduction-gather-non-scheduled-extracts.ll | 12 +-
12 files changed, 162 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1b418ae5185ee..06ea9f8ec9693 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -539,10 +539,13 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
ShuffleMode CommonShuffleMode = Unknown;
Mask.assign(VL.size(), PoisonMaskElem);
for (unsigned I = 0, E = VL.size(); I < E; ++I) {
- // Undef can be represented as an undef element in a vector.
+ // Undef, or a copyable lane modeled on an extract main op, can be
+ // represented as an undef element in a vector.
if (isa<UndefValue>(VL[I]))
continue;
- auto *EI = cast<ExtractElementInst>(VL[I]);
+ auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
+ if (!EI)
+ continue;
if (isa<ScalableVectorType>(EI->getVectorOperandType()))
return std::nullopt;
auto *Vec = EI->getVectorOperand();
@@ -7448,7 +7451,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
SmallVector<int> ReusedMask(TE.ReuseShuffleIndices.begin(),
TE.ReuseShuffleIndices.end());
if (TE.hasState() && TE.getOpcode() == Instruction::ExtractElement &&
- all_of(TE.Scalars, [Sz](Value *V) {
+ !TE.hasCopyableElements() && all_of(TE.Scalars, [Sz](Value *V) {
if (isa<PoisonValue>(V))
return true;
std::optional<unsigned> Idx = getExtractIndex(cast<Instruction>(V));
@@ -7647,7 +7650,10 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
allSameType(TE.Scalars)) {
// TODO: add analysis of other gather nodes with extractelement
// instructions and other values/instructions, not only undefs.
- if (((TE.hasState() && TE.getOpcode() == Instruction::ExtractElement) ||
+ // Nodes with copyable lanes may mix in non-extract lanes, for which the
+ // extract-index order is not applicable.
+ if (((TE.hasState() && TE.getOpcode() == Instruction::ExtractElement &&
+ !TE.hasCopyableElements()) ||
(all_of(TE.Scalars, IsaPred<UndefValue, ExtractElementInst>) &&
any_of(TE.Scalars, IsaPred<ExtractElementInst>))) &&
all_of(TE.Scalars, [](Value *V) {
@@ -10157,6 +10163,8 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
}
case Instruction::ExtractElement:
if (any_of(VL, [&](Value *V) {
+ if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ return false;
auto *EI = dyn_cast<ExtractElementInst>(V);
if (!EI)
return true;
@@ -10170,6 +10178,13 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
[[fallthrough]];
case Instruction::ExtractValue: {
bool Reuse = canReuseExtract(VL, CurrentOrder);
+ // Copyable lanes are inserted into the reused source vector at their own
+ // index, which is correct only for the identity extract order (empty
+ // CurrentOrder) and without a reuse shuffle; other cases fall back to
+ // gather.
+ if (S.areInstructionsWithCopyableElements() &&
+ (!Reuse || !ReuseShuffleIndices.empty()))
+ return TreeEntry::NeedToGather;
if (Reuse || !CurrentOrder.empty())
return TreeEntry::Vectorize;
SmallVector<unsigned> Indices;
@@ -11150,10 +11165,14 @@ class InstructionsCompatibilityAnalysis {
/// Checks if \p I can be the main op for copyable analysis: a supported
/// binary operator, fmuladd, or an integer min/max intrinsic, the only
/// call with a well-defined idempotent value (FP min/max lacks one because of
- /// NaNs).
+ /// NaNs). An extractelement with constant index from a fixed vector is also
+ /// supported: the matching lanes reuse the source vector and the copyable
+ /// lanes are inserted into it.
static bool isSupportedMainOp(Instruction *I) {
return isSupportedOpcode(I->getOpcode()) || isa<MinMaxIntrinsic>(I) ||
- RecurrenceDescriptor::isFMulAddIntrinsic(I);
+ RecurrenceDescriptor::isFMulAddIntrinsic(I) ||
+ (isa<ExtractElementInst>(I) && isVectorLikeInstWithConstOps(I) &&
+ isa<FixedVectorType>(I->getOperand(0)->getType()));
}
/// Identifies the best candidate value, which represents main opcode
@@ -11768,6 +11787,18 @@ class InstructionsCompatibilityAnalysis {
return OrigS;
if (!WithProfitabilityCheck)
return S;
+ // ExtractElement copyable nodes reuse the source vector and insert the
+ // copyable lanes; the binary-operator operand heuristics below do not
+ // apply, so defer profitability to the full tree cost.
+ if (isa<ExtractElementInst>(MainOp)) {
+ // A load as a copyable lane would be pulled out of the consecutive-load
+ // vectorization; keep the original state so the node can be split.
+ if (any_of(VL, [&](Value *V) {
+ return S.isCopyableElement(V) && isa<LoadInst>(V);
+ }))
+ return OrigS;
+ return S;
+ }
// Check if it is profitable to vectorize the instruction.
unsigned CopyableNum =
count_if(VL, [&](Value *V) { return S.isCopyableElement(V); });
@@ -11954,6 +11985,14 @@ class InstructionsCompatibilityAnalysis {
if (S.areInstructionsWithCopyableElements()) {
MainOp = S.getMainOp();
MainOpcode = S.getOpcode();
+ // ExtractElement copyable nodes carry a single operand (the shared
+ // source vector); copyable lanes are inserted during codegen and do
+ // not contribute an operand column.
+ if (MainOpcode == Instruction::ExtractElement) {
+ Operands.assign(1,
+ BoUpSLP::ValueList(VL.size(), MainOp->getOperand(0)));
+ return Operands;
+ }
// Excludes the trailing callee operand (2 for min/max, 3 for fmuladd).
// getNumberOfPotentiallyCommutativeOps collapses fmuladd to 2 and must
// not be used here. Only the 2-operand case is commutative-normalized.
@@ -13504,9 +13543,6 @@ bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL,
const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
assert(It != VL.end() && "Expected at least one extract instruction.");
auto *E0 = cast<Instruction>(*It);
- assert(
- all_of(VL, IsaPred<UndefValue, ExtractElementInst, ExtractValueInst>) &&
- "Invalid opcode");
// Check if all of the extracts come from the same vector and from the
// correct offset.
Value *Vec = E0->getOperand(0);
@@ -13533,9 +13569,11 @@ bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL,
SmallVector<int> Indices(E, PoisonMaskElem);
unsigned MinIdx = NElts, MaxIdx = 0;
for (auto [I, V] : enumerate(VL)) {
- auto *Inst = dyn_cast<Instruction>(V);
- if (!Inst)
+ // Non-extract lanes (copyable elements modeled on an extract main op, or
+ // undefs) are treated as holes.
+ if (!isa<ExtractElementInst, ExtractValueInst>(V))
continue;
+ auto *Inst = cast<Instruction>(V);
if (Inst->getOperand(0) != Vec)
return false;
if (auto *EE = dyn_cast<ExtractElementInst>(Inst))
@@ -17015,11 +17053,21 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (ShuffleOrOp == Instruction::ExtractValue && !E->StructEVIndices.empty())
return CommonCost;
APInt DemandedElts;
+ APInt CopyableInsertElts;
VectorType *SrcVecTy = nullptr;
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
+ // Copyable lanes are not extracts; they are inserted into the reused
+ // source vector, so charge the insert to the vector side only.
+ if (E->isCopyableElement(UniqueValues[Idx])) {
+ if (CopyableInsertElts.isZero())
+ CopyableInsertElts = APInt::getZero(E->getVectorFactor());
+ CopyableInsertElts.setBit(Idx);
+ return InstructionCost(TTI::TCC_Free);
+ }
+
auto *I = cast<Instruction>(UniqueValues[Idx]);
if (!SrcVecTy) {
if (ShuffleOrOp == Instruction::ExtractElement) {
@@ -17058,11 +17106,19 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return InstructionCost(TTI::TCC_Free);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
- return CommonCost - (DemandedElts.isZero()
- ? TTI::TCC_Free
- : TTI.getScalarizationOverhead(
- SrcVecTy, DemandedElts, /*Insert=*/false,
- /*Extract=*/true, CostKind));
+ return CommonCost +
+ (CopyableInsertElts.isZero()
+ ? TTI::TCC_Free
+ : TTI.getScalarizationOverhead(
+ cast<VectorType>(
+ getWidenedType(OrigScalarTy, E->getVectorFactor())),
+ CopyableInsertElts, /*Insert=*/true,
+ /*Extract=*/false, CostKind)) -
+ (DemandedElts.isZero()
+ ? TTI::TCC_Free
+ : TTI.getScalarizationOverhead(SrcVecTy, DemandedElts,
+ /*Insert=*/false,
+ /*Extract=*/true, CostKind));
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
@@ -18082,8 +18138,11 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
[this](Value *V) { return EphValues.contains(V); }) &&
(allConstant(TE->Scalars) || isSplat(TE->Scalars) ||
TE->Scalars.size() < Limit ||
+ // Nodes with copyable lanes may mix in non-extract lanes, which
+ // are not representable as a shuffle of the source vector.
(((TE->hasState() &&
- TE->getOpcode() == Instruction::ExtractElement) ||
+ TE->getOpcode() == Instruction::ExtractElement &&
+ !TE->hasCopyableElements()) ||
all_of(TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
isFixedVectorShuffle(TE->Scalars, Mask, AC)) ||
(TE->hasState() && TE->getOpcode() == Instruction::Load &&
@@ -18470,11 +18529,14 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
// Check if any of the gather node forms an insertelement buildvector
// somewhere. TreeSize >= 1 is guaranteed, so the multi-node case reduces to
// a simple TreeSize > 1 short-circuit.
+ // A gather with copyable lanes is not a real instruction node; do not let
+ // its state qualify it as a buildvector-forming node.
const bool IsAllowedSingleBVNode =
- TreeSize > 1 || (FrontHasState && !Front.isAltShuffle() &&
- FrontOpcode != Instruction::PHI &&
- FrontOpcode != Instruction::GetElementPtr &&
- allSameBlock(Front.Scalars));
+ TreeSize > 1 ||
+ (FrontHasState && !Front.isAltShuffle() && !Front.hasCopyableElements() &&
+ FrontOpcode != Instruction::PHI &&
+ FrontOpcode != Instruction::GetElementPtr &&
+ allSameBlock(Front.Scalars));
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && all_of(TE->Scalars, [&](Value *V) {
return isa<ExtractElementInst, Constant>(V) ||
@@ -23473,6 +23535,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Value *V = E->getSingleOperand(0);
setInsertPointAfterBundle(E);
V = FinalShuffle(V, E);
+ // Insert the copyable lanes (non-extract scalars modeled on the extract
+ // main op) into the reused source vector. The identity extract order and
+ // absence of a reuse shuffle are guaranteed when the node is created, so
+ // a lane maps to its own index.
+ if (E->hasCopyableElements()) {
+ assert(E->ReorderIndices.empty() && E->ReuseShuffleIndices.empty() &&
+ "Copyable extract lanes require identity order and no reuse.");
+ for (auto [Idx, Scalar] : enumerate(E->Scalars))
+ if (E->isCopyableElement(Scalar))
+ V = Builder.CreateInsertElement(V, Scalar, Builder.getInt32(Idx));
+ }
E->VectorizedValue = V;
return V;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index f602ff125bd8c..c98a4d1f2c182 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -66,7 +66,6 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[DIFF_CHECK3817:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP76:%.*]] = add i64 [[TMP53]], 1
; CHECK-NEXT: [[DIFF_CHECK3820:%.*]] = icmp ult i64 [[TMP76]], [[TMP0]]
-; CHECK-NEXT: [[TMP98:%.*]] = shl <2 x i64> [[TMP12]], splat (i64 1)
; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x i64> [[TMP63]], i64 [[INDVAR37888]], i64 0
; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> <i64 1, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 1, i32 poison, i32 poison>
@@ -80,6 +79,7 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP100:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <32 x i64> [[TMP90]], <32 x i64> [[TMP73]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 36, i32 37, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP98:%.*]] = shl <2 x i64> [[TMP12]], splat (i64 1)
; CHECK-NEXT: [[TMP125:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <32 x i64> [[TMP87]], <32 x i64> [[TMP125]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP59:%.*]] = insertelement <32 x i64> [[TMP88]], i64 [[TMP54]], i64 12
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/load-store.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/load-store.ll
index dc022108dfc1e..0a40632c3e4f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/load-store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/load-store.ll
@@ -274,10 +274,9 @@ define void @shared-chain-ordering(ptr %dest, ptr %p, i64 %offset) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P]], align 4
; CHECK-NEXT: [[E1:%.*]] = load i16, ptr [[INC]], align 2
; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[INCS0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i64 0
-; CHECK-NEXT: store i16 [[TMP1]], ptr [[DEST]], align 4
-; CHECK-NEXT: [[INCS:%.*]] = getelementptr inbounds i16, ptr [[DEST]], i64 1
-; CHECK-NEXT: store i16 [[E1]], ptr [[INCS]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[E1]], i64 1
+; CHECK-NEXT: store <2 x i16> [[TMP2]], ptr [[DEST]], align 4
; CHECK-NEXT: ret void
;
; DEFAULT-LABEL: @shared-chain-ordering(
@@ -287,10 +286,9 @@ define void @shared-chain-ordering(ptr %dest, ptr %p, i64 %offset) {
; DEFAULT-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P]], align 4
; DEFAULT-NEXT: [[E1:%.*]] = load i16, ptr [[INC]], align 2
; DEFAULT-NEXT: store <4 x i16> [[TMP0]], ptr [[INCS0]], align 4
-; DEFAULT-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i64 0
-; DEFAULT-NEXT: store i16 [[TMP1]], ptr [[DEST]], align 4
-; DEFAULT-NEXT: [[INCS:%.*]] = getelementptr inbounds i16, ptr [[DEST]], i64 1
-; DEFAULT-NEXT: store i16 [[E1]], ptr [[INCS]], align 2
+; DEFAULT-NEXT: [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 poison>
+; DEFAULT-NEXT: [[TMP2:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[E1]], i64 1
+; DEFAULT-NEXT: store <2 x i16> [[TMP2]], ptr [[DEST]], align 4
; DEFAULT-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable-extractelement-in-stores.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable-extractelement-in-stores.ll
index a377ca2e44f48..09a006e1cf689 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable-extractelement-in-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable-extractelement-in-stores.ll
@@ -12,22 +12,8 @@ define void @test(i8 %s, ptr %out) {
; CHECK-NEXT: [[I0:%.*]] = insertelement <16 x i8> poison, i8 [[S]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <16 x i8> [[I0]], <16 x i8> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: [[V:%.*]] = add <16 x i8> [[SPLAT]], <i8 poison, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>
-; CHECK-NEXT: store i8 [[S]], ptr [[OUT]], align 1
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[V]], <16 x i8> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[P1]], align 1
-; CHECK-NEXT: [[P9:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 9
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[V]], <16 x i8> poison, <4 x i32> <i32 9, i32 10, i32 11, i32 12>
-; CHECK-NEXT: store <4 x i8> [[TMP2]], ptr [[P9]], align 1
-; CHECK-NEXT: [[E13:%.*]] = extractelement <16 x i8> [[V]], i64 13
-; CHECK-NEXT: [[P13:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 13
-; CHECK-NEXT: store i8 [[E13]], ptr [[P13]], align 1
-; CHECK-NEXT: [[E14:%.*]] = extractelement <16 x i8> [[V]], i64 14
-; CHECK-NEXT: [[P14:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 14
-; CHECK-NEXT: store i8 [[E14]], ptr [[P14]], align 1
-; CHECK-NEXT: [[E15:%.*]] = extractelement <16 x i8> [[V]], i64 15
-; CHECK-NEXT: [[P15:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 15
-; CHECK-NEXT: store i8 [[E15]], ptr [[P15]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[V]], <16 x i8> [[I0]], <16 x i32> <i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: store <16 x i8> [[TMP1]], ptr [[OUT]], align 1
; CHECK-NEXT: ret void
;
%i0 = insertelement <16 x i8> poison, i8 %s, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
index 9464b12308034..6bcf25b701ea6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
@@ -7,8 +7,8 @@ define void @foo(double %i) {
; CHECK-NEXT: bb:
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double 0.000000e+00>, double [[I]], i64 2
; CHECK-NEXT: [[TMP1:%.*]] = fsub <4 x double> zeroinitializer, [[TMP0]]
-; CHECK-NEXT: [[I82:%.*]] = fsub double 0.000000e+00, poison
; CHECK-NEXT: [[I103:%.*]] = fsub double 0.000000e+00, [[I]]
+; CHECK-NEXT: [[I82:%.*]] = fsub double 0.000000e+00, poison
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <8 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 0, i32 poison, i32 1>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> <double 0.000000e+00, double poison, double poison, double poison, double 0.000000e+00, double poison, double poison, double poison>, <8 x i32> <i32 8, i32 1, i32 poison, i32 3, i32 12, i32 5, i32 poison, i32 7>
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[I82]], i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
index c10caf1f9d5c3..6b073b10bac9b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
@@ -317,9 +317,7 @@ define i1 @logical_and_icmp_clamp_extra_use_select(<4 x i32> %x) {
define i1 @logical_and_icmp_clamp_v8i32(<8 x i32> %x, <8 x i32> %y) {
; CHECK-LABEL: @logical_and_icmp_clamp_v8i32(
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[X:%.*]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[Y:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> <i32 42, i32 42, i32 42, i32 42, i32 poison, i32 poison, i32 poison, i32 poison>, <8 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[Y:%.*]], <8 x i32> <i32 42, i32 42, i32 42, i32 42, i32 poison, i32 poison, i32 poison, i32 poison>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <8 x i32> [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <8 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP5]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
index 2fb0061c8c518..c1431b8a1cf8b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
@@ -8,7 +8,7 @@ define i32 @test(i32 %arg) {
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 0>, i32 [[ARG]], i64 1
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[OP_RDX5:%.*]], %[[BB1]] ]
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[OP_RDX3:%.*]], %[[BB1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> [[TMP0]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = or <4 x i32> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
@@ -21,8 +21,10 @@ define i32 @test(i32 %arg) {
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[RDX_OP]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP14]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP12]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = mul i32 [[TMP13]], 0
; CHECK-NEXT: [[OP_RDX4:%.*]] = mul i32 0, [[TMP9]]
-; CHECK-NEXT: [[OP_RDX5]] = mul i32 [[OP_RDX4]], [[TMP13]]
+; CHECK-NEXT: [[OP_RDX2:%.*]] = mul i32 [[OP_RDX]], 0
+; CHECK-NEXT: [[OP_RDX3]] = mul i32 [[OP_RDX2]], [[OP_RDX4]]
; CHECK-NEXT: br label %[[BB1]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
index 0708f24c238f2..dbda518f04dce 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
@@ -13,7 +13,7 @@ define i32 @test(ptr %f, i1 %tobool.i.4, i32 %retval.0.i.219) {
; CHECK-NEXT: br i1 false, label %[[D_EXIT_3]], label %[[D_EXIT_6:.*]]
; CHECK: [[D_EXIT_3]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[ENTRY]] ], [ poison, %[[IF_END_I_1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 1, i32 0>, i32 [[RETVAL_0_I_219]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[RETVAL_0_I_219]], i64 0
; CHECK-NEXT: br i1 [[TOBOOL_I_4]], label %[[D_EXIT_4:.*]], label %[[D_EXIT_6]]
; CHECK: [[D_EXIT_4]]:
; CHECK-NEXT: br label %[[D_EXIT_6]]
@@ -21,22 +21,29 @@ define i32 @test(ptr %f, i1 %tobool.i.4, i32 %retval.0.i.219) {
; CHECK-NEXT: br i1 false, label %[[D_EXIT_6]], label %[[D_EXIT_7:.*]]
; CHECK: [[D_EXIT_6]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP1]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ [[TMP1]], %[[D_EXIT_4]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP2]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ zeroinitializer, %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP2]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
; CHECK-NEXT: br label %[[D_EXIT_7]]
; CHECK: [[D_EXIT_7]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ [[TMP3]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP4]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 1, i32 1, i32 3, i32 3>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <12 x i32> [[TMP9]], <12 x i32> [[TMP10]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <12 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <12 x i32> [[TMP12]], <12 x i32> <i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 1, i32 poison, i32 1, i32 1, i32 poison>, <12 x i32> <i32 0, i32 1, i32 14, i32 15, i32 16, i32 5, i32 6, i32 19, i32 poison, i32 21, i32 22, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <12 x i32> [[TMP13]], i32 [[TMP0]], i64 8
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <12 x i32> [[TMP18]], i32 [[RETVAL_0_I_219]], i64 11
-; CHECK-NEXT: [[TMP16:%.*]] = add <12 x i32> [[TMP15]], [[TMP11]]
-; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v12i32(<12 x i32> [[TMP16]])
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ [[TMP3]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP6]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[RETVAL_0_I_219]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i32> [[TMP11]], [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> <i32 poison, i32 1, i32 poison, i32 1, i32 1, i32 poison, i32 poison, i32 1>, i32 [[RETVAL_0_I_219]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 3, i32 4, i32 0, i32 0, i32 7>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[TMP15]], <4 x i32> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
+; CHECK-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP14]], [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i32> [[TMP20]], [[TMP12]]
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i32> [[RDX_OP]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> [[TMP21]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP22]])
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll
index 155603d1f2a04..20942c45f4409 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll
@@ -35,15 +35,15 @@ define i32 @foo(i32 %v2, ptr byval(%struct.S2) align 8 %v5, i1 %cmp3644) {
; CHECK-NEXT: [[TMP10:%.*]] = icmp slt <8 x i32> [[TMP35]], zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP34]], <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, <8 x i32> <i32 8, i32 9, i32 0, i32 3, i32 0, i32 5, i32 6, i32 15>
; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP9]], <8 x i32> [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = load <21 x i32>, ptr [[V5]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <21 x i32> [[TMP13]], <21 x i32> poison, <4 x i32> <i32 0, i32 1, i32 14, i32 20>
-; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i32> [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = srem <4 x i32> [[TMP15]], splat (i32 1)
-; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> zeroinitializer, <4 x i32> zeroinitializer, <4 x i32> [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP12]], <8 x i32> poison, <4 x i32> <i32 2, i32 5, i32 6, i32 3>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP18:%.*]] = load <21 x i32>, ptr [[V5]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <21 x i32> [[TMP18]], <21 x i32> poison, <4 x i32> <i32 0, i32 1, i32 14, i32 20>
+; CHECK-NEXT: [[TMP20:%.*]] = freeze <4 x i32> [[TMP19]]
+; CHECK-NEXT: [[TMP36:%.*]] = srem <4 x i32> [[TMP20]], splat (i32 1)
+; CHECK-NEXT: [[TMP37:%.*]] = select <4 x i1> zeroinitializer, <4 x i32> zeroinitializer, <4 x i32> [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <8 x i32> [[TMP12]], <8 x i32> poison, <4 x i32> <i32 2, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <4 x i32> [[TMP38]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> [[TMP38]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP22:%.*]] = mul <8 x i32> zeroinitializer, [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = urem <8 x i32> [[TMP22]], splat (i32 46337)
; CHECK-NEXT: [[TMP24:%.*]] = mul <8 x i32> [[TMP12]], [[TMP23]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
index a057d8415cf64..4610dabeb51b8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
@@ -7,12 +7,13 @@ define i32 @test(i1 %tobool46.not) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[U:.*]]
; CHECK: [[U]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 undef, i32 undef>, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 undef, i32 undef>, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: br i1 false, label %[[IF_END3:.*]], label %[[IF_END51:.*]]
; CHECK: [[IF_END3]]:
; CHECK-NEXT: br i1 false, label %[[AB:.*]], label %[[AA:.*]]
@@ -21,18 +22,19 @@ define i32 @test(i1 %tobool46.not) {
; CHECK: [[IF_END44]]:
; CHECK-NEXT: br i1 [[TOBOOL46_NOT]], label %[[AA]], label %[[U]]
; CHECK: [[AA]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: br i1 false, label %[[IF_END51]], label %[[V:.*]]
; CHECK: [[V]]:
; CHECK-NEXT: br label %[[IF_END51]]
; CHECK: [[IF_END51]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP6]], %[[AA]] ], [ [[TMP7]], %[[V]] ], [ [[TMP0]], %[[U]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <8 x i32> [ [[TMP13]], %[[AA]] ], [ zeroinitializer, %[[V]] ], [ [[TMP5]], %[[U]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP15]], %[[AA]] ], [ [[TMP16]], %[[V]] ], [ [[TMP2]], %[[U]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <8 x i32> [ [[TMP13]], %[[AA]] ], [ zeroinitializer, %[[V]] ], [ [[TMP6]], %[[U]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP11]])
; CHECK-NEXT: ret i32 [[TMP12]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
index d805e95c28079..a03800b2eebcc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
@@ -4,18 +4,16 @@
define i1 @test() {
; CHECK-LABEL: define i1 @test() {
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 0, 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i1> <i1 poison, i1 false>, i1 [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 1, i32 0>, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 1, i32 0
; CHECK-NEXT: [[TMP20:%.*]] = freeze <8 x i32> poison
; CHECK-NEXT: [[TMP21:%.*]] = select <8 x i1> zeroinitializer, <8 x i32> [[TMP20]], <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> <i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP22:%.*]] = mul <8 x i32> [[TMP11]], zeroinitializer
; CHECK-NEXT: [[TMP23:%.*]] = urem <8 x i32> [[TMP22]], splat (i32 46337)
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <8 x i32> [[TMP23]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 2, i32 5, i32 3, i32 6, i32 7>
-; CHECK-NEXT: [[TMP25:%.*]] = add <8 x i32> [[TMP21]], [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = or <8 x i32> [[TMP21]], [[TMP24]]
+; CHECK-NEXT: [[TMP25:%.*]] = add <8 x i32> [[TMP21]], [[TMP23]]
+; CHECK-NEXT: [[TMP26:%.*]] = or <8 x i32> [[TMP21]], [[TMP23]]
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <8 x i32> [[TMP25]], <8 x i32> [[TMP26]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 5, i32 14, i32 7>
; CHECK-NEXT: [[TMP28:%.*]] = urem <8 x i32> [[TMP27]], splat (i32 46337)
; CHECK-NEXT: [[TMP29:%.*]] = icmp ult <8 x i32> [[TMP28]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll b/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
index 5834ad3a106e8..18f08ae9824f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
@@ -26,16 +26,14 @@ define void @tes() {
; AARCH64-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> zeroinitializer, i64 0
; AARCH64-NEXT: br label [[TMP5:%.*]]
; AARCH64: 5:
-; AARCH64-NEXT: [[TMP6:%.*]] = select i1 false, i1 false, i1 false
-; AARCH64-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i1 [[TMP0]], i1 false
-; AARCH64-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i1 [[TMP1]], i1 false
-; AARCH64-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i1 false, i1 false
+; AARCH64-NEXT: [[OP_RDX:%.*]] = select i1 [[TMP0]], i1 [[TMP1]], i1 false
+; AARCH64-NEXT: [[TMP9:%.*]] = select i1 [[OP_RDX]], i1 [[TMP4]], i1 false
; AARCH64-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i1 [[TMP3]], i1 false
-; AARCH64-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i1 [[TMP4]], i1 false
+; AARCH64-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i1 false, i1 false
; AARCH64-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]]
-; AARCH64: 12:
+; AARCH64: 6:
; AARCH64-NEXT: ret void
-; AARCH64: 13:
+; AARCH64: 7:
; AARCH64-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list