[llvm] [SLP]Vectorize operand chains of non-vectorizable instructions (PR #194248)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 26 10:43:32 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/194248
>From 0e141ad86e368b11f593d6043fe16b9084c355f5 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 26 Apr 2026 10:11:06 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 6 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 246 +++++++++++++++++-
.../SLPVectorizer/AArch64/gather-cost.ll | 2 -
...otriviallyvectorizableintrinsicoperands.ll | 159 +++++------
.../RISCV/revec-strided-store.ll | 7 +-
.../Transforms/SLPVectorizer/RISCV/revec.ll | 7 +-
.../RISCV/strided-unsupported-type.ll | 6 +-
.../test/Transforms/SLPVectorizer/X86/call.ll | 20 +-
.../X86/control-dependent-schedule.ll | 6 +-
.../control-deps-schedule-data-recalculate.ll | 5 +-
.../SLPVectorizer/X86/conversion-fp16.ll | 18 +-
.../Transforms/SLPVectorizer/X86/lookahead.ll | 159 +++++++----
.../X86/non-vectorizable-inst-operand.ll | 81 +++---
.../Transforms/SLPVectorizer/X86/odd_store.ll | 20 +-
.../SLPVectorizer/X86/operandorder.ll | 12 +-
.../X86/parent-node-non-schedulable.ll | 72 +++--
.../X86/phi-comparator-fix-vec-ops-compare.ll | 14 +-
.../SLPVectorizer/X86/schedule_budget.ll | 17 +-
.../Transforms/SLPVectorizer/X86/simplebb.ll | 13 +-
.../X86/split-node-marked-to-gather.ll | 26 +-
.../SLPVectorizer/semanticly-same.ll | 7 +-
21 files changed, 554 insertions(+), 349 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 877c83291170b..db822dc345395 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -143,6 +143,12 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
bool vectorizeCmpInsts(iterator_range<ItT> CmpInsts, BasicBlock *BB,
slpvectorizer::BoUpSLP &R);
+ /// Tries to vectorize the operand chains of the non-vectorizable
+ /// instructions in \p Insts.
+ template <typename ItT>
+ bool vectorizeNonVectorizableInsts(iterator_range<ItT> Insts, BasicBlock *BB,
+ slpvectorizer::BoUpSLP &R);
+
/// Tries to vectorize constructs started from InsertValueInst or
/// InsertElementInst instructions.
bool vectorizeInserts(InstSetVector &Instructions, BasicBlock *BB,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 13cab4fd1e5c8..5817dcae33cab 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30082,6 +30082,216 @@ bool SLPVectorizerPass::vectorizeCmpInsts(iterator_range<ItT> CmpInsts,
return Changed;
}
+/// Returns true if \p I is an instruction whose result the SLP vectorizer
+/// cannot turn into a vector instruction directly, but whose operand chains
+/// may still be worth vectorizing as bundle seeds.
+static bool isNonVectorizableInst(const Instruction *I,
+ const TargetLibraryInfo *TLI) {
+ if (const auto *CB = dyn_cast<CallBase>(I)) {
+ if (CB->isInlineAsm())
+ return false;
+ if (const auto *II = dyn_cast<IntrinsicInst>(CB)) {
+ if (II->isAssumeLikeIntrinsic())
+ return false;
+ if (isa<AnyMemIntrinsic>(II))
+ return false;
+ }
+ if (const auto *CI = dyn_cast<CallInst>(CB)) {
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
+ if (isTriviallyVectorizable(ID))
+ return false;
+ if (!VFDatabase::getMappings(*CI).empty())
+ return false;
+ }
+ // Skip vector-returning calls in non-revec mode - we cannot turn their
+ // results into wider vectors here.
+ return SLPReVec || !CB->getType()->isVectorTy();
+ }
+ if (isa<AtomicRMWInst, AtomicCmpXchgInst>(I))
+ return true;
+ if (const auto *RI = dyn_cast<ReturnInst>(I))
+ return RI->getNumOperands() > 0;
+ return false;
+}
+
+/// Visits the value operands of \p I that are candidates for operand-chain
+/// vectorization.
+template <typename Func>
+static void forEachOperandChainCandidate(Instruction *I, Func F) {
+ if (auto *CB = dyn_cast<CallBase>(I)) {
+ for (auto [Idx, U] : enumerate(CB->args()))
+ F(U.get(), Idx);
+ return;
+ }
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ F(SI->getValueOperand(), 0);
+ return;
+ }
+ if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
+ F(AI->getValOperand(), 0);
+ return;
+ }
+ if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
+ F(AI->getCompareOperand(), 0);
+ F(AI->getNewValOperand(), 1);
+ return;
+ }
+ if (auto *RI = dyn_cast<ReturnInst>(I)) {
+ if (RI->getNumOperands() > 0)
+ F(RI->getReturnValue(), 0);
+ return;
+ }
+ for (auto [Idx, U] : enumerate(I->operands()))
+ F(U.get(), Idx);
+}
+
+template <typename ItT>
+bool SLPVectorizerPass::vectorizeNonVectorizableInsts(
+ iterator_range<ItT> InstRange, BasicBlock *BB, BoUpSLP &R) {
+ SmallVector<Instruction *> Insts;
+ for (Instruction *I : InstRange)
+ Insts.push_back(I);
+ if (Insts.empty())
+ return false;
+ stable_sort(Insts, [](const Instruction *A, const Instruction *B) {
+ return A->getOpcode() < B->getOpcode();
+ });
+
+ bool Changed = false;
+ // Pass 1 - try to find horizontal reductions feeding the root operands.
+ SmallPtrSet<Value *, 8> RootSeen;
+ for (Instruction *I : Insts) {
+ if (R.isDeleted(I))
+ continue;
+ bool RootDeleted = false;
+ forEachOperandChainCandidate(I, [&](Value *Op, unsigned /*Position*/) {
+ if (RootDeleted)
+ return;
+ auto *RootOp = dyn_cast<Instruction>(Op);
+ if (!RootOp || RootOp->getParent() != BB || R.isDeleted(RootOp) ||
+ isa<ShuffleVectorInst>(RootOp) ||
+ !isValidElementType(RootOp->getType()))
+ return;
+ if (!RootSeen.insert(RootOp).second)
+ return;
+ Changed |= vectorizeRootInstruction(nullptr, RootOp, BB, R);
+ if (R.isDeleted(I))
+ RootDeleted = true;
+ });
+ }
+ // Pass 2 - collect the operand instructions across all roots and try to
+ // vectorize them as bundles.
+ struct OperandGroupKey {
+ enum class Kind : unsigned {
+ NonCall = 0,
+ Intrinsic,
+ NamedFunction,
+ IndirectCall,
+ };
+ Kind RootKind;
+ unsigned KindID; // Intrinsic ID, opcode (+ atomicrmw op) or value ID.
+ StringRef FuncName; // Non-empty only for NamedFunction.
+ unsigned Position; // Operand slot within the root.
+
+ bool operator==(const OperandGroupKey &O) const {
+ return RootKind == O.RootKind && KindID == O.KindID &&
+ FuncName == O.FuncName && Position == O.Position;
+ }
+ bool operator!=(const OperandGroupKey &O) const { return !(*this == O); }
+ bool less(const OperandGroupKey &O) const {
+ if (RootKind != O.RootKind)
+ return static_cast<unsigned>(RootKind) <
+ static_cast<unsigned>(O.RootKind);
+ if (KindID != O.KindID)
+ return KindID < O.KindID;
+ if (int C = FuncName.compare(O.FuncName))
+ return C < 0;
+ return Position < O.Position;
+ }
+ };
+ SmallDenseMap<Value *, OperandGroupKey> OpKeys;
+
+ auto BuildKey = [](Instruction *I, unsigned Position) -> OperandGroupKey {
+ if (auto *CB = dyn_cast<CallBase>(I)) {
+ if (auto *II = dyn_cast<IntrinsicInst>(CB))
+ return {OperandGroupKey::Kind::Intrinsic,
+ II->getIntrinsicID(),
+ {},
+ Position};
+ if (Function *F = CB->getCalledFunction())
+ return {OperandGroupKey::Kind::NamedFunction, 0, F->getName(),
+ Position};
+ return {OperandGroupKey::Kind::IndirectCall,
+ CB->getCalledOperand()->getValueID(),
+ {},
+ Position};
+ }
+ unsigned Discriminator = I->getOpcode();
+ if (auto *AI = dyn_cast<AtomicRMWInst>(I))
+ Discriminator |= static_cast<unsigned>(AI->getOperation()) << 16;
+ return {OperandGroupKey::Kind::NonCall, Discriminator, {}, Position};
+ };
+
+ auto OperandSorter = [&OpKeys](Value *V1, Value *V2) -> bool {
+ if (V1 == V2)
+ return false;
+ const OperandGroupKey &K1 = OpKeys.at(V1);
+ const OperandGroupKey &K2 = OpKeys.at(V2);
+ if (K1 != K2)
+ return K1.less(K2);
+ auto *I1 = cast<Instruction>(V1);
+ auto *I2 = cast<Instruction>(V2);
+ if (I1->getType()->getTypeID() != I2->getType()->getTypeID())
+ return I1->getType()->getTypeID() < I2->getType()->getTypeID();
+ if (I1->getType()->getScalarSizeInBits() !=
+ I2->getType()->getScalarSizeInBits())
+ return I1->getType()->getScalarSizeInBits() <
+ I2->getType()->getScalarSizeInBits();
+ if (I1->getOpcode() != I2->getOpcode())
+ return I1->getOpcode() < I2->getOpcode();
+ return I1->comesBefore(I2);
+ };
+
+ auto AreCompatibleOperands = [&OpKeys](ArrayRef<Value *> VL,
+ Value *V) -> bool {
+ if (VL.empty() || VL.back() == V)
+ return true;
+ const OperandGroupKey &KBack = OpKeys.at(VL.back());
+ const OperandGroupKey &K = OpKeys.at(V);
+ if (KBack != K)
+ return false;
+ auto *I1 = cast<Instruction>(VL.back());
+ auto *I2 = cast<Instruction>(V);
+ return I1->getType() == I2->getType() && I1->getOpcode() == I2->getOpcode();
+ };
+
+ SmallVector<Value *> Operands;
+ SmallPtrSet<Value *, 8> Seen;
+ for (Instruction *I : Insts) {
+ if (R.isDeleted(I))
+ continue;
+ forEachOperandChainCandidate(I, [&](Value *Op, unsigned Position) {
+ auto *OpI = dyn_cast<Instruction>(Op);
+ if (!OpI || OpI->getParent() != BB || R.isDeleted(OpI) ||
+ isa<ShuffleVectorInst>(OpI) || !isValidElementType(OpI->getType()))
+ return;
+ if (!Seen.insert(OpI).second)
+ return;
+ OpKeys.try_emplace(OpI, BuildKey(I, Position));
+ Operands.push_back(OpI);
+ });
+ }
+ if (Operands.size() <= 1)
+ return Changed;
+ Changed |= tryToVectorizeSequence<Value>(
+ Operands, OperandSorter, AreCompatibleOperands,
+ [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
+ return tryToVectorizeList(Candidates, R, MaxVFOnly);
+ },
+ /*MaxVFOnly=*/true, R);
+ return Changed;
+}
+
bool SLPVectorizerPass::vectorizeInserts(InstSetVector &Instructions,
BasicBlock *BB, BoUpSLP &R) {
assert(all_of(Instructions, IsaPred<InsertElementInst, InsertValueInst>) &&
@@ -30354,21 +30564,32 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
InstSetVector PostProcessInserts;
SmallSetVector<CmpInst *, 8> PostProcessCmps;
- // Vectorizes Inserts in `PostProcessInserts` and if `VectorizeCmps` is true
- // also vectorizes `PostProcessCmps`.
- auto VectorizeInsertsAndCmps = [&](bool VectorizeCmps) {
+ // Non-vectorizable root instructions other than stores: calls
+ // (regular and intrinsic), invokes, callbrs, atomic RMW/cmpxchg, and
+ // returns.
+ SmallSetVector<Instruction *, 8> PostProcessInsts;
+ // Stores are processed after all other instructions/roots.
+ SmallSetVector<StoreInst *, 8> PostProcessStores;
+ auto VectorizeInsertsAndCmps = [&](bool AtTerminator) {
bool Changed = vectorizeInserts(PostProcessInserts, BB, R);
- if (VectorizeCmps) {
+ if (AtTerminator) {
Changed |= vectorizeCmpInsts(reverse(PostProcessCmps), BB, R);
PostProcessCmps.clear();
+ Changed |=
+ vectorizeNonVectorizableInsts(reverse(PostProcessInsts), BB, R);
+ PostProcessInsts.clear();
}
PostProcessInserts.clear();
return Changed;
};
- // Returns true if `I` is in `PostProcessInserts` or `PostProcessCmps`.
+ // Returns true if `I` is in any of the post-process sets.
auto IsInPostProcessInstrs = [&](Instruction *I) {
if (auto *Cmp = dyn_cast<CmpInst>(I))
return PostProcessCmps.contains(Cmp);
+ if (PostProcessInsts.contains(I))
+ return true;
+ if (auto *SI = dyn_cast<StoreInst>(I))
+ return PostProcessStores.contains(SI);
return isa<InsertElementInst, InsertValueInst>(I) &&
PostProcessInserts.contains(I);
};
@@ -30391,7 +30612,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// We may go through BB multiple times so skip the one we have checked.
if (!VisitedInstrs.insert(&*It).second) {
if (HasNoUsers(&*It) &&
- VectorizeInsertsAndCmps(/*VectorizeCmps=*/It->isTerminator())) {
+ VectorizeInsertsAndCmps(/*AtTerminator=*/It->isTerminator())) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
Changed = true;
@@ -30471,7 +30692,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// top-tree instructions to try to vectorize as many instructions as
// possible.
OpsChanged |=
- VectorizeInsertsAndCmps(/*VectorizeCmps=*/It->isTerminator());
+ VectorizeInsertsAndCmps(/*AtTerminator=*/It->isTerminator());
if (OpsChanged) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
@@ -30484,10 +30705,17 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
if (isa<InsertElementInst, InsertValueInst>(It))
PostProcessInserts.insert(&*It);
- else if (isa<CmpInst>(It))
- PostProcessCmps.insert(cast<CmpInst>(&*It));
+ else if (auto *CI = dyn_cast<CmpInst>(It))
+ PostProcessCmps.insert(CI);
+ else if (auto *SI = dyn_cast<StoreInst>(It))
+ PostProcessStores.insert(SI);
+ else if (isNonVectorizableInst(&*It, TLI))
+ PostProcessInsts.insert(&*It);
}
+ // Late post-process: run operand-chain vectorization for stores.
+ Changed |= vectorizeNonVectorizableInsts(reverse(PostProcessStores), BB, R);
+
return Changed;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
index c7eaeee5a0618..931667ed61dc3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -12,8 +12,6 @@ target triple = "aarch64--linux-gnu"
; REMARK-NEXT: - String: 'Vectorized horizontal reduction with cost '
; REMARK-NEXT: - Cost: '-8'
;
-; REMARK-NOT: Function: gather_load
-
define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
; CHECK-LABEL: @gather_multiple_use(
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/notriviallyvectorizableintrinsicoperands.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/notriviallyvectorizableintrinsicoperands.ll
index 26d61cb6bc28d..37cbea147570d 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/notriviallyvectorizableintrinsicoperands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/notriviallyvectorizableintrinsicoperands.ll
@@ -63,18 +63,14 @@ define amdgpu_kernel void @test_amdgcn_exp_log(ptr addrspace(1) %input, ptr addr
; GCN-LABEL: define amdgpu_kernel void @test_amdgcn_exp_log(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IN0:%.*]] = load float, ptr addrspace(1) [[INPUT]], align 4
-; GCN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[IN1:%.*]] = load float, ptr addrspace(1) [[PTR1]], align 4
-; GCN-NEXT: [[SCALE0:%.*]] = load float, ptr addrspace(1) [[SCALES]], align 4
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[SCALE1:%.*]] = load float, ptr addrspace(1) [[SPTR1]], align 4
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract float [[IN0]], 0x3FC0527DC0000000
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract float [[IN1]], 0x3FC0527DC0000000
-; GCN-NEXT: [[SUB0:%.*]] = fsub contract float [[MUL0]], [[SCALE0]]
-; GCN-NEXT: [[SUB1:%.*]] = fsub contract float [[MUL1]], [[SCALE1]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[INPUT]], align 4
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[SCALES]], align 4
+; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[TMP0]], splat (float 0x3FC0527DC0000000)
+; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x float> [[TMP2]], [[TMP1]]
+; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
; GCN-NEXT: [[LOG0:%.*]] = tail call float @llvm.amdgcn.log.f32(float [[EXP0]])
+; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB1]])
; GCN-NEXT: [[LOG1:%.*]] = tail call float @llvm.amdgcn.log.f32(float [[EXP1]])
; GCN-NEXT: [[SUM:%.*]] = fadd fast float [[LOG0]], [[LOG1]]
@@ -105,17 +101,13 @@ define amdgpu_kernel void @test_amdgcn_exp_f16(ptr addrspace(1) %input, ptr addr
; GCN-LABEL: define amdgpu_kernel void @test_amdgcn_exp_f16(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IN0:%.*]] = load half, ptr addrspace(1) [[INPUT]], align 2
-; GCN-NEXT: [[PTR1:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[IN1:%.*]] = load half, ptr addrspace(1) [[PTR1]], align 2
-; GCN-NEXT: [[SCALE0:%.*]] = load half, ptr addrspace(1) [[SCALES]], align 2
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[SCALE1:%.*]] = load half, ptr addrspace(1) [[SPTR1]], align 2
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract half [[IN0]], 0xH3E14
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract half [[IN1]], 0xH3E14
-; GCN-NEXT: [[SUB0:%.*]] = fsub contract half [[MUL0]], [[SCALE0]]
-; GCN-NEXT: [[SUB1:%.*]] = fsub contract half [[MUL1]], [[SCALE1]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
+; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 0xH3E14)
+; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
+; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
+; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
; GCN-NEXT: [[SUM:%.*]] = fadd fast half [[EXP0]], [[EXP1]]
; GCN-NEXT: store half [[SUM]], ptr addrspace(1) [[OUTPUT]], align 2
@@ -143,17 +135,13 @@ define amdgpu_kernel void @kernel_f16(ptr addrspace(1) %input, ptr addrspace(1)
; GCN-LABEL: define amdgpu_kernel void @kernel_f16(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IN0:%.*]] = load half, ptr addrspace(1) [[INPUT]], align 2
-; GCN-NEXT: [[PTR1:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[IN1:%.*]] = load half, ptr addrspace(1) [[PTR1]], align 2
-; GCN-NEXT: [[SCALE0:%.*]] = load half, ptr addrspace(1) [[SCALES]], align 2
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[SCALE1:%.*]] = load half, ptr addrspace(1) [[SPTR1]], align 2
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract half [[IN0]], 0xH3E14
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract half [[IN1]], 0xH3E14
-; GCN-NEXT: [[SUB0:%.*]] = fsub contract half [[MUL0]], [[SCALE0]]
-; GCN-NEXT: [[SUB1:%.*]] = fsub contract half [[MUL1]], [[SCALE1]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
+; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 0xH3E14)
+; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
+; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
+; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
; GCN-NEXT: [[LOG0:%.*]] = tail call half @llvm.amdgcn.log.f16(half [[EXP0]])
; GCN-NEXT: [[LOG1:%.*]] = tail call half @llvm.amdgcn.log.f16(half [[EXP1]])
@@ -185,17 +173,13 @@ define amdgpu_kernel void @look_through_reuse_shuffle(
; GCN-LABEL: define amdgpu_kernel void @look_through_reuse_shuffle(
; GCN-SAME: ptr addrspace(1) noalias [[INPUT:%.*]], ptr addrspace(1) noalias [[SCALES:%.*]], ptr addrspace(1) noalias [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IPTR1:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[IN0:%.*]] = load half, ptr addrspace(1) [[INPUT]], align 2
-; GCN-NEXT: [[IN1:%.*]] = load half, ptr addrspace(1) [[IPTR1]], align 2
-; GCN-NEXT: [[S0:%.*]] = load half, ptr addrspace(1) [[SCALES]], align 2
-; GCN-NEXT: [[S1:%.*]] = load half, ptr addrspace(1) [[SPTR1]], align 2
-; GCN-NEXT: [[ADD0:%.*]] = fadd contract half [[IN0]], 0xH3E14
-; GCN-NEXT: [[ADD1:%.*]] = fadd contract half [[IN1]], 0xH3E14
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract half [[ADD0]], [[S0]]
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract half [[ADD1]], [[S1]]
+; GCN-NEXT: [[TMP4:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
+; GCN-NEXT: [[TMP5:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
+; GCN-NEXT: [[TMP6:%.*]] = fadd contract <2 x half> [[TMP4]], splat (half 0xH3E14)
+; GCN-NEXT: [[TMP3:%.*]] = fmul contract <2 x half> [[TMP6]], [[TMP5]]
+; GCN-NEXT: [[MUL0:%.*]] = extractelement <2 x half> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[MUL0]])
+; GCN-NEXT: [[MUL1:%.*]] = extractelement <2 x half> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[MUL1]])
; GCN-NEXT: [[TMP0:%.*]] = insertelement <4 x half> poison, half [[EXP0]], i32 0
; GCN-NEXT: [[TMP1:%.*]] = insertelement <4 x half> [[TMP0]], half [[EXP1]], i32 1
@@ -237,31 +221,23 @@ define amdgpu_kernel void @wider_exp2_f32(ptr addrspace(1) %input, ptr addrspace
; GCN-LABEL: define amdgpu_kernel void @wider_exp2_f32(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IN0:%.*]] = load float, ptr addrspace(1) [[INPUT]], align 4
-; GCN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[IN1:%.*]] = load float, ptr addrspace(1) [[PTR1]], align 4
; GCN-NEXT: [[PTR2:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 2
-; GCN-NEXT: [[IN2:%.*]] = load float, ptr addrspace(1) [[PTR2]], align 4
-; GCN-NEXT: [[PTR3:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 3
-; GCN-NEXT: [[IN3:%.*]] = load float, ptr addrspace(1) [[PTR3]], align 4
-; GCN-NEXT: [[SCALE0:%.*]] = load float, ptr addrspace(1) [[SCALES]], align 4
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[SCALE1:%.*]] = load float, ptr addrspace(1) [[SPTR1]], align 4
; GCN-NEXT: [[SPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[SCALES]], i64 2
-; GCN-NEXT: [[SCALE2:%.*]] = load float, ptr addrspace(1) [[SPTR2]], align 4
-; GCN-NEXT: [[SPTR3:%.*]] = getelementptr float, ptr addrspace(1) [[SCALES]], i64 3
-; GCN-NEXT: [[SCALE3:%.*]] = load float, ptr addrspace(1) [[SPTR3]], align 4
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract float [[IN0]], 0x3FC0527DC0000000
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract float [[IN1]], 0x3FC0527DC0000000
-; GCN-NEXT: [[MUL2:%.*]] = fmul contract float [[IN2]], 0x3FC0527DC0000000
-; GCN-NEXT: [[MUL3:%.*]] = fmul contract float [[IN3]], 0x3FC0527DC0000000
-; GCN-NEXT: [[SUB0:%.*]] = fsub contract float [[MUL0]], [[SCALE0]]
-; GCN-NEXT: [[SUB1:%.*]] = fsub contract float [[MUL1]], [[SCALE1]]
-; GCN-NEXT: [[SUB2:%.*]] = fsub contract float [[MUL2]], [[SCALE2]]
-; GCN-NEXT: [[SUB3:%.*]] = fsub contract float [[MUL3]], [[SCALE3]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[INPUT]], align 4
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[SCALES]], align 4
+; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[TMP0]], splat (float 0x3FC0527DC0000000)
+; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x float> [[TMP2]], [[TMP1]]
+; GCN-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[PTR2]], align 4
+; GCN-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[SPTR2]], align 4
+; GCN-NEXT: [[TMP6:%.*]] = fmul contract <2 x float> [[TMP4]], splat (float 0x3FC0527DC0000000)
+; GCN-NEXT: [[TMP7:%.*]] = fsub contract <2 x float> [[TMP6]], [[TMP5]]
+; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
+; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB1]])
+; GCN-NEXT: [[SUB2:%.*]] = extractelement <2 x float> [[TMP7]], i32 0
; GCN-NEXT: [[EXP2:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB2]])
+; GCN-NEXT: [[SUB3:%.*]] = extractelement <2 x float> [[TMP7]], i32 1
; GCN-NEXT: [[EXP3:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB3]])
; GCN-NEXT: [[SUM01:%.*]] = fadd fast float [[EXP0]], [[EXP1]]
; GCN-NEXT: [[SUM23:%.*]] = fadd fast float [[EXP2]], [[EXP3]]
@@ -313,31 +289,23 @@ define amdgpu_kernel void @wider_exp2_half(ptr addrspace(1) %input, ptr addrspac
; GCN-LABEL: define amdgpu_kernel void @wider_exp2_half(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[IN0:%.*]] = load half, ptr addrspace(1) [[INPUT]], align 2
-; GCN-NEXT: [[PTR1:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 1
-; GCN-NEXT: [[IN1:%.*]] = load half, ptr addrspace(1) [[PTR1]], align 2
; GCN-NEXT: [[PTR2:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 2
-; GCN-NEXT: [[IN2:%.*]] = load half, ptr addrspace(1) [[PTR2]], align 2
-; GCN-NEXT: [[PTR3:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 3
-; GCN-NEXT: [[IN3:%.*]] = load half, ptr addrspace(1) [[PTR3]], align 2
-; GCN-NEXT: [[SCALE0:%.*]] = load half, ptr addrspace(1) [[SCALES]], align 2
-; GCN-NEXT: [[SPTR1:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 1
-; GCN-NEXT: [[SCALE1:%.*]] = load half, ptr addrspace(1) [[SPTR1]], align 2
; GCN-NEXT: [[SPTR2:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 2
-; GCN-NEXT: [[SCALE2:%.*]] = load half, ptr addrspace(1) [[SPTR2]], align 2
-; GCN-NEXT: [[SPTR3:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 3
-; GCN-NEXT: [[SCALE3:%.*]] = load half, ptr addrspace(1) [[SPTR3]], align 2
-; GCN-NEXT: [[MUL0:%.*]] = fmul contract half [[IN0]], 0xH3E14
-; GCN-NEXT: [[MUL1:%.*]] = fmul contract half [[IN1]], 0xH3E14
-; GCN-NEXT: [[MUL2:%.*]] = fmul contract half [[IN2]], 0xH3E14
-; GCN-NEXT: [[MUL3:%.*]] = fmul contract half [[IN3]], 0xH3E14
-; GCN-NEXT: [[SUB0:%.*]] = fsub contract half [[MUL0]], [[SCALE0]]
-; GCN-NEXT: [[SUB1:%.*]] = fsub contract half [[MUL1]], [[SCALE1]]
-; GCN-NEXT: [[SUB2:%.*]] = fsub contract half [[MUL2]], [[SCALE2]]
-; GCN-NEXT: [[SUB3:%.*]] = fsub contract half [[MUL3]], [[SCALE3]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
+; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 0xH3E14)
+; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
+; GCN-NEXT: [[TMP4:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR2]], align 2
+; GCN-NEXT: [[TMP5:%.*]] = load <2 x half>, ptr addrspace(1) [[SPTR2]], align 2
+; GCN-NEXT: [[TMP6:%.*]] = fmul contract <2 x half> [[TMP4]], splat (half 0xH3E14)
+; GCN-NEXT: [[TMP7:%.*]] = fsub contract <2 x half> [[TMP6]], [[TMP5]]
+; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i32 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
+; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i32 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
+; GCN-NEXT: [[SUB2:%.*]] = extractelement <2 x half> [[TMP7]], i32 0
; GCN-NEXT: [[EXP2:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB2]])
+; GCN-NEXT: [[SUB3:%.*]] = extractelement <2 x half> [[TMP7]], i32 1
; GCN-NEXT: [[EXP3:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB3]])
; GCN-NEXT: [[SUM01:%.*]] = fadd fast half [[EXP0]], [[EXP1]]
; GCN-NEXT: [[SUM23:%.*]] = fadd fast half [[EXP2]], [[EXP3]]
@@ -391,23 +359,21 @@ define amdgpu_kernel void @kernel_div_scale(ptr addrspace(1) %num, ptr addrspace
; GCN-LABEL: define amdgpu_kernel void @kernel_div_scale(
; GCN-SAME: ptr addrspace(1) [[NUM:%.*]], ptr addrspace(1) [[DEN:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[N0:%.*]] = load float, ptr addrspace(1) [[NUM]], align 4
-; GCN-NEXT: [[NPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[NUM]], i64 1
-; GCN-NEXT: [[N1:%.*]] = load float, ptr addrspace(1) [[NPTR1]], align 4
; GCN-NEXT: [[NPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[NUM]], i64 2
; GCN-NEXT: [[N2:%.*]] = load float, ptr addrspace(1) [[NPTR2]], align 4
-; GCN-NEXT: [[D0:%.*]] = load float, ptr addrspace(1) [[DEN]], align 4
-; GCN-NEXT: [[DPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[DEN]], i64 1
-; GCN-NEXT: [[D1:%.*]] = load float, ptr addrspace(1) [[DPTR1]], align 4
; GCN-NEXT: [[DPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[DEN]], i64 2
; GCN-NEXT: [[D2:%.*]] = load float, ptr addrspace(1) [[DPTR2]], align 4
-; GCN-NEXT: [[MUL_N0:%.*]] = fmul float [[N0]], 2.000000e+00
-; GCN-NEXT: [[MUL_N1:%.*]] = fmul float [[N1]], 2.000000e+00
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[NUM]], align 4
+; GCN-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[TMP0]], splat (float 2.000000e+00)
; GCN-NEXT: [[MUL_N2:%.*]] = fmul float [[N2]], 2.000000e+00
-; GCN-NEXT: [[MUL_D0:%.*]] = fmul float [[D0]], 4.000000e+00
-; GCN-NEXT: [[MUL_D1:%.*]] = fmul float [[D1]], 4.000000e+00
+; GCN-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[DEN]], align 4
+; GCN-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], splat (float 4.000000e+00)
; GCN-NEXT: [[MUL_D2:%.*]] = fmul float [[D2]], 4.000000e+00
+; GCN-NEXT: [[MUL_N0:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; GCN-NEXT: [[MUL_D0:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
; GCN-NEXT: [[DS0:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N0]], float [[MUL_D0]], i1 false)
+; GCN-NEXT: [[MUL_N1:%.*]] = extractelement <2 x float> [[TMP1]], i32 1
+; GCN-NEXT: [[MUL_D1:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
; GCN-NEXT: [[DS1:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N1]], float [[MUL_D1]], i1 false)
; GCN-NEXT: [[DS2:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N2]], float [[MUL_D2]], i1 false)
; GCN-NEXT: [[R0:%.*]] = extractvalue { float, i1 } [[DS0]], 0
@@ -451,20 +417,17 @@ define amdgpu_kernel void @kernel_fmed3(ptr addrspace(1) %a, ptr addrspace(1) %b
; GCN-LABEL: define amdgpu_kernel void @kernel_fmed3(
; GCN-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
; GCN-NEXT: [[ENTRY:.*:]]
-; GCN-NEXT: [[A0:%.*]] = load float, ptr addrspace(1) [[A]], align 4
-; GCN-NEXT: [[APTR1:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 1
-; GCN-NEXT: [[A1:%.*]] = load float, ptr addrspace(1) [[APTR1]], align 4
; GCN-NEXT: [[APTR2:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 2
; GCN-NEXT: [[A2:%.*]] = load float, ptr addrspace(1) [[APTR2]], align 4
-; GCN-NEXT: [[B0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
-; GCN-NEXT: [[BPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 1
-; GCN-NEXT: [[B1:%.*]] = load float, ptr addrspace(1) [[BPTR1]], align 4
; GCN-NEXT: [[BPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 2
; GCN-NEXT: [[B2:%.*]] = load float, ptr addrspace(1) [[BPTR2]], align 4
-; GCN-NEXT: [[ADD0:%.*]] = fadd float [[A0]], [[B0]]
-; GCN-NEXT: [[ADD1:%.*]] = fadd float [[A1]], [[B1]]
+; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[A]], align 4
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[B]], align 4
+; GCN-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP0]], [[TMP1]]
; GCN-NEXT: [[ADD2:%.*]] = fadd float [[A2]], [[B2]]
+; GCN-NEXT: [[ADD0:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
; GCN-NEXT: [[MED0:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD0]], float [[ADD0]], float 1.000000e+00)
+; GCN-NEXT: [[ADD1:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; GCN-NEXT: [[MED1:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD1]], float [[ADD1]], float 1.000000e+00)
; GCN-NEXT: [[MED2:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD2]], float [[ADD2]], float 1.000000e+00)
; GCN-NEXT: [[SUM01:%.*]] = fadd float [[MED0]], [[MED1]]
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
index be1417d85b6f0..47b806acb437a 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
@@ -5,11 +5,12 @@
define void @strided_load_and_store(ptr %in, ptr %out) {
; CHECK-LABEL: @strided_load_and_store(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[IN:%.*]], i64 16
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[IN]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 2 [[IN:%.*]], i64 16, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i64> [[TMP0]] to <16 x i8>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[OUT:%.*]], i64 16
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[OUT]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: store <8 x i8> [[TMP2]], ptr [[TMP3]], align 2
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
index e13dfce8c29f3..016726e5ae371 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
@@ -177,11 +177,10 @@ define ptr @test4() {
; NONPOWEROF2-NEXT: [[TMP8:%.*]] = phi <6 x float> [ poison, [[TMP6:%.*]] ], [ [[TMP5]], [[TMP0:%.*]] ]
; NONPOWEROF2-NEXT: br label [[TMP9:%.*]]
; NONPOWEROF2: 10:
-; NONPOWEROF2-NEXT: [[TMP10:%.*]] = shufflevector <6 x float> [[TMP8]], <6 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; NONPOWEROF2-NEXT: [[TMP11:%.*]] = fmul <3 x float> zeroinitializer, [[TMP10]]
-; NONPOWEROF2-NEXT: [[TMP12:%.*]] = shufflevector <6 x float> [[TMP8]], <6 x float> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; NONPOWEROF2-NEXT: [[TMP13:%.*]] = fmul <3 x float> zeroinitializer, [[TMP12]]
+; NONPOWEROF2-NEXT: [[TMP12:%.*]] = fmul <6 x float> zeroinitializer, [[TMP8]]
+; NONPOWEROF2-NEXT: [[TMP11:%.*]] = shufflevector <6 x float> [[TMP12]], <6 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
; NONPOWEROF2-NEXT: [[TMP14:%.*]] = call reassoc nsz float @llvm.vector.reduce.fadd.v3f32(float 0.000000e+00, <3 x float> [[TMP11]])
+; NONPOWEROF2-NEXT: [[TMP13:%.*]] = shufflevector <6 x float> [[TMP12]], <6 x float> poison, <3 x i32> <i32 3, i32 4, i32 5>
; NONPOWEROF2-NEXT: [[TMP15:%.*]] = call reassoc nsz float @llvm.vector.reduce.fadd.v3f32(float 0.000000e+00, <3 x float> [[TMP13]])
; NONPOWEROF2-NEXT: [[TMP16:%.*]] = tail call float @llvm.sqrt.f32(float [[TMP14]])
; NONPOWEROF2-NEXT: [[TMP17:%.*]] = tail call float @llvm.sqrt.f32(float [[TMP15]])
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-unsupported-type.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-unsupported-type.ll
index c0e1ab56c110b..0131d173fb70e 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-unsupported-type.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-unsupported-type.ll
@@ -27,11 +27,11 @@ define void @stores(ptr noalias %p) {
; CHECK-LABEL: define void @stores(
; CHECK-SAME: ptr noalias [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[_M_VALUE_IMAGP_I266:%.*]] = getelementptr { fp128, fp128 }, ptr null, i64 0, i32 1
-; CHECK-NEXT: [[TMP0:%.*]] = load fp128, ptr null, align 16
-; CHECK-NEXT: [[TMP1:%.*]] = load fp128, ptr [[_M_VALUE_IMAGP_I266]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x fp128>, ptr null, align 16
; CHECK-NEXT: [[P1:%.*]] = getelementptr fp128, ptr [[P]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x fp128> [[TMP2]], i32 0
; CHECK-NEXT: store fp128 [[TMP0]], ptr [[P1]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x fp128> [[TMP2]], i32 1
; CHECK-NEXT: store fp128 [[TMP1]], ptr [[P]], align 16
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index 548a63c3a8cb1..a4fdd107b9eff 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -156,11 +156,11 @@ define void @sqrt_libm_no_errno(ptr %a, ptr %b) {
define void @sqrt_libm_errno(ptr %a, ptr %b) {
; CHECK-LABEL: @sqrt_libm_errno(
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[IDX1]], align 8
-; CHECK-NEXT: [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[A0]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #[[ATTR4]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[TMP2]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[TMP3]]) #[[ATTR4]]
; CHECK-NEXT: store double [[SQRT1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
; CHECK-NEXT: store double [[SQRT2]], ptr [[IDX2]], align 8
@@ -180,11 +180,11 @@ define void @sqrt_libm_errno(ptr %a, ptr %b) {
; Negative test case
define void @round_custom(ptr %a, ptr %b) {
; CHECK-LABEL: @round_custom(
-; CHECK-NEXT: [[A0:%.*]] = load i64, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load i64, ptr [[IDX1]], align 8
-; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #[[ATTR3]]
-; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #[[ATTR3]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[TMP2]]) #[[ATTR3]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[TMP3]]) #[[ATTR3]]
; CHECK-NEXT: store i64 [[ROUND1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 1
; CHECK-NEXT: store i64 [[ROUND2]], ptr [[IDX2]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll b/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll
index 8602c256ad8c2..fb2ed69adb84d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll
@@ -20,10 +20,10 @@ define i32 @test(i32 %0, i32 %1) {
; CHECK-NEXT: [[SUB8:%.*]] = add i32 [[ADD7]], [[TMP0]]
; CHECK-NEXT: store i32 [[SUB8]], ptr [[ARRAYINIT_ELEMENT5]], align 8
; CHECK-NEXT: [[ARRAYINIT_ELEMENT9:%.*]] = getelementptr i8, ptr [[K]], i64 12
-; CHECK-NEXT: [[ADD13:%.*]] = add i32 [[TMP1]], 1
-; CHECK-NEXT: [[ADD10:%.*]] = add i32 [[ADD13]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP0]], 2
+; CHECK-NEXT: [[ADD10:%.*]] = add i32 [[TMP2]], [[TMP1]]
; CHECK-NEXT: [[ADD11:%.*]] = add i32 [[ADD10]], [[ADD1]]
-; CHECK-NEXT: [[ADD12:%.*]] = add i32 [[ADD11]], [[TMP0]]
+; CHECK-NEXT: [[ADD12:%.*]] = add i32 [[ADD11]], 1
; CHECK-NEXT: store i32 [[ADD12]], ptr [[ARRAYINIT_ELEMENT9]], align 4
; CHECK-NEXT: [[CALL15:%.*]] = call i32 (ptr, ...) @printf(ptr null, ptr [[K]])
; CHECK-NEXT: ret i32 [[CALL15]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
index 1ec65da663fee..1b120c060bba9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
@@ -13,10 +13,9 @@ define i32 @test(i32 %mul) {
; CHECK-NEXT: [[ADD5:%.*]] = add i32 [[CALL]], [[MUL1]]
; CHECK-NEXT: store i32 [[ADD5]], ptr [[H]], align 16
; CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[H]], i64 4
-; CHECK-NEXT: [[ADD6:%.*]] = add i32 0, 0
+; CHECK-NEXT: [[ADD6:%.*]] = add i32 0, [[ADD4]]
; CHECK-NEXT: [[ADD7:%.*]] = add i32 [[ADD6]], [[MUL]]
-; CHECK-NEXT: [[ADD9:%.*]] = add i32 [[ADD7]], [[ADD4]]
-; CHECK-NEXT: store i32 [[ADD9]], ptr [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYINIT_ELEMENT]], align 4
; CHECK-NEXT: [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr i8, ptr [[H]], i64 8
; CHECK-NEXT: [[ADD11:%.*]] = or i32 [[ADD]], 0
; CHECK-NEXT: [[ADD12:%.*]] = add i32 [[ADD11]], [[ADD4]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
index f23043f0c47f4..07442eaef7d7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/conversion-fp16.ll
@@ -565,25 +565,23 @@ define void @fpround_v2xf64_v2xf16(ptr %s0, ptr %d0) {
;
; CHECK-F16C-LABEL: define void @fpround_v2xf64_v2xf16(
; CHECK-F16C-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
-; CHECK-F16C-NEXT: [[S1:%.*]] = getelementptr inbounds double, ptr [[S0]], i64 1
-; CHECK-F16C-NEXT: [[L0:%.*]] = load double, ptr [[S0]], align 4
-; CHECK-F16C-NEXT: [[L1:%.*]] = load double, ptr [[S1]], align 4
-; CHECK-F16C-NEXT: [[T0:%.*]] = fptrunc double [[L0]] to half
-; CHECK-F16C-NEXT: [[T1:%.*]] = fptrunc double [[L1]] to half
+; CHECK-F16C-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[S0]], align 4
+; CHECK-F16C-NEXT: [[TMP2:%.*]] = fptrunc <2 x double> [[TMP1]] to <2 x half>
; CHECK-F16C-NEXT: [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-F16C-NEXT: [[T0:%.*]] = extractelement <2 x half> [[TMP2]], i32 0
; CHECK-F16C-NEXT: store half [[T0]], ptr [[D0]], align 2
+; CHECK-F16C-NEXT: [[T1:%.*]] = extractelement <2 x half> [[TMP2]], i32 1
; CHECK-F16C-NEXT: store half [[T1]], ptr [[D1]], align 2
; CHECK-F16C-NEXT: ret void
;
; CHECK-AVX512-LABEL: define void @fpround_v2xf64_v2xf16(
; CHECK-AVX512-SAME: ptr [[S0:%.*]], ptr [[D0:%.*]]) #[[ATTR0]] {
-; CHECK-AVX512-NEXT: [[S1:%.*]] = getelementptr inbounds double, ptr [[S0]], i64 1
-; CHECK-AVX512-NEXT: [[L0:%.*]] = load double, ptr [[S0]], align 4
-; CHECK-AVX512-NEXT: [[L1:%.*]] = load double, ptr [[S1]], align 4
-; CHECK-AVX512-NEXT: [[T0:%.*]] = fptrunc double [[L0]] to half
-; CHECK-AVX512-NEXT: [[T1:%.*]] = fptrunc double [[L1]] to half
+; CHECK-AVX512-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[S0]], align 4
+; CHECK-AVX512-NEXT: [[TMP2:%.*]] = fptrunc <2 x double> [[TMP1]] to <2 x half>
; CHECK-AVX512-NEXT: [[D1:%.*]] = getelementptr inbounds half, ptr [[D0]], i64 1
+; CHECK-AVX512-NEXT: [[T0:%.*]] = extractelement <2 x half> [[TMP2]], i32 0
; CHECK-AVX512-NEXT: store half [[T0]], ptr [[D0]], align 2
+; CHECK-AVX512-NEXT: [[T1:%.*]] = extractelement <2 x half> [[TMP2]], i32 1
; CHECK-AVX512-NEXT: store half [[T1]], ptr [[D1]], align 2
; CHECK-AVX512-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
index 33639473fe1f0..58430035cf6c1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
@@ -199,31 +199,54 @@ entry:
; It is more profitable to reorder the operands of the RHS add, because A[1] has an external use.
define void @lookahead_external_uses(ptr %A, ptr %B, ptr %C, ptr %D, ptr %S, ptr %Ext1, ptr %Ext2) {
-; CHECK-LABEL: @lookahead_external_uses(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[IDXB2:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 2
-; CHECK-NEXT: [[IDXA2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
-; CHECK-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
-; CHECK-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
-; CHECK-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[IDXA1]], align 8
-; CHECK-NEXT: [[B2:%.*]] = load double, ptr [[IDXB2]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[IDXA2]], align 8
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
-; CHECK-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
-; CHECK-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
-; CHECK-NEXT: [[SUBA1B2:%.*]] = fsub fast double [[A1]], [[B2]]
-; CHECK-NEXT: [[SUBA2B1:%.*]] = fsub fast double [[A2]], [[B1]]
-; CHECK-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
-; CHECK-NEXT: [[ADD1:%.*]] = fadd fast double [[SUBA1B2]], [[SUBA2B1]]
-; CHECK-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
-; CHECK-NEXT: store double [[ADD0]], ptr [[S]], align 8
-; CHECK-NEXT: store double [[ADD1]], ptr [[IDXS1]], align 8
-; CHECK-NEXT: store double [[A1]], ptr [[EXT1:%.*]], align 8
-; CHECK-NEXT: ret void
+; SSE-LABEL: @lookahead_external_uses(
+; SSE-NEXT: entry:
+; SSE-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
+; SSE-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 1
+; SSE-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; SSE-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; SSE-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
+; SSE-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
+; SSE-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
+; SSE-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
+; SSE-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[IDXA1]], align 8
+; SSE-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[IDXB1]], align 8
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; SSE-NEXT: [[TMP3:%.*]] = fsub fast <2 x double> [[TMP0]], [[TMP2]]
+; SSE-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
+; SSE-NEXT: [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP3]])
+; SSE-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
+; SSE-NEXT: store double [[ADD0]], ptr [[S]], align 8
+; SSE-NEXT: store double [[TMP4]], ptr [[IDXS1]], align 8
+; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+; SSE-NEXT: store double [[TMP5]], ptr [[EXT1:%.*]], align 8
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @lookahead_external_uses(
+; AVX-NEXT: entry:
+; AVX-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
+; AVX-NEXT: [[IDXB2:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 2
+; AVX-NEXT: [[IDXA2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
+; AVX-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; AVX-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; AVX-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; AVX-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
+; AVX-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
+; AVX-NEXT: [[A1:%.*]] = load double, ptr [[IDXA1]], align 8
+; AVX-NEXT: [[B2:%.*]] = load double, ptr [[IDXB2]], align 8
+; AVX-NEXT: [[A2:%.*]] = load double, ptr [[IDXA2]], align 8
+; AVX-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
+; AVX-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
+; AVX-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
+; AVX-NEXT: [[SUBA1B2:%.*]] = fsub fast double [[A1]], [[B2]]
+; AVX-NEXT: [[SUBA2B1:%.*]] = fsub fast double [[A2]], [[B1]]
+; AVX-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
+; AVX-NEXT: [[ADD1:%.*]] = fadd fast double [[SUBA1B2]], [[SUBA2B1]]
+; AVX-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
+; AVX-NEXT: store double [[ADD0]], ptr [[S]], align 8
+; AVX-NEXT: store double [[ADD1]], ptr [[IDXS1]], align 8
+; AVX-NEXT: store double [[A1]], ptr [[EXT1:%.*]], align 8
+; AVX-NEXT: ret void
;
entry:
@@ -277,35 +300,63 @@ entry:
; from A get vectorized instead of the loads from B.
;
define void @lookahead_limit_users_budget(ptr %A, ptr %B, ptr %C, ptr %D, ptr %S, ptr %Ext1, ptr %Ext2, ptr %Ext3, ptr %Ext4, ptr %Ext5) {
-; CHECK-LABEL: @lookahead_limit_users_budget(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[IDXB2:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 2
-; CHECK-NEXT: [[IDXA2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
-; CHECK-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
-; CHECK-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
-; CHECK-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[IDXA1]], align 8
-; CHECK-NEXT: [[B2:%.*]] = load double, ptr [[IDXB2]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[IDXA2]], align 8
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
-; CHECK-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
-; CHECK-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
-; CHECK-NEXT: [[SUBA1B2:%.*]] = fsub fast double [[A1]], [[B2]]
-; CHECK-NEXT: [[SUBA2B1:%.*]] = fsub fast double [[A2]], [[B1]]
-; CHECK-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
-; CHECK-NEXT: [[ADD1:%.*]] = fadd fast double [[SUBA1B2]], [[SUBA2B1]]
-; CHECK-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
-; CHECK-NEXT: store double [[ADD0]], ptr [[S]], align 8
-; CHECK-NEXT: store double [[ADD1]], ptr [[IDXS1]], align 8
-; CHECK-NEXT: store double [[A1]], ptr [[EXT3:%.*]], align 8
-; CHECK-NEXT: store double [[A1]], ptr [[EXT2:%.*]], align 8
-; CHECK-NEXT: store double [[A1]], ptr [[EXT6:%.*]], align 8
-; CHECK-NEXT: store double [[B1]], ptr [[EXT4:%.*]], align 8
-; CHECK-NEXT: store double [[B1]], ptr [[EXT5:%.*]], align 8
-; CHECK-NEXT: ret void
+; SSE-LABEL: @lookahead_limit_users_budget(
+; SSE-NEXT: entry:
+; SSE-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
+; SSE-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 1
+; SSE-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; SSE-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; SSE-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
+; SSE-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
+; SSE-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
+; SSE-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
+; SSE-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[IDXA1]], align 8
+; SSE-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
+; SSE-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[IDXB1]], align 8
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; SSE-NEXT: [[TMP3:%.*]] = fsub fast <2 x double> [[TMP0]], [[TMP2]]
+; SSE-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
+; SSE-NEXT: [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP3]])
+; SSE-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
+; SSE-NEXT: store double [[ADD0]], ptr [[S]], align 8
+; SSE-NEXT: store double [[TMP4]], ptr [[IDXS1]], align 8
+; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+; SSE-NEXT: store double [[TMP5]], ptr [[EXT1:%.*]], align 8
+; SSE-NEXT: store double [[TMP5]], ptr [[EXT2:%.*]], align 8
+; SSE-NEXT: store double [[TMP5]], ptr [[EXT3:%.*]], align 8
+; SSE-NEXT: store double [[B1]], ptr [[EXT4:%.*]], align 8
+; SSE-NEXT: store double [[B1]], ptr [[EXT5:%.*]], align 8
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @lookahead_limit_users_budget(
+; AVX-NEXT: entry:
+; AVX-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
+; AVX-NEXT: [[IDXB2:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i64 2
+; AVX-NEXT: [[IDXA2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
+; AVX-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; AVX-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; AVX-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; AVX-NEXT: [[C0:%.*]] = load double, ptr [[C:%.*]], align 8
+; AVX-NEXT: [[D0:%.*]] = load double, ptr [[D:%.*]], align 8
+; AVX-NEXT: [[A1:%.*]] = load double, ptr [[IDXA1]], align 8
+; AVX-NEXT: [[B2:%.*]] = load double, ptr [[IDXB2]], align 8
+; AVX-NEXT: [[A2:%.*]] = load double, ptr [[IDXA2]], align 8
+; AVX-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
+; AVX-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
+; AVX-NEXT: [[SUBC0D0:%.*]] = fsub fast double [[C0]], [[D0]]
+; AVX-NEXT: [[SUBA1B2:%.*]] = fsub fast double [[A1]], [[B2]]
+; AVX-NEXT: [[SUBA2B1:%.*]] = fsub fast double [[A2]], [[B1]]
+; AVX-NEXT: [[ADD0:%.*]] = fadd fast double [[SUBA0B0]], [[SUBC0D0]]
+; AVX-NEXT: [[ADD1:%.*]] = fadd fast double [[SUBA1B2]], [[SUBA2B1]]
+; AVX-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[S:%.*]], i64 1
+; AVX-NEXT: store double [[ADD0]], ptr [[S]], align 8
+; AVX-NEXT: store double [[ADD1]], ptr [[IDXS1]], align 8
+; AVX-NEXT: store double [[A1]], ptr [[EXT1:%.*]], align 8
+; AVX-NEXT: store double [[A1]], ptr [[EXT2:%.*]], align 8
+; AVX-NEXT: store double [[A1]], ptr [[EXT3:%.*]], align 8
+; AVX-NEXT: store double [[B1]], ptr [[EXT4:%.*]], align 8
+; AVX-NEXT: store double [[B1]], ptr [[EXT5:%.*]], align 8
+; AVX-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
index dc91352d0f2b9..4e489a2ac87db 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
@@ -14,12 +14,11 @@ declare i32 @__gxx_personality_v0(...)
define void @test_two_calls_with_vectorizable_operands(ptr %p, ptr %out0, ptr %out1) {
; CHECK-LABEL: define void @test_two_calls_with_vectorizable_operands(
; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul double [[A0]], 2.000000e+00
-; CHECK-NEXT: [[M1:%.*]] = fmul double [[A1]], 3.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[TMP1]], <double 2.000000e+00, double 3.000000e+00>
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
; CHECK-NEXT: [[R0:%.*]] = call double @user_func(double [[M0]])
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
; CHECK-NEXT: [[R1:%.*]] = call double @user_func(double [[M1]])
; CHECK-NEXT: store double [[R0]], ptr [[OUT0]], align 8
; CHECK-NEXT: store double [[R1]], ptr [[OUT1]], align 8
@@ -42,17 +41,15 @@ define void @test_two_calls_with_vectorizable_operands(ptr %p, ptr %out0, ptr %o
define void @test_two_calls_paired_operands(ptr %p, ptr %q, ptr %out0, ptr %out1) {
; CHECK-LABEL: define void @test_two_calls_paired_operands(
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
-; CHECK-NEXT: [[Q1:%.*]] = getelementptr double, ptr [[Q]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul double [[A0]], 2.000000e+00
-; CHECK-NEXT: [[M1:%.*]] = fmul double [[A1]], 3.000000e+00
-; CHECK-NEXT: [[N0:%.*]] = fadd double [[B0]], 4.000000e+00
-; CHECK-NEXT: [[N1:%.*]] = fadd double [[B1]], 5.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[TMP1]], <double 2.000000e+00, double 3.000000e+00>
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[Q]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], <double 4.000000e+00, double 5.000000e+00>
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: [[N0:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
; CHECK-NEXT: [[R0:%.*]] = call double @another_user_func(double [[M0]], double [[N0]])
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-NEXT: [[N1:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
; CHECK-NEXT: [[R1:%.*]] = call double @another_user_func(double [[M1]], double [[N1]])
; CHECK-NEXT: store double [[R0]], ptr [[OUT0]], align 8
; CHECK-NEXT: store double [[R1]], ptr [[OUT1]], align 8
@@ -151,20 +148,15 @@ eh:
define void @test_atomicrmw_value_chain(ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %src) {
; CHECK-LABEL: define void @test_atomicrmw_value_chain(
; CHECK-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A0:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT: [[SP1:%.*]] = getelementptr i32, ptr [[SRC]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load i32, ptr [[SP1]], align 4
-; CHECK-NEXT: [[SP2:%.*]] = getelementptr i32, ptr [[SRC]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load i32, ptr [[SP2]], align 4
-; CHECK-NEXT: [[SP3:%.*]] = getelementptr i32, ptr [[SRC]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load i32, ptr [[SP3]], align 4
-; CHECK-NEXT: [[V0:%.*]] = mul i32 [[A0]], 7
-; CHECK-NEXT: [[V1:%.*]] = mul i32 [[A1]], 7
-; CHECK-NEXT: [[V2:%.*]] = mul i32 [[A2]], 7
-; CHECK-NEXT: [[V3:%.*]] = mul i32 [[A3]], 7
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], splat (i32 7)
+; CHECK-NEXT: [[V0:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
; CHECK-NEXT: [[OLD0:%.*]] = atomicrmw add ptr [[P1]], i32 [[V0]] seq_cst, align 4
+; CHECK-NEXT: [[V1:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
; CHECK-NEXT: [[OLD1:%.*]] = atomicrmw add ptr [[P2]], i32 [[V1]] seq_cst, align 4
+; CHECK-NEXT: [[V2:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
; CHECK-NEXT: [[OLD2:%.*]] = atomicrmw add ptr [[P3]], i32 [[V2]] seq_cst, align 4
+; CHECK-NEXT: [[V3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
; CHECK-NEXT: [[OLD3:%.*]] = atomicrmw add ptr [[P4]], i32 [[V3]] seq_cst, align 4
; CHECK-NEXT: ret void
;
@@ -189,17 +181,17 @@ define void @test_atomicrmw_value_chain(ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr
define void @test_cmpxchg_value_chain(ptr %p1, ptr %p2, ptr %src_a, ptr %src_b) {
; CHECK-LABEL: define void @test_cmpxchg_value_chain(
; CHECK-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[SRC_A:%.*]], ptr [[SRC_B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A0:%.*]] = load i32, ptr [[SRC_A]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr i32, ptr [[SRC_A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load i32, ptr [[AP1]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load i32, ptr [[SRC_B]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr i32, ptr [[SRC_B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load i32, ptr [[BP1]], align 4
-; CHECK-NEXT: [[CMP0:%.*]] = mul i32 [[A0]], 11
-; CHECK-NEXT: [[CMP1:%.*]] = mul i32 [[A1]], 11
-; CHECK-NEXT: [[NEW0:%.*]] = mul i32 [[B0]], 13
-; CHECK-NEXT: [[NEW1:%.*]] = mul i32 [[B1]], 13
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[SRC_A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[SRC_B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i32> [[TMP5]], <i32 11, i32 11, i32 13, i32 13>
+; CHECK-NEXT: [[CMP0:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
+; CHECK-NEXT: [[NEW0:%.*]] = extractelement <4 x i32> [[TMP6]], i32 2
; CHECK-NEXT: [[R0:%.*]] = cmpxchg ptr [[P1]], i32 [[CMP0]], i32 [[NEW0]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[CMP1:%.*]] = extractelement <4 x i32> [[TMP6]], i32 1
+; CHECK-NEXT: [[NEW1:%.*]] = extractelement <4 x i32> [[TMP6]], i32 3
; CHECK-NEXT: [[R1:%.*]] = cmpxchg ptr [[P2]], i32 [[CMP1]], i32 [[NEW1]] seq_cst seq_cst, align 4
; CHECK-NEXT: ret void
;
@@ -243,20 +235,15 @@ b:
define void @test_scattered_stores(ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %src) {
; CHECK-LABEL: define void @test_scattered_stores(
; CHECK-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[SP1:%.*]] = getelementptr double, ptr [[SRC]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[SP1]], align 8
-; CHECK-NEXT: [[SP2:%.*]] = getelementptr double, ptr [[SRC]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[SP2]], align 8
-; CHECK-NEXT: [[SP3:%.*]] = getelementptr double, ptr [[SRC]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[SP3]], align 8
-; CHECK-NEXT: [[V0:%.*]] = fmul double [[A0]], 2.000000e+00
-; CHECK-NEXT: [[V1:%.*]] = fmul double [[A1]], 3.000000e+00
-; CHECK-NEXT: [[V2:%.*]] = fmul double [[A2]], 5.000000e+00
-; CHECK-NEXT: [[V3:%.*]] = fmul double [[A3]], 7.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[TMP1]], <double 2.000000e+00, double 3.000000e+00, double 5.000000e+00, double 7.000000e+00>
+; CHECK-NEXT: [[V0:%.*]] = extractelement <4 x double> [[TMP2]], i32 0
; CHECK-NEXT: store double [[V0]], ptr [[P1]], align 8
+; CHECK-NEXT: [[V1:%.*]] = extractelement <4 x double> [[TMP2]], i32 1
; CHECK-NEXT: store double [[V1]], ptr [[P2]], align 8
+; CHECK-NEXT: [[V2:%.*]] = extractelement <4 x double> [[TMP2]], i32 2
; CHECK-NEXT: store double [[V2]], ptr [[P3]], align 8
+; CHECK-NEXT: [[V3:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
; CHECK-NEXT: store double [[V3]], ptr [[P4]], align 8
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
index f198971265720..cef62c6cae3d5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
@@ -18,20 +18,18 @@ define i32 @foo(ptr noalias nocapture %A, ptr noalias nocapture %B, float %T) {
; CHECK-NEXT: [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
; CHECK-NEXT: store i8 [[TMP6]], ptr [[A:%.*]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11
-; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fmul float [[TMP8]], [[T]]
-; CHECK-NEXT: [[TMP10:%.*]] = fpext float [[TMP9]] to double
-; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fptosi double [[TMP11]] to i8
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x float>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> poison, float [[T]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x float> [[TMP9]], [[TMP11]]
+; CHECK-NEXT: [[TMP17:%.*]] = fpext <2 x float> [[TMP16]] to <2 x double>
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP17]], <double 5.000000e+00, double 6.000000e+00>
+; CHECK-NEXT: [[TMP15:%.*]] = fptosi <2 x double> [[TMP14]] to <2 x i8>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 12
-; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4
-; CHECK-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[T]]
-; CHECK-NEXT: [[TMP17:%.*]] = fpext float [[TMP16]] to double
-; CHECK-NEXT: [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00
-; CHECK-NEXT: [[TMP19:%.*]] = fptosi double [[TMP18]] to i8
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
; CHECK-NEXT: store i8 [[TMP19]], ptr [[TMP20]], align 1
; CHECK-NEXT: ret i32 undef
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index de72521345435..4aa83e10d7d5f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -352,7 +352,6 @@ define void @good_load_order() {
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
; CHECK-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
@@ -360,8 +359,9 @@ define void @good_load_order() {
; CHECK-NEXT: [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
; CHECK-NEXT: store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT: [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
+; CHECK-NEXT: [[TMP15:%.*]] = load <2 x float>, ptr [[ARRAYIDX31]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP13]] = extractelement <2 x float> [[TMP15]], i32 1
; CHECK-NEXT: [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
; CHECK-NEXT: store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
@@ -384,7 +384,6 @@ define void @good_load_order() {
; SSE2-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
; SSE2-NEXT: [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
; SSE2-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
-; SSE2-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
; SSE2-NEXT: [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
; SSE2-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
; SSE2-NEXT: [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
@@ -392,8 +391,9 @@ define void @good_load_order() {
; SSE2-NEXT: [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
; SSE2-NEXT: store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
; SSE2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; SSE2-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
-; SSE2-NEXT: [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
+; SSE2-NEXT: [[TMP15:%.*]] = load <2 x float>, ptr [[ARRAYIDX31]], align 4
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; SSE2-NEXT: [[TMP13]] = extractelement <2 x float> [[TMP15]], i32 1
; SSE2-NEXT: [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
; SSE2-NEXT: store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
; SSE2-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-non-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-non-schedulable.ll
index 95199448c6485..90bb4dd136e94 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-non-schedulable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-non-schedulable.ll
@@ -18,18 +18,20 @@ define void @test(ptr %0, i64 %1, i64 %2, i1 %3, i64 %4, i64 %5) {
; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP17]], align 4
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[TMP1]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i64 0, [[TMP18]]
-; CHECK-NEXT: [[TMP21:%.*]] = sub i64 0, [[TMP16]]
-; CHECK-NEXT: [[TMP22:%.*]] = sub i64 0, [[TMP14]]
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP24:%.*]] = sub i64 0, [[TMP1]]
-; CHECK-NEXT: [[TMP25:%.*]] = sub i64 0, [[TMP1]]
-; CHECK-NEXT: [[TMP26:%.*]] = sub i64 0, [[TMP10]]
; CHECK-NEXT: [[TMP27:%.*]] = sub i64 0, [[TMP8]]
; CHECK-NEXT: [[TMP28:%.*]] = sub i64 0, [[TMP19]]
-; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP14]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = ashr i64 [[TMP29]], 14
-; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP14]], 1
-; CHECK-NEXT: [[TMP32:%.*]] = ashr i64 [[TMP31]], 14
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <6 x i64> <i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <6 x i64> [[TMP23]], <6 x i64> <i64 poison, i64 -1, i64 poison, i64 poison, i64 poison, i64 poison>, <6 x i32> <i32 1, i32 7, i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <6 x i64> [[TMP24]], i64 [[TMP14]], i32 3
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <6 x i64> [[TMP25]], i64 [[TMP16]], i32 4
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <6 x i64> [[TMP26]], i64 [[TMP10]], i32 5
+; CHECK-NEXT: [[TMP47:%.*]] = sub <6 x i64> [[TMP23]], [[TMP46]]
+; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <6 x i64> [[TMP46]], <6 x i64> poison, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT: [[TMP30:%.*]] = add <2 x i64> [[TMP29]], splat (i64 1)
+; CHECK-NEXT: [[TMP31:%.*]] = ashr <2 x i64> [[TMP30]], splat (i64 14)
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <6 x i64> [[TMP47]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <2 x i64> [[TMP31]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <8 x i64> [[TMP32]], <8 x i64> [[TMP57]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; CHECK-NEXT: br i1 [[TMP3]], label %[[BB52:.*]], label %[[BB37:.*]]
; CHECK: [[BB37]]:
; CHECK-NEXT: [[TMP34:%.*]] = ashr i64 [[TMP2]], 2
@@ -44,45 +46,31 @@ define void @test(ptr %0, i64 %1, i64 %2, i1 %3, i64 %4, i64 %5) {
; CHECK-NEXT: [[TMP43:%.*]] = lshr i64 [[TMP1]], 1
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP42]], [[TMP43]]
; CHECK-NEXT: [[TMP45:%.*]] = ashr i64 [[TMP44]], 2
-; CHECK-NEXT: [[TMP46:%.*]] = ashr i64 [[TMP5]], 2
-; CHECK-NEXT: [[TMP47:%.*]] = ashr i64 [[TMP4]], 2
-; CHECK-NEXT: [[TMP48:%.*]] = ashr i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP49:%.*]] = ashr i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP50:%.*]] = ashr i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP51:%.*]] = ashr i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP52:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP53:%.*]] = ashr i64 [[TMP52]], 2
-; CHECK-NEXT: [[TMP78:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP79:%.*]] = ashr i64 [[TMP78]], 2
+; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <6 x i64> [[TMP23]], <6 x i64> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP49:%.*]] = add <2 x i64> [[TMP48]], splat (i64 1)
+; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <2 x i64> [[TMP48]], <2 x i64> poison, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0>
+; CHECK-NEXT: [[TMP51:%.*]] = insertelement <8 x i64> [[TMP50]], i64 [[TMP4]], i32 3
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <8 x i64> [[TMP51]], i64 [[TMP5]], i32 4
+; CHECK-NEXT: [[TMP53:%.*]] = shufflevector <2 x i64> [[TMP49]], <2 x i64> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1>
+; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <2 x i64> [[TMP49]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <8 x i64> [[TMP54]], <8 x i64> [[TMP52]], <8 x i32> <i32 8, i32 9, i32 0, i32 11, i32 12, i32 1, i32 14, i32 15>
+; CHECK-NEXT: [[TMP56:%.*]] = ashr <8 x i64> [[TMP55]], splat (i64 2)
; CHECK-NEXT: [[TMP80:%.*]] = add i64 [[TMP1]], 1
; CHECK-NEXT: [[TMP81:%.*]] = ashr i64 [[TMP80]], 2
; CHECK-NEXT: br label %[[BB52]]
; CHECK: [[BB52]]:
-; CHECK-NEXT: [[TMP57:%.*]] = phi i64 [ [[TMP51]], %[[BB37]] ], [ [[TMP24]], [[TMP6:%.*]] ]
-; CHECK-NEXT: [[TMP58:%.*]] = phi i64 [ [[TMP50]], %[[BB37]] ], [ [[TMP32]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP64:%.*]] = phi i64 [ [[TMP53]], %[[BB37]] ], [ [[TMP25]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP71:%.*]] = phi i64 [ [[TMP79]], %[[BB37]] ], [ [[TMP26]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP77:%.*]] = phi i64 [ [[TMP81]], %[[BB37]] ], [ [[TMP27]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP55:%.*]] = phi i64 [ [[TMP49]], %[[BB37]] ], [ [[TMP30]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP54:%.*]] = phi i64 [ [[TMP48]], %[[BB37]] ], [ [[TMP23]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP65:%.*]] = phi i64 [ [[TMP47]], %[[BB37]] ], [ [[TMP22]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP69:%.*]] = phi i64 [ [[TMP46]], %[[BB37]] ], [ [[TMP21]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP74:%.*]] = phi i64 [ [[TMP45]], %[[BB37]] ], [ [[TMP20]], [[TMP6]] ]
+; CHECK-NEXT: [[TMP77:%.*]] = phi i64 [ [[TMP81]], %[[BB37]] ], [ [[TMP27]], [[TMP6:%.*]] ]
+; CHECK-NEXT: [[TMP61:%.*]] = phi i64 [ [[TMP45]], %[[BB37]] ], [ [[TMP20]], [[TMP6]] ]
; CHECK-NEXT: [[TMP76:%.*]] = phi i64 [ [[TMP41]], %[[BB37]] ], [ [[TMP28]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP60:%.*]] = phi i64 [ [[TMP34]], %[[BB37]] ], [ [[TMP12]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP61:%.*]] = phi i64 [ [[TMP35]], %[[BB37]] ], [ [[TMP13]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP67:%.*]] = phi i64 [ [[TMP37]], %[[BB37]] ], [ [[TMP33]], [[TMP6]] ]
-; CHECK-NEXT: [[TMP56:%.*]] = or i64 [[TMP54]], [[TMP55]]
-; CHECK-NEXT: [[TMP59:%.*]] = or i64 [[TMP57]], [[TMP58]]
-; CHECK-NEXT: [[TMP62:%.*]] = or i64 [[TMP60]], [[TMP61]]
-; CHECK-NEXT: [[TMP63:%.*]] = or i64 [[TMP59]], [[TMP56]]
-; CHECK-NEXT: [[TMP66:%.*]] = or i64 [[TMP64]], [[TMP65]]
-; CHECK-NEXT: [[TMP68:%.*]] = or i64 [[TMP67]], [[TMP62]]
-; CHECK-NEXT: [[TMP70:%.*]] = or i64 [[TMP69]], [[TMP66]]
+; CHECK-NEXT: [[TMP71:%.*]] = phi i64 [ [[TMP34]], %[[BB37]] ], [ [[TMP12]], [[TMP6]] ]
+; CHECK-NEXT: [[TMP70:%.*]] = phi i64 [ [[TMP35]], %[[BB37]] ], [ [[TMP13]], [[TMP6]] ]
+; CHECK-NEXT: [[TMP63:%.*]] = phi i64 [ [[TMP37]], %[[BB37]] ], [ [[TMP33]], [[TMP6]] ]
+; CHECK-NEXT: [[TMP66:%.*]] = phi <8 x i64> [ [[TMP56]], %[[BB37]] ], [ [[TMP58]], [[TMP6]] ]
; CHECK-NEXT: [[TMP72:%.*]] = or i64 [[TMP71]], [[TMP70]]
; CHECK-NEXT: [[TMP73:%.*]] = or i64 [[TMP63]], [[TMP72]]
-; CHECK-NEXT: [[TMP75:%.*]] = or i64 [[TMP74]], [[TMP73]]
-; CHECK-NEXT: store i64 [[TMP68]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP69:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP66]])
+; CHECK-NEXT: [[TMP75:%.*]] = or i64 [[TMP69]], [[TMP61]]
+; CHECK-NEXT: store i64 [[TMP73]], ptr [[TMP0]], align 4
; CHECK-NEXT: store i64 [[TMP76]], ptr null, align 4
; CHECK-NEXT: store i64 [[TMP77]], ptr [[TMP0]], align 4
; CHECK-NEXT: store i64 [[TMP75]], ptr null, align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
index 42d3d8049a897..2f6e58ce675d1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
@@ -6,19 +6,17 @@ define void @test({ <2 x float>, float } %0, <2 x float> %1, i1 %2) {
; CHECK-SAME: { <2 x float>, float } [[TMP0:%.*]], <2 x float> [[TMP1:%.*]], i1 [[TMP2:%.*]]) {
; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x float>, float } [[TMP0]], 0
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x float> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB9:.*]], label %[[BB8:.*]]
; CHECK: [[BB8]]:
; CHECK-NEXT: br label %[[BB9]]
; CHECK: [[BB9]]:
-; CHECK-NEXT: [[TMP10:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP7]], [[TMP3:%.*]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP6]], [[TMP3]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP5]], [[TMP3]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = fpext float [[TMP12]] to double
+; CHECK-NEXT: [[TMP11:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP5]], [[TMP3:%.*]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x float> [ zeroinitializer, %[[BB8]] ], [ [[TMP1]], [[TMP3]] ]
; CHECK-NEXT: [[TMP14:%.*]] = fpext float [[TMP11]] to double
-; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double [[TMP13]], double [[TMP14]], double 0.000000e+00)
-; CHECK-NEXT: [[TMP16:%.*]] = fpext float [[TMP10]] to double
+; CHECK-NEXT: [[TMP15:%.*]] = fpext <2 x float> [[TMP9]] to <2 x double>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP15]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double [[TMP14]], double [[TMP12]], double 0.000000e+00)
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP15]], i32 0
; CHECK-NEXT: [[TMP17:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double 0.000000e+00, double [[TMP16]], double 0.000000e+00)
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
index f574428ba5e75..b5c7e0e3cde06 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
@@ -13,17 +13,8 @@ declare void @unknown()
define void @test(ptr %a, ptr %b, ptr %c, ptr %d) {
; LOBUDGET-LABEL: @test(
; LOBUDGET-NEXT: entry:
-; LOBUDGET-NEXT: [[L0:%.*]] = load float, ptr [[A:%.*]], align 4
-; LOBUDGET-NEXT: [[A1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; LOBUDGET-NEXT: [[L1:%.*]] = load float, ptr [[A1]], align 4
-; LOBUDGET-NEXT: [[A2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; LOBUDGET-NEXT: [[L2:%.*]] = load float, ptr [[A2]], align 4
-; LOBUDGET-NEXT: [[A3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; LOBUDGET-NEXT: [[L3:%.*]] = load float, ptr [[A3]], align 4
-; LOBUDGET-NEXT: [[L00:%.*]] = fadd float [[L0]], [[L0]]
-; LOBUDGET-NEXT: [[L10:%.*]] = fadd float [[L1]], [[L1]]
-; LOBUDGET-NEXT: [[L20:%.*]] = fadd float [[L2]], [[L2]]
-; LOBUDGET-NEXT: [[L30:%.*]] = fadd float [[L3]], [[L3]]
+; LOBUDGET-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[A:%.*]], align 4
+; LOBUDGET-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[TMP2]], [[TMP2]]
; LOBUDGET-NEXT: call void @unknown()
; LOBUDGET-NEXT: call void @unknown()
; LOBUDGET-NEXT: call void @unknown()
@@ -52,12 +43,16 @@ define void @test(ptr %a, ptr %b, ptr %c, ptr %d) {
; LOBUDGET-NEXT: call void @unknown()
; LOBUDGET-NEXT: call void @unknown()
; LOBUDGET-NEXT: call void @unknown()
+; LOBUDGET-NEXT: [[L00:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
; LOBUDGET-NEXT: store float [[L00]], ptr [[B:%.*]], align 4
; LOBUDGET-NEXT: [[B1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; LOBUDGET-NEXT: [[L10:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
; LOBUDGET-NEXT: store float [[L10]], ptr [[B1]], align 4
; LOBUDGET-NEXT: [[B2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; LOBUDGET-NEXT: [[L20:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
; LOBUDGET-NEXT: store float [[L20]], ptr [[B2]], align 4
; LOBUDGET-NEXT: [[B3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; LOBUDGET-NEXT: [[L30:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
; LOBUDGET-NEXT: store float [[L30]], ptr [[B3]], align 4
; LOBUDGET-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C:%.*]], align 4
; LOBUDGET-NEXT: store <4 x float> [[TMP0]], ptr [[D:%.*]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
index ceddfa7e8be35..c63bac27d08ed 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
@@ -83,16 +83,13 @@ define void @test_volatile_load(ptr %a, ptr %b, ptr %c) {
; Don't vectorize volatile stores.
define void @test_volatile_store(ptr %a, ptr %b, ptr %c) {
; CHECK-LABEL: @test_volatile_store(
-; CHECK-NEXT: [[I0:%.*]] = load double, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[I1:%.*]] = load double, ptr [[B:%.*]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[I0]], [[I1]]
-; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
-; CHECK-NEXT: [[I3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
-; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
-; CHECK-NEXT: [[I4:%.*]] = load double, ptr [[ARRAYIDX4]], align 8
-; CHECK-NEXT: [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[MUL:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
; CHECK-NEXT: store volatile double [[MUL]], ptr [[C:%.*]], align 8
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds double, ptr [[C]], i64 1
+; CHECK-NEXT: [[MUL5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
; CHECK-NEXT: store volatile double [[MUL5]], ptr [[ARRAYIDX5]], align 8
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-marked-to-gather.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-marked-to-gather.ll
index 3dc1da94b3430..8474e9cae508f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-marked-to-gather.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-marked-to-gather.ll
@@ -11,17 +11,9 @@ define i32 @test(i128 %add134.i8203) {
; CHECK: [[CONT90_I]]:
; CHECK-NEXT: [[T_I_SROA_488_1:%.*]] = phi i64 [ [[SUB9_I3318]], %[[ENTRY]] ]
; CHECK-NEXT: [[T_I_SROA_457_1:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[T_I_SROA_426_1:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[CONV61_I8293:%.*]] = zext i64 [[T_I_SROA_426_1]] to i128
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <2 x i64> [[TMP0]] to <2 x i1>
; CHECK-NEXT: [[CONV66_I8295:%.*]] = zext i64 [[T_I_SROA_457_1]] to i128
-; CHECK-NEXT: [[MUL127_I8312:%.*]] = mul i128 0, [[CONV66_I8295]]
-; CHECK-NEXT: [[MUL132_I8313:%.*]] = mul i128 0, [[CONV61_I8293]]
-; CHECK-NEXT: [[ADD135_I8317:%.*]] = or i128 0, [[MUL132_I8313]]
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i128 [[ADD135_I8317]] to i64
-; CHECK-NEXT: [[CONV137_I8321:%.*]] = and i64 [[TMP0]], 0
-; CHECK-NEXT: [[ADD153_I8337:%.*]] = or i128 0, [[MUL127_I8312]]
-; CHECK-NEXT: [[ADD155_I8339:%.*]] = or i128 [[ADD153_I8337]], 0
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[ADD155_I8339]] to i64
; CHECK-NEXT: [[CONV76_I8298:%.*]] = zext i64 [[T_I_SROA_488_1]] to i128
; CHECK-NEXT: [[MUL57_I8179:%.*]] = mul i128 0, [[CONV66_I8295]]
; CHECK-NEXT: [[ADD135_I8204:%.*]] = or i128 [[ADD134_I8203]], [[MUL57_I8179]]
@@ -30,7 +22,6 @@ define i32 @test(i128 %add134.i8203) {
; CHECK-NEXT: [[ADD179_I8250:%.*]] = or i64 0, [[CONV137_I8208]]
; CHECK-NEXT: [[MUL45_I8175:%.*]] = mul i128 0, [[CONV76_I8298]]
; CHECK-NEXT: [[ADD153_I8224:%.*]] = or i128 0, [[MUL45_I8175]]
-; CHECK-NEXT: [[ADD155_I8226:%.*]] = or i128 [[ADD153_I8224]], 0
; CHECK-NEXT: [[SHR168_I8352:%.*]] = lshr i128 0, 0
; CHECK-NEXT: [[TMP3:%.*]] = trunc i128 [[SHR168_I8352]] to i64
; CHECK-NEXT: [[CONV177_I8361:%.*]] = and i64 [[TMP3]], 0
@@ -40,12 +31,17 @@ define i32 @test(i128 %add134.i8203) {
; CHECK-NEXT: [[CONV171_I8355:%.*]] = and i64 [[TMP4]], 0
; CHECK-NEXT: [[ARRAYIDX191_I8374:%.*]] = getelementptr i8, ptr poison, i64 64
; CHECK-NEXT: store i64 [[CONV171_I8355]], ptr [[ARRAYIDX191_I8374]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = mul <2 x i1> zeroinitializer, [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = or <2 x i1> zeroinitializer, [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = and <2 x i1> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
+; CHECK-NEXT: [[CONV137_I8321:%.*]] = zext i1 [[TMP13]] to i64
; CHECK-NEXT: [[SHR180_I8364:%.*]] = lshr i64 [[CONV137_I8321]], 1
-; CHECK-NEXT: [[CONV159_I8343:%.*]] = and i64 [[TMP1]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; CHECK-NEXT: [[CONV159_I8343:%.*]] = zext i1 [[TMP10]] to i64
; CHECK-NEXT: [[ADD182_I8366:%.*]] = or i64 [[SHR180_I8364]], [[CONV159_I8343]]
; CHECK-NEXT: [[SHR183_I8367:%.*]] = lshr i64 [[ADD182_I8366]], 0
-; CHECK-NEXT: [[CONV165_I8349:%.*]] = and i64 0, 0
-; CHECK-NEXT: [[ADD187_I8371:%.*]] = or i64 [[SHR183_I8367]], [[CONV165_I8349]]
+; CHECK-NEXT: [[ADD187_I8371:%.*]] = or i64 [[SHR183_I8367]], 0
; CHECK-NEXT: [[ARRAYIDX190_I8373:%.*]] = getelementptr i8, ptr poison, i64 56
; CHECK-NEXT: store i64 [[ADD187_I8371]], ptr [[ARRAYIDX190_I8373]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = trunc i128 0 to i64
@@ -57,7 +53,7 @@ define i32 @test(i128 %add134.i8203) {
; CHECK-NEXT: [[ARRAYIDX191_I8261:%.*]] = getelementptr i8, ptr poison, i64 104
; CHECK-NEXT: store i64 [[CONV171_I8242]], ptr [[ARRAYIDX191_I8261]], align 8
; CHECK-NEXT: [[SHR180_I8251:%.*]] = lshr i64 [[ADD179_I8250]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = trunc i128 [[ADD155_I8226]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i128 [[ADD153_I8224]] to i64
; CHECK-NEXT: [[ADD182_I8253:%.*]] = or i64 [[SHR180_I8251]], [[TMP7]]
; CHECK-NEXT: [[SHR183_I8254:%.*]] = lshr i64 [[ADD182_I8253]], 0
; CHECK-NEXT: [[ADD187_I8258:%.*]] = or i64 [[SHR183_I8254]], 0
diff --git a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
index e09b223b14599..c656ddec09dae 100644
--- a/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
+++ b/llvm/test/Transforms/SLPVectorizer/semanticly-same.ll
@@ -321,14 +321,17 @@ define void @lshr_div2(ptr %p, ptr %s) {
; X86-NEXT: [[P3:%.*]] = getelementptr i16, ptr [[P]], i64 3
; X86-NEXT: [[L0:%.*]] = load i16, ptr [[P]], align 2
; X86-NEXT: [[L3:%.*]] = load i16, ptr [[P3]], align 2
-; X86-NEXT: [[DIV0:%.*]] = lshr i16 [[L0]], 3
-; X86-NEXT: [[DIV3:%.*]] = lshr i16 [[L3]], 5
+; X86-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[L0]], i32 0
+; X86-NEXT: [[TMP4:%.*]] = insertelement <2 x i16> [[TMP3]], i16 [[L3]], i32 1
+; X86-NEXT: [[TMP2:%.*]] = lshr <2 x i16> [[TMP4]], <i16 3, i16 5>
; X86-NEXT: [[S1:%.*]] = getelementptr i16, ptr [[S:%.*]], i64 1
; X86-NEXT: [[S3:%.*]] = getelementptr i16, ptr [[S]], i64 3
; X86-NEXT: [[TMP0:%.*]] = load <2 x i16>, ptr [[P1]], align 2
; X86-NEXT: [[TMP1:%.*]] = udiv <2 x i16> [[TMP0]], <i16 8, i16 2>
+; X86-NEXT: [[DIV0:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
; X86-NEXT: store i16 [[DIV0]], ptr [[S]], align 2
; X86-NEXT: store <2 x i16> [[TMP1]], ptr [[S1]], align 2
+; X86-NEXT: [[DIV3:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
; X86-NEXT: store i16 [[DIV3]], ptr [[S3]], align 2
; X86-NEXT: ret void
;
>From bb4aebb8699b438d6105e1c30a40ec1a27e07603 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 26 Apr 2026 10:43:21 -0700
Subject: [PATCH 2/2] Small improvements
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 ++++++++++++-------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5817dcae33cab..ec44e19d8c2fe 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30141,8 +30141,7 @@ static void forEachOperandChainCandidate(Instruction *I, Func F) {
F(RI->getReturnValue(), 0);
return;
}
- for (auto [Idx, U] : enumerate(I->operands()))
- F(U.get(), Idx);
+ llvm_unreachable("Unexpected instruction kind for operand-chain seeding");
}
template <typename ItT>
@@ -30189,12 +30188,15 @@ bool SLPVectorizerPass::vectorizeNonVectorizableInsts(
IndirectCall,
};
Kind RootKind;
- unsigned KindID; // Intrinsic ID, opcode (+ atomicrmw op) or value ID.
+ unsigned KindID; // Intrinsic ID for Intrinsic, opcode for NonCall,
+ // callee value-kind for IndirectCall, 0 for
+ // NamedFunction.
+ unsigned SubOp; // AtomicRMW operation; 0 otherwise.
StringRef FuncName; // Non-empty only for NamedFunction.
unsigned Position; // Operand slot within the root.
bool operator==(const OperandGroupKey &O) const {
- return RootKind == O.RootKind && KindID == O.KindID &&
+ return RootKind == O.RootKind && KindID == O.KindID && SubOp == O.SubOp &&
FuncName == O.FuncName && Position == O.Position;
}
bool operator!=(const OperandGroupKey &O) const { return !(*this == O); }
@@ -30204,6 +30206,8 @@ bool SLPVectorizerPass::vectorizeNonVectorizableInsts(
static_cast<unsigned>(O.RootKind);
if (KindID != O.KindID)
return KindID < O.KindID;
+ if (SubOp != O.SubOp)
+ return SubOp < O.SubOp;
if (int C = FuncName.compare(O.FuncName))
return C < 0;
return Position < O.Position;
@@ -30216,20 +30220,23 @@ bool SLPVectorizerPass::vectorizeNonVectorizableInsts(
if (auto *II = dyn_cast<IntrinsicInst>(CB))
return {OperandGroupKey::Kind::Intrinsic,
II->getIntrinsicID(),
+ 0,
{},
Position};
if (Function *F = CB->getCalledFunction())
- return {OperandGroupKey::Kind::NamedFunction, 0, F->getName(),
+ return {OperandGroupKey::Kind::NamedFunction, 0, 0, F->getName(),
Position};
return {OperandGroupKey::Kind::IndirectCall,
CB->getCalledOperand()->getValueID(),
+ 0,
{},
Position};
}
- unsigned Discriminator = I->getOpcode();
+ unsigned SubOp = 0;
if (auto *AI = dyn_cast<AtomicRMWInst>(I))
- Discriminator |= static_cast<unsigned>(AI->getOperation()) << 16;
- return {OperandGroupKey::Kind::NonCall, Discriminator, {}, Position};
+ SubOp = static_cast<unsigned>(AI->getOperation());
+ return {
+ OperandGroupKey::Kind::NonCall, I->getOpcode(), SubOp, {}, Position};
};
auto OperandSorter = [&OpKeys](Value *V1, Value *V2) -> bool {
More information about the llvm-commits
mailing list