[llvm] [X86] Add pass to coalesce redundant vector constant materializations (PR #211619)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 12:51:16 PDT 2026
https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/211619
>From 35e03552cfa267e7e9796d96a792b84dae431636 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Thu, 23 Jul 2026 13:49:36 -0400
Subject: [PATCH] [X86] Add pass to coalesce redundant vector constant
materializations
During Instruction Selection, it is common to emit multiple redundant pseudo-instructions to materialize identical vector constants (e.g., V_SET0, AVX512_128_SETALLONES, FsFLD0SD) within the same basic block or across dominated regions. If left unoptimized, these redundancies persist through the pipeline, forcing the register allocator to assign multiple physical registers and emit unnecessary instructions (like redundant vpxors). This unnecessarily increases code size and execution port pressure.
This looks for redundant all-ones and all-zeros to coalesce.
---
llvm/lib/Target/X86/CMakeLists.txt | 1 +
llvm/lib/Target/X86/X86.h | 12 +
.../Target/X86/X86OptimizeVectorConstants.cpp | 304 +++++++++++
llvm/lib/Target/X86/X86PassRegistry.def | 1 +
llvm/lib/Target/X86/X86TargetMachine.cpp | 2 +
.../test/CodeGen/X86/2012-01-12-extract-sv.ll | 1 -
.../CodeGen/X86/apx/memfold-no-physreg.ll | 42 +-
.../X86/bitcast-int-to-vector-bool-sext.ll | 4 +-
.../test/CodeGen/X86/coalesce-vector-zeros.ll | 21 +
...alescer-copy-from-erasable-implicit-def.ll | 1 -
llvm/test/CodeGen/X86/combine-icmp.ll | 5 +-
llvm/test/CodeGen/X86/dpbusd.ll | 36 +-
llvm/test/CodeGen/X86/dpbusd_const.ll | 63 +--
llvm/test/CodeGen/X86/half.ll | 7 +-
llvm/test/CodeGen/X86/masked_gather.ll | 196 ++++---
llvm/test/CodeGen/X86/masked_load.ll | 502 +++++++++---------
llvm/test/CodeGen/X86/opt-pipeline.ll | 2 +
llvm/test/CodeGen/X86/pr120093.ll | 6 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 10 +-
llvm/test/CodeGen/X86/vec_ss_load_fold.ll | 78 +--
.../X86/vector-constants-coalescing.mir | 68 +++
llvm/test/CodeGen/X86/vector-pcmp.ll | 3 +-
.../CodeGen/X86/vector-reduce-add-sext.ll | 39 +-
llvm/test/CodeGen/X86/vector-reduce-ctpop.ll | 7 +-
.../CodeGen/X86/zero_extend_vector_inreg.ll | 31 +-
.../zero_extend_vector_inreg_of_broadcast.ll | 36 +-
...d_vector_inreg_of_broadcast_from_memory.ll | 67 ++-
27 files changed, 919 insertions(+), 626 deletions(-)
create mode 100644 llvm/lib/Target/X86/X86OptimizeVectorConstants.cpp
create mode 100644 llvm/test/CodeGen/X86/coalesce-vector-zeros.ll
create mode 100644 llvm/test/CodeGen/X86/vector-constants-coalescing.mir
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 62987bdbd1c2b..3f8876036f774 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -75,6 +75,7 @@ set(sources
X86MachineFunctionInfo.cpp
X86MacroFusion.cpp
X86OptimizeLEAs.cpp
+ X86OptimizeVectorConstants.cpp
X86PadShortFunction.cpp
X86PartialReduction.cpp
X86RegisterInfo.cpp
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 48dedd9d2a758..054449dad5ee3 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -131,6 +131,17 @@ class X86FixupVectorConstantsPass
FunctionPass *createX86FixupVectorConstantsLegacyPass();
+/// Return a pass that coalesces redundant vector zero and all-ones
+/// materializations into subregister extracts.
+class X86OptimizeVectorConstantsPass
+ : public OptionalPassInfoMixin<X86OptimizeVectorConstantsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
+FunctionPass *createX86OptimizeVectorConstantsLegacyPass();
+
/// Return a pass that removes redundant LEA instructions and redundant address
/// recalculations.
class X86OptimizeLEAsPass : public OptionalPassInfoMixin<X86OptimizeLEAsPass> {
@@ -483,6 +494,7 @@ void initializeX86ArgumentStackSlotLegacyPass(PassRegistry &);
void initializeX86AsmPrinterPass(PassRegistry &);
void initializeX86FixupInstTuningLegacyPass(PassRegistry &);
void initializeX86FixupVectorConstantsLegacyPass(PassRegistry &);
+void initializeX86OptimizeVectorConstantsLegacyPass(PassRegistry &);
void initializeWinEHStateLegacyPass(PassRegistry &);
void initializeX86AvoidSFBLegacyPass(PassRegistry &);
void initializeX86AvoidTrailingCallLegacyPassPass(PassRegistry &);
diff --git a/llvm/lib/Target/X86/X86OptimizeVectorConstants.cpp b/llvm/lib/Target/X86/X86OptimizeVectorConstants.cpp
new file mode 100644
index 0000000000000..9cf478f2ed07f
--- /dev/null
+++ b/llvm/lib/Target/X86/X86OptimizeVectorConstants.cpp
@@ -0,0 +1,304 @@
+//===- X86OptimizeVectorConstants.cpp - Optimize Vector Constants ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass coalesces redundant vector zero and all-ones materializations into
+// subregister copies or register replacements from a single larger
+// materialization, reducing redundant instructions before register allocation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+#include <array>
+#include <optional>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-optimize-vector-constants"
+
+STATISTIC(NumZeroConstsCoalesced, "Number of zero vector constants coalesced");
+STATISTIC(NumAllOnesConstsCoalesced,
+ "Number of all-ones vector constants coalesced");
+
+namespace {
+
+/// The kind of vector constant being materialized.
+enum class VectorConstKind : unsigned {
+ Zero,
+ AllOnes,
+ Count,
+};
+
+/// Describes a recognized vector constant materialization.
+struct VectorConstInfo {
+ unsigned Size; // 128, 256, or 512
+ VectorConstKind Kind;
+};
+
+class X86OptimizeVectorConstantsImpl {
+ MachineRegisterInfo *MRI = nullptr;
+ const TargetInstrInfo *TII = nullptr;
+ const X86Subtarget *Subtarget = nullptr;
+ bool Changed = false;
+
+ /// Classify a machine instruction as a vector constant materialization.
+ /// Returns the size and kind (zero vs all-ones) if recognized.
+ static std::optional<VectorConstInfo>
+ classifyConstMaterialization(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ // Zeros
+ case X86::V_SET0:
+ case X86::AVX512_128_SET0:
+ case X86::FsFLD0SH:
+ case X86::FsFLD0SS:
+ case X86::FsFLD0SD:
+ case X86::FsFLD0F128:
+ case X86::AVX512_FsFLD0SH:
+ case X86::AVX512_FsFLD0SS:
+ case X86::AVX512_FsFLD0SD:
+ case X86::AVX512_FsFLD0F128:
+ return VectorConstInfo{128, VectorConstKind::Zero};
+ case X86::AVX_SET0:
+ case X86::AVX512_256_SET0:
+ return VectorConstInfo{256, VectorConstKind::Zero};
+ case X86::AVX512_512_SET0:
+ return VectorConstInfo{512, VectorConstKind::Zero};
+ // All-Ones
+ case X86::V_SETALLONES:
+ case X86::AVX512_128_SETALLONES:
+ return VectorConstInfo{128, VectorConstKind::AllOnes};
+ case X86::AVX1_SETALLONES:
+ case X86::AVX2_SETALLONES:
+ case X86::AVX512_256_SETALLONES:
+ return VectorConstInfo{256, VectorConstKind::AllOnes};
+ case X86::AVX512_512_SETALLONES:
+ return VectorConstInfo{512, VectorConstKind::AllOnes};
+ default:
+ return std::nullopt;
+ }
+ }
+
+ static unsigned getSizeIndex(unsigned Size) {
+ switch (Size) {
+ case 128:
+ return 0;
+ case 256:
+ return 1;
+ case 512:
+ return 2;
+ }
+ llvm_unreachable("Unexpected vector size");
+ }
+
+ static unsigned getSubReg(unsigned ParentSize, unsigned ChildSize) {
+ // ActiveSetsT only stores a materialization for equal or larger sizes.
+ assert(ParentSize >= ChildSize && "Invalid sizes for subregister");
+ if (ParentSize == ChildSize)
+ return 0;
+ if (ParentSize == 512 && ChildSize == 256)
+ return X86::sub_ymm;
+ if (ParentSize == 512 && ChildSize == 128)
+ return X86::sub_xmm;
+ if (ParentSize == 256 && ChildSize == 128)
+ return X86::sub_xmm;
+ llvm_unreachable("Unexpected size combination");
+ }
+
+ /// Active[Kind][SizeIdx] tracks the closest dominating materialization of
+ /// the given kind and size-or-larger.
+ static constexpr unsigned NumKinds =
+ static_cast<unsigned>(VectorConstKind::Count);
+ static constexpr unsigned NumSizes = 3;
+ using ActiveSetsT =
+ std::array<std::array<MachineInstr *, NumSizes>, NumKinds>;
+
+ void processNode(MachineDomTreeNode *Node, ActiveSetsT Active);
+
+public:
+ bool run(MachineFunction &MF, MachineDominatorTree &MDT);
+};
+
+class X86OptimizeVectorConstantsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ X86OptimizeVectorConstantsLegacy() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return "X86 Optimize Vector Constants";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineLoopInfoWrapperPass>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ X86OptimizeVectorConstantsImpl Impl;
+ return Impl.run(
+ MF, getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree());
+ }
+};
+
+char X86OptimizeVectorConstantsLegacy::ID = 0;
+
+} // end anonymous namespace
+
+INITIALIZE_PASS_BEGIN(X86OptimizeVectorConstantsLegacy, DEBUG_TYPE,
+ "X86 Optimize Vector Constants", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(X86OptimizeVectorConstantsLegacy, DEBUG_TYPE,
+ "X86 Optimize Vector Constants", false, false)
+
+FunctionPass *llvm::createX86OptimizeVectorConstantsLegacyPass() {
+ return new X86OptimizeVectorConstantsLegacy();
+}
+
+PreservedAnalyses
+X86OptimizeVectorConstantsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ X86OptimizeVectorConstantsImpl Impl;
+ auto &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
+ bool Changed = Impl.run(MF, MDT);
+ if (!Changed)
+ return PreservedAnalyses::all();
+ auto PA = PreservedAnalyses::none();
+ PA.preserve<MachineDominatorTreeAnalysis>();
+ PA.preserve<MachineLoopAnalysis>();
+ return PA;
+}
+
+void X86OptimizeVectorConstantsImpl::processNode(MachineDomTreeNode *Node,
+ ActiveSetsT Active) {
+ MachineBasicBlock *MBB = Node->getBlock();
+
+ for (auto &MI : llvm::make_early_inc_range(*MBB)) {
+ auto OptInfo = classifyConstMaterialization(MI);
+ if (!OptInfo)
+ continue;
+ if (!MI.getOperand(0).isReg() || !MI.getOperand(0).getReg().isVirtual())
+ continue;
+
+ unsigned Size = OptInfo->Size;
+ unsigned KindIdx = static_cast<unsigned>(OptInfo->Kind);
+ unsigned SizeIdx = getSizeIndex(Size);
+
+ MachineInstr *DomConst = Active[KindIdx][SizeIdx];
+ if (DomConst) {
+ auto DomInfo = classifyConstMaterialization(*DomConst);
+ assert(DomInfo && "Active constant must classify");
+ unsigned DomSize = DomInfo->Size;
+
+ Register OldReg = MI.getOperand(0).getReg();
+ Register NewReg = DomConst->getOperand(0).getReg();
+
+ unsigned SubIdx = getSubReg(DomSize, Size);
+
+ const TargetRegisterClass *OldRC = MRI->getRegClass(OldReg);
+ const TargetRegisterClass *NewRC = MRI->getRegClass(NewReg);
+
+ LLVM_DEBUG(dbgs() << "Coalescing: " << MI
+ << " with dominator: " << *DomConst);
+
+ if (!Subtarget->hasAVX()) {
+ bool HasTwoAddressUser = false;
+ for (const MachineOperand &UseMO : MRI->use_operands(OldReg)) {
+ const MachineInstr *UseMI = UseMO.getParent();
+ for (unsigned I = 0, E = UseMI->getNumOperands(); I != E; ++I) {
+ const MachineOperand &MO = UseMI->getOperand(I);
+ if (MO.isReg() && MO.isUse() && MO.isTied()) {
+ HasTwoAddressUser = true;
+ break;
+ }
+ }
+ if (HasTwoAddressUser)
+ break;
+ }
+
+ // Avoid coalescing on non-AVX targets if the constant is consumed by a
+ // two-address instruction, as extending live ranges can disrupt
+ // TwoAddressInstructionPass commutativity and force extra copies.
+ if (HasTwoAddressUser) {
+ LLVM_DEBUG(dbgs() << "Skipping coalesce due to two-address user on "
+ "non-AVX target\n");
+ continue;
+ }
+ }
+
+ if (DomSize == Size && OldRC == NewRC) {
+ // The constant pseudo has no side effects and DomConst dominates this
+ // definition, so all uses can be redirected to the dominating vreg.
+ assert(MRI->hasOneDef(OldReg) &&
+ "Expected single-def vreg from isel constant pseudo");
+ MRI->replaceRegWith(OldReg, NewReg);
+ } else {
+ // Emit a COPY with a subregister index. Later register allocation
+ // and copy coalescing will reconcile any compatible register-class
+ // constraints.
+ BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
+ TII->get(TargetOpcode::COPY), OldReg)
+ .addReg(NewReg, RegState(), SubIdx);
+ }
+ // clearKillFlags must be run unconditionally, as either branch adds a new
+ // use of NewReg which may invalidate a previous kill flag.
+ MRI->clearKillFlags(NewReg);
+
+ MI.eraseFromParent();
+ switch (OptInfo->Kind) {
+ case VectorConstKind::Zero:
+ ++NumZeroConstsCoalesced;
+ break;
+ case VectorConstKind::AllOnes:
+ ++NumAllOnesConstsCoalesced;
+ break;
+ case VectorConstKind::Count:
+ llvm_unreachable("Invalid vector constant kind");
+ }
+ Changed = true;
+ continue;
+ }
+
+ // If we didn't coalesce, MI becomes the closest active constant for its
+ // kind at its size and all smaller sizes.
+ for (unsigned I = 0; I <= SizeIdx; ++I)
+ Active[KindIdx][I] = &MI;
+ }
+
+ for (auto *Child : *Node)
+ processNode(Child, Active);
+}
+
+bool X86OptimizeVectorConstantsImpl::run(MachineFunction &MF,
+ MachineDominatorTree &MDT) {
+ this->MRI = &MF.getRegInfo();
+ this->TII = MF.getSubtarget().getInstrInfo();
+ this->Subtarget = &MF.getSubtarget<X86Subtarget>();
+ this->Changed = false;
+
+ ActiveSetsT Active{};
+ processNode(MDT.getRootNode(), Active);
+
+ return Changed;
+}
diff --git a/llvm/lib/Target/X86/X86PassRegistry.def b/llvm/lib/Target/X86/X86PassRegistry.def
index 45e7d0ebdbf7b..14f1ff9aaa2b0 100644
--- a/llvm/lib/Target/X86/X86PassRegistry.def
+++ b/llvm/lib/Target/X86/X86PassRegistry.def
@@ -57,6 +57,7 @@ MACHINE_FUNCTION_PASS("x86-lower-tile-copy", X86LowerTileCopyPass())
MACHINE_FUNCTION_PASS("x86-lvi-load", X86LoadValueInjectionLoadHardeningPass())
MACHINE_FUNCTION_PASS("x86-lvi-ret", X86LoadValueInjectionRetHardeningPass())
MACHINE_FUNCTION_PASS("x86-optimize-leas", X86OptimizeLEAsPass())
+MACHINE_FUNCTION_PASS("x86-optimize-vector-constants", X86OptimizeVectorConstantsPass())
MACHINE_FUNCTION_PASS("x86-postlegalizer-combiner-pass", X86PostLegalizerCombinerPass())
MACHINE_FUNCTION_PASS("x86-pre-tile-config", X86PreTileConfigPass())
MACHINE_FUNCTION_PASS("x86-prelegalizer-combiner-pass", X86PreLegalizerCombinerPass())
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 932669b5cbac6..6a634b8247090 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -104,6 +104,7 @@ extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
initializeX86AsmPrinterPass(PR);
initializeX86FixupInstTuningLegacyPass(PR);
initializeX86FixupVectorConstantsLegacyPass(PR);
+ initializeX86OptimizeVectorConstantsLegacyPass(PR);
initializeX86DynAllocaExpanderLegacyPass(PR);
initializeX86SuppressAPXForRelocationLegacyPass(PR);
initializeX86WinEHUnwindV2LegacyPass(PR);
@@ -517,6 +518,7 @@ void X86PassConfig::addPreRegAlloc() {
addPass(&LiveRangeShrinkID);
addPass(createX86FixupSetCCLegacyPass());
addPass(createX86OptimizeLEAsLegacyPass());
+ addPass(createX86OptimizeVectorConstantsLegacyPass());
addPass(createX86CallFrameOptimizationLegacyPass());
addPass(createX86AvoidStoreForwardingBlocksLegacyPass());
}
diff --git a/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll b/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
index 65273870c3dfb..f8b6d3eb25dab 100644
--- a/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
+++ b/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
@@ -10,7 +10,6 @@ define void @endless_loop() {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
; AVX1-NEXT: vmovaps %ymm0, (%eax)
; AVX1-NEXT: vmovaps %ymm1, (%eax)
diff --git a/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
index 5142a5ef0c9e8..a458ba31581fe 100644
--- a/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
+++ b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
@@ -55,6 +55,7 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: subq %rax, %rsp
; CHECK-NEXT: andq $-32, %rsp
; CHECK-NEXT: movl %ecx, %r8d
+; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: xorl %r15d, %r15d
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: kmovw %k2, 14(%rbx) # 2-byte Spill
@@ -116,6 +117,7 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: movq 296(%rbp), %r27
; CHECK-NEXT: movq 288(%rbp), %r20
; CHECK-NEXT: movq 280(%rbp), %r21
+; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: movq 24(%rbx), %r8 # 8-byte Reload
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
; CHECK-NEXT: subq %r21, %r15
@@ -136,12 +138,11 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: movl %r24d, %r27d
; CHECK-NEXT: movq 336(%rbp), %r24
; CHECK-NEXT: movq 216(%rbp), %r28
-; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-NEXT: vmovdqu32 %ymm0, (%r28,%r24) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm2, (%r28,%r24) {%k2}
; CHECK-NEXT: movl %r27d, %r24d
; CHECK-NEXT: movq %r20, %r27
; CHECK-NEXT: movq %rax, %r20
-; CHECK-NEXT: vmovups %ymm0, (%rcx) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%rcx) {%k2}
; CHECK-NEXT: cmpl %edx, %r25d
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
; CHECK-NEXT: je .LBB0_6
@@ -150,27 +151,26 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: jmp .LBB0_13
; CHECK-NEXT: .LBB0_3: # %vector_loop.preheader
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_4: # %vector_loop
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vmovdqu (%r9), %ymm1
-; CHECK-NEXT: vpord (%r18), %ymm1, %ymm1
-; CHECK-NEXT: vmovups %ymm0, (%r23) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r12) {%k2}
-; CHECK-NEXT: vmovups (%r13,%r15), %ymm2
-; CHECK-NEXT: vcmpltps (%r30,%r15), %ymm2, %k1
-; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k1 {%k1}
-; CHECK-NEXT: vmovups (%r10), %ymm1 {%k1} {z}
-; CHECK-NEXT: vmovups %ymm1, (%r22) {%k2}
-; CHECK-NEXT: vmovdqu32 %ymm0, (%rdi) {%k2}
-; CHECK-NEXT: vmovdqu32 %ymm0, (%r16) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r19) {%k2}
-; CHECK-NEXT: vmovups (%r31), %ymm1 {%k2} {z}
-; CHECK-NEXT: vmovups %ymm1, (%r17,%r29,4) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r11) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%rsi) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r14) {%k2}
+; CHECK-NEXT: vmovdqu (%r9), %ymm0
+; CHECK-NEXT: vpord (%r18), %ymm0, %ymm0
+; CHECK-NEXT: vmovups %ymm2, (%r23) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%r12) {%k2}
+; CHECK-NEXT: vmovups (%r13,%r15), %ymm1
+; CHECK-NEXT: vcmpltps (%r30,%r15), %ymm1, %k1
+; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k1 {%k1}
+; CHECK-NEXT: vmovups (%r10), %ymm0 {%k1} {z}
+; CHECK-NEXT: vmovups %ymm0, (%r22) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm2, (%rdi) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm2, (%r16) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%r19) {%k2}
+; CHECK-NEXT: vmovups (%r31), %ymm0 {%k2} {z}
+; CHECK-NEXT: vmovups %ymm0, (%r17,%r29,4) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%r11) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%rsi) {%k2}
+; CHECK-NEXT: vmovups %ymm2, (%r14) {%k2}
; CHECK-NEXT: incq %rax
; CHECK-NEXT: addq $4, %r15
; CHECK-NEXT: jmp .LBB0_4
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index 474be4465d9b7..1e6fb82a43322 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -729,10 +729,8 @@ define <8 x i32> @PR157382(ptr %p0, ptr %p1, ptr %p2) {
; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4
; AVX2-NEXT: vpcmpgtd %ymm3, %ymm1, %ymm1
; AVX2-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm5, %xmm2, %xmm2
; AVX2-NEXT: vpmovsxbd %xmm2, %ymm2
; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/coalesce-vector-zeros.ll b/llvm/test/CodeGen/X86/coalesce-vector-zeros.ll
new file mode 100644
index 0000000000000..4ae2884145eda
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coalesce-vector-zeros.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s
+
+; Verify that a scalar double 0.0 and a <4 x double> zeroinitializer in the same
+; function share the same zero register rather than generating separate vxorpd instructions.
+define double @test_coalesce_zeros(ptr %p, double %d) {
+; CHECK-LABEL: test_coalesce_zeros:
+; CHECK: # %bb.0:
+; CHECK: vxorp{{s|d}} %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}}
+; CHECK-NOT: vxorpd
+; CHECK-NOT: vxorps
+; CHECK: vmovap{{s|d}} %ymm
+; CHECK: vmaxsd
+; CHECK: retq
+
+entry:
+ store <4 x double> zeroinitializer, ptr %p
+ %z_scalar = call double @llvm.maxnum.f64(double 0.0, double %d)
+ ret double %z_scalar
+}
+
+declare double @llvm.maxnum.f64(double, double)
diff --git a/llvm/test/CodeGen/X86/coalescer-copy-from-erasable-implicit-def.ll b/llvm/test/CodeGen/X86/coalescer-copy-from-erasable-implicit-def.ll
index 7b92777cf54c3..d65edd50e7feb 100644
--- a/llvm/test/CodeGen/X86/coalescer-copy-from-erasable-implicit-def.ll
+++ b/llvm/test/CodeGen/X86/coalescer-copy-from-erasable-implicit-def.ll
@@ -31,7 +31,6 @@ define <16 x i32> @f1() {
; CHECK-NEXT: je .LBB0_5
; CHECK-NEXT: # %bb.6:
; CHECK-NEXT: xorps %xmm0, %xmm0
-; CHECK-NEXT: xorps %xmm1, %xmm1
; CHECK-NEXT: xorps %xmm2, %xmm2
; CHECK-NEXT: xorps %xmm3, %xmm3
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/combine-icmp.ll b/llvm/test/CodeGen/X86/combine-icmp.ll
index 3b44339dc0b4a..b9cdaa04c1055 100644
--- a/llvm/test/CodeGen/X86/combine-icmp.ll
+++ b/llvm/test/CodeGen/X86/combine-icmp.ll
@@ -646,9 +646,8 @@ define i8 @concat_icmp_v8i64_v4i64(<4 x i64> %a0, <4 x i64> %a1) {
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
; AVX2-NEXT: vpackssdw %xmm3, %xmm0, %xmm0
; AVX2-NEXT: vpcmpeqq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpackssdw %xmm3, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index 7bd22d57347b3..a22cb9c1bec38 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -335,30 +335,17 @@ entry:
declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
-; AVXVNNI-AVX-LABEL: vpdpbusd_128:
-; AVXVNNI-AVX: # %bb.0: # %entry
-; AVXVNNI-AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVXVNNI-AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVXVNNI-AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
-; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
-; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2
-; AVXVNNI-AVX-NEXT: vmovd %xmm2, %eax
-; AVXVNNI-AVX-NEXT: addl %edx, %eax
-; AVXVNNI-AVX-NEXT: retq
-;
-; AVXVNNI-AVX512-LABEL: vpdpbusd_128:
-; AVXVNNI-AVX512: # %bb.0: # %entry
-; AVXVNNI-AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVXVNNI-AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
-; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
-; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2
-; AVXVNNI-AVX512-NEXT: vmovd %xmm2, %eax
-; AVXVNNI-AVX512-NEXT: addl %edx, %eax
-; AVXVNNI-AVX512-NEXT: retq
+; AVXVNNI-LABEL: vpdpbusd_128:
+; AVXVNNI: # %bb.0: # %entry
+; AVXVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVXVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVXVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVXVNNI-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVXVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVXVNNI-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2
+; AVXVNNI-NEXT: vmovd %xmm2, %eax
+; AVXVNNI-NEXT: addl %edx, %eax
+; AVXVNNI-NEXT: retq
;
; AVX512VNNI-LABEL: vpdpbusd_128:
; AVX512VNNI: # %bb.0: # %entry
@@ -381,7 +368,6 @@ define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
-; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512VLVNNI-NEXT: vpdpbusd %xmm1, %xmm0, %xmm2
; AVX512VLVNNI-NEXT: vmovd %xmm2, %eax
; AVX512VLVNNI-NEXT: addl %edx, %eax
diff --git a/llvm/test/CodeGen/X86/dpbusd_const.ll b/llvm/test/CodeGen/X86/dpbusd_const.ll
index 1d6c3f7c5c6e8..6c2715b75f56d 100644
--- a/llvm/test/CodeGen/X86/dpbusd_const.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_const.ll
@@ -25,24 +25,14 @@ entry:
}
define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {
-; AVXVNNI-AVX-LABEL: mul_4xi8_zc:
-; AVXVNNI-AVX: # %bb.0: # %entry
-; AVXVNNI-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVXVNNI-AVX-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVXVNNI-AVX-NEXT: vmovd %xmm1, %eax
-; AVXVNNI-AVX-NEXT: addl %edi, %eax
-; AVXVNNI-AVX-NEXT: retq
-;
-; AVXVNNI-AVX512-LABEL: mul_4xi8_zc:
-; AVXVNNI-AVX512: # %bb.0: # %entry
-; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVXVNNI-AVX512-NEXT: vmovd %xmm1, %eax
-; AVXVNNI-AVX512-NEXT: addl %edi, %eax
-; AVXVNNI-AVX512-NEXT: retq
+; AVXVNNI-LABEL: mul_4xi8_zc:
+; AVXVNNI: # %bb.0: # %entry
+; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVXVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVXVNNI-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVXVNNI-NEXT: vmovd %xmm1, %eax
+; AVXVNNI-NEXT: addl %edi, %eax
+; AVXVNNI-NEXT: retq
;
; AVX512VNNI-LABEL: mul_4xi8_zc:
; AVX512VNNI: # %bb.0: # %entry
@@ -59,7 +49,6 @@ define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {
; AVX512VLVNNI: # %bb.0: # %entry
; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLVNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
; AVX512VLVNNI-NEXT: vmovd %xmm1, %eax
; AVX512VLVNNI-NEXT: addl %edi, %eax
@@ -125,26 +114,15 @@ entry:
}
define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {
-; AVXVNNI-AVX-LABEL: mul_4xi8_cs:
-; AVXVNNI-AVX: # %bb.0: # %entry
-; AVXVNNI-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVXVNNI-AVX-NEXT: vmovd {{.*#+}} xmm2 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %xmm0, %xmm2, %xmm1
-; AVXVNNI-AVX-NEXT: vmovd %xmm1, %eax
-; AVXVNNI-AVX-NEXT: addl %edi, %eax
-; AVXVNNI-AVX-NEXT: retq
-;
-; AVXVNNI-AVX512-LABEL: mul_4xi8_cs:
-; AVXVNNI-AVX512: # %bb.0: # %entry
-; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVXVNNI-AVX512-NEXT: vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %xmm0, %xmm1, %xmm2
-; AVXVNNI-AVX512-NEXT: vmovd %xmm2, %eax
-; AVXVNNI-AVX512-NEXT: addl %edi, %eax
-; AVXVNNI-AVX512-NEXT: retq
+; AVXVNNI-LABEL: mul_4xi8_cs:
+; AVXVNNI: # %bb.0: # %entry
+; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVXVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVXVNNI-NEXT: vmovd {{.*#+}} xmm2 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVXVNNI-NEXT: {vex} vpdpbusd %xmm0, %xmm2, %xmm1
+; AVXVNNI-NEXT: vmovd %xmm1, %eax
+; AVXVNNI-NEXT: addl %edi, %eax
+; AVXVNNI-NEXT: retq
;
; AVX512VNNI-LABEL: mul_4xi8_cs:
; AVX512VNNI: # %bb.0: # %entry
@@ -162,10 +140,9 @@ define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {
; AVX512VLVNNI: # %bb.0: # %entry
; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512VLVNNI-NEXT: vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLVNNI-NEXT: vpdpbusd %xmm0, %xmm1, %xmm2
-; AVX512VLVNNI-NEXT: vmovd %xmm2, %eax
+; AVX512VLVNNI-NEXT: vmovd {{.*#+}} xmm2 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512VLVNNI-NEXT: vpdpbusd %xmm0, %xmm2, %xmm1
+; AVX512VLVNNI-NEXT: vmovd %xmm1, %eax
; AVX512VLVNNI-NEXT: addl %edi, %eax
; AVX512VLVNNI-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index b6a4a12eb0fac..b5296e342b50f 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -1058,10 +1058,9 @@ define void @main.158() #0 {
; BWON-F16C-LABEL: main.158:
; BWON-F16C: # %bb.0: # %entry
; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0
-; BWON-F16C-NEXT: vmovss {{.*#+}} xmm1 = [8.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; BWON-F16C-NEXT: vucomiss %xmm0, %xmm1
-; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm1
+; BWON-F16C-NEXT: vmovss {{.*#+}} xmm2 = [8.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; BWON-F16C-NEXT: vucomiss %xmm1, %xmm2
; BWON-F16C-NEXT: jae .LBB20_2
; BWON-F16C-NEXT: # %bb.1: # %entry
; BWON-F16C-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
diff --git a/llvm/test/CodeGen/X86/masked_gather.ll b/llvm/test/CodeGen/X86/masked_gather.ll
index d55c09cf4a033..2bff974dae456 100644
--- a/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/llvm/test/CodeGen/X86/masked_gather.ll
@@ -1530,11 +1530,11 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
;
; AVX1-LABEL: gather_v8i32_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm3, %xmm1
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm1
; AVX1-NEXT: vmovmskps %ymm1, %eax
; AVX1-NEXT: testb $1, %al
; AVX1-NEXT: # implicit-def: $ymm1
@@ -1561,16 +1561,15 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX1-NEXT: testb $-128, %al
; AVX1-NEXT: je .LBB5_16
; AVX1-NEXT: .LBB5_15: # %cond.load19
-; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
+; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm4[7]
; AVX1-NEXT: .LBB5_16: # %else20
-; AVX1-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT: vmovmskps %ymm3, %eax
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm3, %xmm4
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm5
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX1-NEXT: vmovmskps %ymm4, %eax
; AVX1-NEXT: testb $1, %al
-; AVX1-NEXT: # implicit-def: $ymm3
+; AVX1-NEXT: # implicit-def: $ymm4
; AVX1-NEXT: jne .LBB5_17
; AVX1-NEXT: # %bb.18: # %else26
; AVX1-NEXT: testb $2, %al
@@ -1594,13 +1593,12 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX1-NEXT: testb $-128, %al
; AVX1-NEXT: je .LBB5_32
; AVX1-NEXT: .LBB5_31: # %cond.load58
-; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm4[7]
+; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm5[7]
; AVX1-NEXT: .LBB5_32: # %else61
-; AVX1-NEXT: vxorps %xmm4, %xmm4, %xmm4
-; AVX1-NEXT: vpcmpeqd %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: vmovmskps %ymm0, %eax
; AVX1-NEXT: testb $1, %al
; AVX1-NEXT: # implicit-def: $ymm0
@@ -1621,16 +1619,16 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm2
; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6,7]
; AVX1-NEXT: .LBB5_42: # %else87
-; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT: vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vpaddd %xmm4, %xmm1, %xmm1
; AVX1-NEXT: testb $32, %al
; AVX1-NEXT: je .LBB5_44
; AVX1-NEXT: # %bb.43: # %cond.load89
-; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
+; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
; AVX1-NEXT: .LBB5_44: # %else92
-; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vpaddd %xmm2, %xmm3, %xmm2
; AVX1-NEXT: testb $64, %al
; AVX1-NEXT: je .LBB5_46
; AVX1-NEXT: # %bb.45: # %cond.load94
@@ -1655,68 +1653,68 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX1-NEXT: testb $2, %al
; AVX1-NEXT: je .LBB5_4
; AVX1-NEXT: .LBB5_3: # %cond.load1
-; AVX1-NEXT: vpinsrd $1, c+12(%rip), %xmm1, %xmm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT: vpinsrd $1, c+12(%rip), %xmm1, %xmm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
; AVX1-NEXT: testb $4, %al
; AVX1-NEXT: je .LBB5_6
; AVX1-NEXT: .LBB5_5: # %cond.load4
-; AVX1-NEXT: vpinsrd $2, c+12(%rip), %xmm1, %xmm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT: vpinsrd $2, c+12(%rip), %xmm1, %xmm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
; AVX1-NEXT: testb $8, %al
; AVX1-NEXT: je .LBB5_8
; AVX1-NEXT: .LBB5_7: # %cond.load7
-; AVX1-NEXT: vpinsrd $3, c+12(%rip), %xmm1, %xmm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT: vpinsrd $3, c+12(%rip), %xmm1, %xmm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
; AVX1-NEXT: testb $16, %al
; AVX1-NEXT: je .LBB5_10
; AVX1-NEXT: .LBB5_9: # %cond.load10
-; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4],ymm1[5,6,7]
+; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4],ymm1[5,6,7]
; AVX1-NEXT: testb $32, %al
; AVX1-NEXT: je .LBB5_12
; AVX1-NEXT: .LBB5_11: # %cond.load13
-; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
+; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5],ymm1[6,7]
; AVX1-NEXT: testb $64, %al
; AVX1-NEXT: je .LBB5_14
; AVX1-NEXT: .LBB5_13: # %cond.load16
-; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm3
-; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6],ymm1[7]
+; AVX1-NEXT: vbroadcastss c+12(%rip), %ymm4
+; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6],ymm1[7]
; AVX1-NEXT: testb $-128, %al
; AVX1-NEXT: jne .LBB5_15
; AVX1-NEXT: jmp .LBB5_16
; AVX1-NEXT: .LBB5_17: # %cond.load23
-; AVX1-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
; AVX1-NEXT: testb $2, %al
; AVX1-NEXT: je .LBB5_20
; AVX1-NEXT: .LBB5_19: # %cond.load28
-; AVX1-NEXT: vpinsrd $1, c+28(%rip), %xmm3, %xmm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT: vpinsrd $1, c+28(%rip), %xmm4, %xmm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
; AVX1-NEXT: testb $4, %al
; AVX1-NEXT: je .LBB5_22
; AVX1-NEXT: .LBB5_21: # %cond.load33
-; AVX1-NEXT: vpinsrd $2, c+28(%rip), %xmm3, %xmm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT: vpinsrd $2, c+28(%rip), %xmm4, %xmm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
; AVX1-NEXT: testb $8, %al
; AVX1-NEXT: je .LBB5_24
; AVX1-NEXT: .LBB5_23: # %cond.load38
-; AVX1-NEXT: vpinsrd $3, c+28(%rip), %xmm3, %xmm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT: vpinsrd $3, c+28(%rip), %xmm4, %xmm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
; AVX1-NEXT: testb $16, %al
; AVX1-NEXT: je .LBB5_26
; AVX1-NEXT: .LBB5_25: # %cond.load43
-; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6,7]
+; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6,7]
; AVX1-NEXT: testb $32, %al
; AVX1-NEXT: je .LBB5_28
; AVX1-NEXT: .LBB5_27: # %cond.load48
-; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5],ymm3[6,7]
+; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6,7]
; AVX1-NEXT: testb $64, %al
; AVX1-NEXT: je .LBB5_30
; AVX1-NEXT: .LBB5_29: # %cond.load53
-; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm4
-; AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6],ymm3[7]
+; AVX1-NEXT: vbroadcastss c+28(%rip), %ymm5
+; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm5[6],ymm4[7]
; AVX1-NEXT: testb $-128, %al
; AVX1-NEXT: jne .LBB5_31
; AVX1-NEXT: jmp .LBB5_32
@@ -1743,8 +1741,8 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
;
; AVX2-NOGATHER-LABEL: gather_v8i32_v8i32:
; AVX2-NOGATHER: # %bb.0:
-; AVX2-NOGATHER-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm1
+; AVX2-NOGATHER-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm1
; AVX2-NOGATHER-NEXT: vmovmskps %ymm1, %eax
; AVX2-NOGATHER-NEXT: testb $1, %al
; AVX2-NOGATHER-NEXT: # implicit-def: $ymm1
@@ -1771,14 +1769,13 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX2-NOGATHER-NEXT: testb $-128, %al
; AVX2-NOGATHER-NEXT: je .LBB5_16
; AVX2-NOGATHER-NEXT: .LBB5_15: # %cond.load19
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
; AVX2-NOGATHER-NEXT: .LBB5_16: # %else20
-; AVX2-NOGATHER-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm2
-; AVX2-NOGATHER-NEXT: vmovmskps %ymm2, %eax
+; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm3
+; AVX2-NOGATHER-NEXT: vmovmskps %ymm3, %eax
; AVX2-NOGATHER-NEXT: testb $1, %al
-; AVX2-NOGATHER-NEXT: # implicit-def: $ymm2
+; AVX2-NOGATHER-NEXT: # implicit-def: $ymm3
; AVX2-NOGATHER-NEXT: jne .LBB5_17
; AVX2-NOGATHER-NEXT: # %bb.18: # %else26
; AVX2-NOGATHER-NEXT: testb $2, %al
@@ -1802,11 +1799,10 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX2-NOGATHER-NEXT: testb $-128, %al
; AVX2-NOGATHER-NEXT: je .LBB5_32
; AVX2-NOGATHER-NEXT: .LBB5_31: # %cond.load58
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm4[7]
; AVX2-NOGATHER-NEXT: .LBB5_32: # %else61
-; AVX2-NOGATHER-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
+; AVX2-NOGATHER-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm0
; AVX2-NOGATHER-NEXT: vmovmskps %ymm0, %eax
; AVX2-NOGATHER-NEXT: testb $1, %al
; AVX2-NOGATHER-NEXT: # implicit-def: $ymm0
@@ -1830,10 +1826,10 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX2-NOGATHER-NEXT: testb $64, %al
; AVX2-NOGATHER-NEXT: je .LBB5_46
; AVX2-NOGATHER-NEXT: .LBB5_45: # %cond.load94
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6],ymm0[7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7]
; AVX2-NOGATHER-NEXT: .LBB5_46: # %else97
-; AVX2-NOGATHER-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NOGATHER-NEXT: vpaddd %ymm3, %ymm1, %ymm1
; AVX2-NOGATHER-NEXT: testb $-128, %al
; AVX2-NOGATHER-NEXT: je .LBB5_48
; AVX2-NOGATHER-NEXT: # %bb.47: # %cond.load99
@@ -1847,68 +1843,68 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX2-NOGATHER-NEXT: testb $2, %al
; AVX2-NOGATHER-NEXT: je .LBB5_4
; AVX2-NOGATHER-NEXT: .LBB5_3: # %cond.load1
-; AVX2-NOGATHER-NEXT: vpinsrd $1, c+12(%rip), %xmm1, %xmm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $1, c+12(%rip), %xmm1, %xmm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $4, %al
; AVX2-NOGATHER-NEXT: je .LBB5_6
; AVX2-NOGATHER-NEXT: .LBB5_5: # %cond.load4
-; AVX2-NOGATHER-NEXT: vpinsrd $2, c+12(%rip), %xmm1, %xmm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $2, c+12(%rip), %xmm1, %xmm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $8, %al
; AVX2-NOGATHER-NEXT: je .LBB5_8
; AVX2-NOGATHER-NEXT: .LBB5_7: # %cond.load7
-; AVX2-NOGATHER-NEXT: vpinsrd $3, c+12(%rip), %xmm1, %xmm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $3, c+12(%rip), %xmm1, %xmm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $16, %al
; AVX2-NOGATHER-NEXT: je .LBB5_10
; AVX2-NOGATHER-NEXT: .LBB5_9: # %cond.load10
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4],ymm1[5,6,7]
; AVX2-NOGATHER-NEXT: testb $32, %al
; AVX2-NOGATHER-NEXT: je .LBB5_12
; AVX2-NOGATHER-NEXT: .LBB5_11: # %cond.load13
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
; AVX2-NOGATHER-NEXT: testb $64, %al
; AVX2-NOGATHER-NEXT: je .LBB5_14
; AVX2-NOGATHER-NEXT: .LBB5_13: # %cond.load16
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm2
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+12(%rip), %ymm3
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6],ymm1[7]
; AVX2-NOGATHER-NEXT: testb $-128, %al
; AVX2-NOGATHER-NEXT: jne .LBB5_15
; AVX2-NOGATHER-NEXT: jmp .LBB5_16
; AVX2-NOGATHER-NEXT: .LBB5_17: # %cond.load23
-; AVX2-NOGATHER-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NOGATHER-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX2-NOGATHER-NEXT: testb $2, %al
; AVX2-NOGATHER-NEXT: je .LBB5_20
; AVX2-NOGATHER-NEXT: .LBB5_19: # %cond.load28
-; AVX2-NOGATHER-NEXT: vpinsrd $1, c+28(%rip), %xmm2, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $1, c+28(%rip), %xmm3, %xmm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $4, %al
; AVX2-NOGATHER-NEXT: je .LBB5_22
; AVX2-NOGATHER-NEXT: .LBB5_21: # %cond.load33
-; AVX2-NOGATHER-NEXT: vpinsrd $2, c+28(%rip), %xmm2, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $2, c+28(%rip), %xmm3, %xmm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $8, %al
; AVX2-NOGATHER-NEXT: je .LBB5_24
; AVX2-NOGATHER-NEXT: .LBB5_23: # %cond.load38
-; AVX2-NOGATHER-NEXT: vpinsrd $3, c+28(%rip), %xmm2, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $3, c+28(%rip), %xmm3, %xmm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $16, %al
; AVX2-NOGATHER-NEXT: je .LBB5_26
; AVX2-NOGATHER-NEXT: .LBB5_25: # %cond.load43
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4],ymm2[5,6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6,7]
; AVX2-NOGATHER-NEXT: testb $32, %al
; AVX2-NOGATHER-NEXT: je .LBB5_28
; AVX2-NOGATHER-NEXT: .LBB5_27: # %cond.load48
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5],ymm3[6,7]
; AVX2-NOGATHER-NEXT: testb $64, %al
; AVX2-NOGATHER-NEXT: je .LBB5_30
; AVX2-NOGATHER-NEXT: .LBB5_29: # %cond.load53
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6],ymm2[7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm4
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6],ymm3[7]
; AVX2-NOGATHER-NEXT: testb $-128, %al
; AVX2-NOGATHER-NEXT: jne .LBB5_31
; AVX2-NOGATHER-NEXT: jmp .LBB5_32
@@ -1917,28 +1913,28 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
; AVX2-NOGATHER-NEXT: testb $2, %al
; AVX2-NOGATHER-NEXT: je .LBB5_36
; AVX2-NOGATHER-NEXT: .LBB5_35: # %cond.load69
-; AVX2-NOGATHER-NEXT: vpinsrd $1, c+28(%rip), %xmm0, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $1, c+28(%rip), %xmm0, %xmm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $4, %al
; AVX2-NOGATHER-NEXT: je .LBB5_38
; AVX2-NOGATHER-NEXT: .LBB5_37: # %cond.load74
-; AVX2-NOGATHER-NEXT: vpinsrd $2, c+28(%rip), %xmm0, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $2, c+28(%rip), %xmm0, %xmm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $8, %al
; AVX2-NOGATHER-NEXT: je .LBB5_40
; AVX2-NOGATHER-NEXT: .LBB5_39: # %cond.load79
-; AVX2-NOGATHER-NEXT: vpinsrd $3, c+28(%rip), %xmm0, %xmm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NOGATHER-NEXT: vpinsrd $3, c+28(%rip), %xmm0, %xmm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX2-NOGATHER-NEXT: testb $16, %al
; AVX2-NOGATHER-NEXT: je .LBB5_42
; AVX2-NOGATHER-NEXT: .LBB5_41: # %cond.load84
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4],ymm0[5,6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6,7]
; AVX2-NOGATHER-NEXT: testb $32, %al
; AVX2-NOGATHER-NEXT: je .LBB5_44
; AVX2-NOGATHER-NEXT: .LBB5_43: # %cond.load89
-; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm3
-; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
+; AVX2-NOGATHER-NEXT: vpbroadcastd c+28(%rip), %ymm2
+; AVX2-NOGATHER-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
; AVX2-NOGATHER-NEXT: testb $64, %al
; AVX2-NOGATHER-NEXT: jne .LBB5_45
; AVX2-NOGATHER-NEXT: jmp .LBB5_46
diff --git a/llvm/test/CodeGen/X86/masked_load.ll b/llvm/test/CodeGen/X86/masked_load.ll
index 88b51499518be..f066a198c20ff 100644
--- a/llvm/test/CodeGen/X86/masked_load.ll
+++ b/llvm/test/CodeGen/X86/masked_load.ll
@@ -432,37 +432,35 @@ define <4 x double> @load_v4f64_v4i32(<4 x i32> %trigger, ptr %addr, <4 x double
define <4 x double> @load_v4f64_v4i32_zero(<4 x i32> %trigger, ptr %addr) {
; SSE-LABEL: load_v4f64_v4i32_zero:
; SSE: ## %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: pxor %xmm0, %xmm0
-; SSE-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE-NEXT: movmskps %xmm1, %eax
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE-NEXT: movmskps %xmm0, %eax
; SSE-NEXT: testb $1, %al
-; SSE-NEXT: xorps %xmm1, %xmm1
-; SSE-NEXT: jne LBB6_1
-; SSE-NEXT: ## %bb.2: ## %else
+; SSE-NEXT: je LBB6_1
+; SSE-NEXT: ## %bb.2: ## %cond.load
+; SSE-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; SSE-NEXT: testb $2, %al
-; SSE-NEXT: jne LBB6_3
-; SSE-NEXT: LBB6_4: ## %else2
-; SSE-NEXT: testb $4, %al
-; SSE-NEXT: jne LBB6_5
-; SSE-NEXT: LBB6_6: ## %else5
-; SSE-NEXT: testb $8, %al
-; SSE-NEXT: jne LBB6_7
-; SSE-NEXT: LBB6_8: ## %else8
-; SSE-NEXT: retq
-; SSE-NEXT: LBB6_1: ## %cond.load
-; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT: jne LBB6_4
+; SSE-NEXT: jmp LBB6_5
+; SSE-NEXT: LBB6_1:
+; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: testb $2, %al
-; SSE-NEXT: je LBB6_4
-; SSE-NEXT: LBB6_3: ## %cond.load1
+; SSE-NEXT: je LBB6_5
+; SSE-NEXT: LBB6_4: ## %cond.load1
; SSE-NEXT: movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; SSE-NEXT: LBB6_5: ## %else2
; SSE-NEXT: testb $4, %al
-; SSE-NEXT: je LBB6_6
-; SSE-NEXT: LBB6_5: ## %cond.load4
+; SSE-NEXT: jne LBB6_6
+; SSE-NEXT: ## %bb.7: ## %else5
+; SSE-NEXT: testb $8, %al
+; SSE-NEXT: jne LBB6_8
+; SSE-NEXT: LBB6_9: ## %else8
+; SSE-NEXT: retq
+; SSE-NEXT: LBB6_6: ## %cond.load4
; SSE-NEXT: movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
; SSE-NEXT: testb $8, %al
-; SSE-NEXT: je LBB6_8
-; SSE-NEXT: LBB6_7: ## %cond.load7
+; SSE-NEXT: je LBB6_9
+; SSE-NEXT: LBB6_8: ## %cond.load7
; SSE-NEXT: movhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
; SSE-NEXT: retq
;
@@ -1702,75 +1700,74 @@ define <4 x float> @load_v4f32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x float>
define <8 x float> @load_v8f32_i8(i8 %trigger, ptr %addr) {
; SSE2-LABEL: load_v8f32_i8:
; SSE2: ## %bb.0:
-; SSE2-NEXT: xorps %xmm0, %xmm0
-; SSE2-NEXT: testb $1, %dil
; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: jne LBB16_1
-; SSE2-NEXT: ## %bb.2: ## %else
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: je LBB16_1
+; SSE2-NEXT: ## %bb.2: ## %cond.load
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: testb $2, %dil
+; SSE2-NEXT: jne LBB16_4
+; SSE2-NEXT: jmp LBB16_5
+; SSE2-NEXT: LBB16_1:
+; SSE2-NEXT: xorps %xmm0, %xmm0
; SSE2-NEXT: testb $2, %dil
-; SSE2-NEXT: jne LBB16_3
-; SSE2-NEXT: LBB16_4: ## %else2
+; SSE2-NEXT: je LBB16_5
+; SSE2-NEXT: LBB16_4: ## %cond.load1
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: LBB16_5: ## %else2
; SSE2-NEXT: testb $4, %dil
-; SSE2-NEXT: jne LBB16_5
-; SSE2-NEXT: LBB16_6: ## %else5
+; SSE2-NEXT: jne LBB16_6
+; SSE2-NEXT: ## %bb.7: ## %else5
; SSE2-NEXT: testb $8, %dil
-; SSE2-NEXT: jne LBB16_7
-; SSE2-NEXT: LBB16_8: ## %else8
+; SSE2-NEXT: jne LBB16_8
+; SSE2-NEXT: LBB16_9: ## %else8
; SSE2-NEXT: testb $16, %dil
-; SSE2-NEXT: jne LBB16_9
-; SSE2-NEXT: LBB16_10: ## %else11
+; SSE2-NEXT: jne LBB16_10
+; SSE2-NEXT: LBB16_11: ## %else11
; SSE2-NEXT: testb $32, %dil
-; SSE2-NEXT: jne LBB16_11
-; SSE2-NEXT: LBB16_12: ## %else14
+; SSE2-NEXT: jne LBB16_12
+; SSE2-NEXT: LBB16_13: ## %else14
; SSE2-NEXT: testb $64, %dil
-; SSE2-NEXT: jne LBB16_13
-; SSE2-NEXT: LBB16_14: ## %else17
+; SSE2-NEXT: jne LBB16_14
+; SSE2-NEXT: LBB16_15: ## %else17
; SSE2-NEXT: testb $-128, %dil
-; SSE2-NEXT: jne LBB16_15
-; SSE2-NEXT: LBB16_16: ## %else20
+; SSE2-NEXT: jne LBB16_16
+; SSE2-NEXT: LBB16_17: ## %else20
; SSE2-NEXT: retq
-; SSE2-NEXT: LBB16_1: ## %cond.load
-; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: testb $2, %dil
-; SSE2-NEXT: je LBB16_4
-; SSE2-NEXT: LBB16_3: ## %cond.load1
-; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: testb $4, %dil
-; SSE2-NEXT: je LBB16_6
-; SSE2-NEXT: LBB16_5: ## %cond.load4
+; SSE2-NEXT: LBB16_6: ## %cond.load4
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
; SSE2-NEXT: testb $8, %dil
-; SSE2-NEXT: je LBB16_8
-; SSE2-NEXT: LBB16_7: ## %cond.load7
+; SSE2-NEXT: je LBB16_9
+; SSE2-NEXT: LBB16_8: ## %cond.load7
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSE2-NEXT: testb $16, %dil
-; SSE2-NEXT: je LBB16_10
-; SSE2-NEXT: LBB16_9: ## %cond.load10
+; SSE2-NEXT: je LBB16_11
+; SSE2-NEXT: LBB16_10: ## %cond.load10
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
; SSE2-NEXT: testb $32, %dil
-; SSE2-NEXT: je LBB16_12
-; SSE2-NEXT: LBB16_11: ## %cond.load13
+; SSE2-NEXT: je LBB16_13
+; SSE2-NEXT: LBB16_12: ## %cond.load13
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: testb $64, %dil
-; SSE2-NEXT: je LBB16_14
-; SSE2-NEXT: LBB16_13: ## %cond.load16
+; SSE2-NEXT: je LBB16_15
+; SSE2-NEXT: LBB16_14: ## %cond.load16
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
; SSE2-NEXT: testb $-128, %dil
-; SSE2-NEXT: je LBB16_16
-; SSE2-NEXT: LBB16_15: ## %cond.load19
+; SSE2-NEXT: je LBB16_17
+; SSE2-NEXT: LBB16_16: ## %cond.load19
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
@@ -1778,63 +1775,62 @@ define <8 x float> @load_v8f32_i8(i8 %trigger, ptr %addr) {
;
; SSE42-LABEL: load_v8f32_i8:
; SSE42: ## %bb.0:
-; SSE42-NEXT: xorps %xmm0, %xmm0
-; SSE42-NEXT: testb $1, %dil
; SSE42-NEXT: xorps %xmm1, %xmm1
-; SSE42-NEXT: jne LBB16_1
-; SSE42-NEXT: ## %bb.2: ## %else
+; SSE42-NEXT: testb $1, %dil
+; SSE42-NEXT: je LBB16_1
+; SSE42-NEXT: ## %bb.2: ## %cond.load
+; SSE42-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SSE42-NEXT: testb $2, %dil
-; SSE42-NEXT: jne LBB16_3
-; SSE42-NEXT: LBB16_4: ## %else2
+; SSE42-NEXT: jne LBB16_4
+; SSE42-NEXT: jmp LBB16_5
+; SSE42-NEXT: LBB16_1:
+; SSE42-NEXT: xorps %xmm0, %xmm0
+; SSE42-NEXT: testb $2, %dil
+; SSE42-NEXT: je LBB16_5
+; SSE42-NEXT: LBB16_4: ## %cond.load1
+; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; SSE42-NEXT: LBB16_5: ## %else2
; SSE42-NEXT: testb $4, %dil
-; SSE42-NEXT: jne LBB16_5
-; SSE42-NEXT: LBB16_6: ## %else5
+; SSE42-NEXT: jne LBB16_6
+; SSE42-NEXT: ## %bb.7: ## %else5
; SSE42-NEXT: testb $8, %dil
-; SSE42-NEXT: jne LBB16_7
-; SSE42-NEXT: LBB16_8: ## %else8
+; SSE42-NEXT: jne LBB16_8
+; SSE42-NEXT: LBB16_9: ## %else8
; SSE42-NEXT: testb $16, %dil
-; SSE42-NEXT: jne LBB16_9
-; SSE42-NEXT: LBB16_10: ## %else11
+; SSE42-NEXT: jne LBB16_10
+; SSE42-NEXT: LBB16_11: ## %else11
; SSE42-NEXT: testb $32, %dil
-; SSE42-NEXT: jne LBB16_11
-; SSE42-NEXT: LBB16_12: ## %else14
+; SSE42-NEXT: jne LBB16_12
+; SSE42-NEXT: LBB16_13: ## %else14
; SSE42-NEXT: testb $64, %dil
-; SSE42-NEXT: jne LBB16_13
-; SSE42-NEXT: LBB16_14: ## %else17
+; SSE42-NEXT: jne LBB16_14
+; SSE42-NEXT: LBB16_15: ## %else17
; SSE42-NEXT: testb $-128, %dil
-; SSE42-NEXT: jne LBB16_15
-; SSE42-NEXT: LBB16_16: ## %else20
+; SSE42-NEXT: jne LBB16_16
+; SSE42-NEXT: LBB16_17: ## %else20
; SSE42-NEXT: retq
-; SSE42-NEXT: LBB16_1: ## %cond.load
-; SSE42-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: testb $2, %dil
-; SSE42-NEXT: je LBB16_4
-; SSE42-NEXT: LBB16_3: ## %cond.load1
-; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; SSE42-NEXT: testb $4, %dil
-; SSE42-NEXT: je LBB16_6
-; SSE42-NEXT: LBB16_5: ## %cond.load4
+; SSE42-NEXT: LBB16_6: ## %cond.load4
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
; SSE42-NEXT: testb $8, %dil
-; SSE42-NEXT: je LBB16_8
-; SSE42-NEXT: LBB16_7: ## %cond.load7
+; SSE42-NEXT: je LBB16_9
+; SSE42-NEXT: LBB16_8: ## %cond.load7
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
; SSE42-NEXT: testb $16, %dil
-; SSE42-NEXT: je LBB16_10
-; SSE42-NEXT: LBB16_9: ## %cond.load10
+; SSE42-NEXT: je LBB16_11
+; SSE42-NEXT: LBB16_10: ## %cond.load10
; SSE42-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE42-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
; SSE42-NEXT: testb $32, %dil
-; SSE42-NEXT: je LBB16_12
-; SSE42-NEXT: LBB16_11: ## %cond.load13
+; SSE42-NEXT: je LBB16_13
+; SSE42-NEXT: LBB16_12: ## %cond.load13
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
; SSE42-NEXT: testb $64, %dil
-; SSE42-NEXT: je LBB16_14
-; SSE42-NEXT: LBB16_13: ## %cond.load16
+; SSE42-NEXT: je LBB16_15
+; SSE42-NEXT: LBB16_14: ## %cond.load16
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
; SSE42-NEXT: testb $-128, %dil
-; SSE42-NEXT: je LBB16_16
-; SSE42-NEXT: LBB16_15: ## %cond.load19
+; SSE42-NEXT: je LBB16_17
+; SSE42-NEXT: LBB16_16: ## %cond.load19
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
; SSE42-NEXT: retq
;
@@ -1896,75 +1892,74 @@ define <8 x float> @load_v8f32_v8i1_zero(<8 x i1> %mask, ptr %addr) {
; SSE2-NEXT: psllw $15, %xmm0
; SSE2-NEXT: packsswb %xmm0, %xmm0
; SSE2-NEXT: pmovmskb %xmm0, %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: testb $1, %al
; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: jne LBB17_1
-; SSE2-NEXT: ## %bb.2: ## %else
+; SSE2-NEXT: testb $1, %al
+; SSE2-NEXT: je LBB17_1
+; SSE2-NEXT: ## %bb.2: ## %cond.load
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: jne LBB17_3
-; SSE2-NEXT: LBB17_4: ## %else2
+; SSE2-NEXT: jne LBB17_4
+; SSE2-NEXT: jmp LBB17_5
+; SSE2-NEXT: LBB17_1:
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: testb $2, %al
+; SSE2-NEXT: je LBB17_5
+; SSE2-NEXT: LBB17_4: ## %cond.load1
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: LBB17_5: ## %else2
; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: jne LBB17_5
-; SSE2-NEXT: LBB17_6: ## %else5
+; SSE2-NEXT: jne LBB17_6
+; SSE2-NEXT: ## %bb.7: ## %else5
; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: jne LBB17_7
-; SSE2-NEXT: LBB17_8: ## %else8
+; SSE2-NEXT: jne LBB17_8
+; SSE2-NEXT: LBB17_9: ## %else8
; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: jne LBB17_9
-; SSE2-NEXT: LBB17_10: ## %else11
+; SSE2-NEXT: jne LBB17_10
+; SSE2-NEXT: LBB17_11: ## %else11
; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: jne LBB17_11
-; SSE2-NEXT: LBB17_12: ## %else14
+; SSE2-NEXT: jne LBB17_12
+; SSE2-NEXT: LBB17_13: ## %else14
; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: jne LBB17_13
-; SSE2-NEXT: LBB17_14: ## %else17
+; SSE2-NEXT: jne LBB17_14
+; SSE2-NEXT: LBB17_15: ## %else17
; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: jne LBB17_15
-; SSE2-NEXT: LBB17_16: ## %else20
+; SSE2-NEXT: jne LBB17_16
+; SSE2-NEXT: LBB17_17: ## %else20
; SSE2-NEXT: retq
-; SSE2-NEXT: LBB17_1: ## %cond.load
-; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: je LBB17_4
-; SSE2-NEXT: LBB17_3: ## %cond.load1
-; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: je LBB17_6
-; SSE2-NEXT: LBB17_5: ## %cond.load4
+; SSE2-NEXT: LBB17_6: ## %cond.load4
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: je LBB17_8
-; SSE2-NEXT: LBB17_7: ## %cond.load7
+; SSE2-NEXT: je LBB17_9
+; SSE2-NEXT: LBB17_8: ## %cond.load7
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: je LBB17_10
-; SSE2-NEXT: LBB17_9: ## %cond.load10
+; SSE2-NEXT: je LBB17_11
+; SSE2-NEXT: LBB17_10: ## %cond.load10
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: je LBB17_12
-; SSE2-NEXT: LBB17_11: ## %cond.load13
+; SSE2-NEXT: je LBB17_13
+; SSE2-NEXT: LBB17_12: ## %cond.load13
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: je LBB17_14
-; SSE2-NEXT: LBB17_13: ## %cond.load16
+; SSE2-NEXT: je LBB17_15
+; SSE2-NEXT: LBB17_14: ## %cond.load16
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: je LBB17_16
-; SSE2-NEXT: LBB17_15: ## %cond.load19
+; SSE2-NEXT: je LBB17_17
+; SSE2-NEXT: LBB17_16: ## %cond.load19
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
@@ -1975,63 +1970,62 @@ define <8 x float> @load_v8f32_v8i1_zero(<8 x i1> %mask, ptr %addr) {
; SSE42-NEXT: psllw $15, %xmm0
; SSE42-NEXT: packsswb %xmm0, %xmm0
; SSE42-NEXT: pmovmskb %xmm0, %eax
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: testb $1, %al
; SSE42-NEXT: xorps %xmm1, %xmm1
-; SSE42-NEXT: jne LBB17_1
-; SSE42-NEXT: ## %bb.2: ## %else
+; SSE42-NEXT: testb $1, %al
+; SSE42-NEXT: je LBB17_1
+; SSE42-NEXT: ## %bb.2: ## %cond.load
+; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SSE42-NEXT: testb $2, %al
-; SSE42-NEXT: jne LBB17_3
-; SSE42-NEXT: LBB17_4: ## %else2
+; SSE42-NEXT: jne LBB17_4
+; SSE42-NEXT: jmp LBB17_5
+; SSE42-NEXT: LBB17_1:
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: testb $2, %al
+; SSE42-NEXT: je LBB17_5
+; SSE42-NEXT: LBB17_4: ## %cond.load1
+; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; SSE42-NEXT: LBB17_5: ## %else2
; SSE42-NEXT: testb $4, %al
-; SSE42-NEXT: jne LBB17_5
-; SSE42-NEXT: LBB17_6: ## %else5
+; SSE42-NEXT: jne LBB17_6
+; SSE42-NEXT: ## %bb.7: ## %else5
; SSE42-NEXT: testb $8, %al
-; SSE42-NEXT: jne LBB17_7
-; SSE42-NEXT: LBB17_8: ## %else8
+; SSE42-NEXT: jne LBB17_8
+; SSE42-NEXT: LBB17_9: ## %else8
; SSE42-NEXT: testb $16, %al
-; SSE42-NEXT: jne LBB17_9
-; SSE42-NEXT: LBB17_10: ## %else11
+; SSE42-NEXT: jne LBB17_10
+; SSE42-NEXT: LBB17_11: ## %else11
; SSE42-NEXT: testb $32, %al
-; SSE42-NEXT: jne LBB17_11
-; SSE42-NEXT: LBB17_12: ## %else14
+; SSE42-NEXT: jne LBB17_12
+; SSE42-NEXT: LBB17_13: ## %else14
; SSE42-NEXT: testb $64, %al
-; SSE42-NEXT: jne LBB17_13
-; SSE42-NEXT: LBB17_14: ## %else17
+; SSE42-NEXT: jne LBB17_14
+; SSE42-NEXT: LBB17_15: ## %else17
; SSE42-NEXT: testb $-128, %al
-; SSE42-NEXT: jne LBB17_15
-; SSE42-NEXT: LBB17_16: ## %else20
+; SSE42-NEXT: jne LBB17_16
+; SSE42-NEXT: LBB17_17: ## %else20
; SSE42-NEXT: retq
-; SSE42-NEXT: LBB17_1: ## %cond.load
-; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: testb $2, %al
-; SSE42-NEXT: je LBB17_4
-; SSE42-NEXT: LBB17_3: ## %cond.load1
-; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; SSE42-NEXT: testb $4, %al
-; SSE42-NEXT: je LBB17_6
-; SSE42-NEXT: LBB17_5: ## %cond.load4
+; SSE42-NEXT: LBB17_6: ## %cond.load4
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
; SSE42-NEXT: testb $8, %al
-; SSE42-NEXT: je LBB17_8
-; SSE42-NEXT: LBB17_7: ## %cond.load7
+; SSE42-NEXT: je LBB17_9
+; SSE42-NEXT: LBB17_8: ## %cond.load7
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
; SSE42-NEXT: testb $16, %al
-; SSE42-NEXT: je LBB17_10
-; SSE42-NEXT: LBB17_9: ## %cond.load10
+; SSE42-NEXT: je LBB17_11
+; SSE42-NEXT: LBB17_10: ## %cond.load10
; SSE42-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE42-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
; SSE42-NEXT: testb $32, %al
-; SSE42-NEXT: je LBB17_12
-; SSE42-NEXT: LBB17_11: ## %cond.load13
+; SSE42-NEXT: je LBB17_13
+; SSE42-NEXT: LBB17_12: ## %cond.load13
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
; SSE42-NEXT: testb $64, %al
-; SSE42-NEXT: je LBB17_14
-; SSE42-NEXT: LBB17_13: ## %cond.load16
+; SSE42-NEXT: je LBB17_15
+; SSE42-NEXT: LBB17_14: ## %cond.load16
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
; SSE42-NEXT: testb $-128, %al
-; SSE42-NEXT: je LBB17_16
-; SSE42-NEXT: LBB17_15: ## %cond.load19
+; SSE42-NEXT: je LBB17_17
+; SSE42-NEXT: LBB17_16: ## %cond.load19
; SSE42-NEXT: insertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
; SSE42-NEXT: retq
;
@@ -3413,75 +3407,74 @@ define <8 x i32> @load_v8i32_v8i1_zero(<8 x i1> %mask, ptr %addr) {
; SSE2-NEXT: psllw $15, %xmm0
; SSE2-NEXT: packsswb %xmm0, %xmm0
; SSE2-NEXT: pmovmskb %xmm0, %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: testb $1, %al
; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: jne LBB27_1
-; SSE2-NEXT: ## %bb.2: ## %else
+; SSE2-NEXT: testb $1, %al
+; SSE2-NEXT: je LBB27_1
+; SSE2-NEXT: ## %bb.2: ## %cond.load
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: jne LBB27_3
-; SSE2-NEXT: LBB27_4: ## %else2
+; SSE2-NEXT: jne LBB27_4
+; SSE2-NEXT: jmp LBB27_5
+; SSE2-NEXT: LBB27_1:
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: testb $2, %al
+; SSE2-NEXT: je LBB27_5
+; SSE2-NEXT: LBB27_4: ## %cond.load1
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: LBB27_5: ## %else2
; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: jne LBB27_5
-; SSE2-NEXT: LBB27_6: ## %else5
+; SSE2-NEXT: jne LBB27_6
+; SSE2-NEXT: ## %bb.7: ## %else5
; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: jne LBB27_7
-; SSE2-NEXT: LBB27_8: ## %else8
+; SSE2-NEXT: jne LBB27_8
+; SSE2-NEXT: LBB27_9: ## %else8
; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: jne LBB27_9
-; SSE2-NEXT: LBB27_10: ## %else11
+; SSE2-NEXT: jne LBB27_10
+; SSE2-NEXT: LBB27_11: ## %else11
; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: jne LBB27_11
-; SSE2-NEXT: LBB27_12: ## %else14
+; SSE2-NEXT: jne LBB27_12
+; SSE2-NEXT: LBB27_13: ## %else14
; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: jne LBB27_13
-; SSE2-NEXT: LBB27_14: ## %else17
+; SSE2-NEXT: jne LBB27_14
+; SSE2-NEXT: LBB27_15: ## %else17
; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: jne LBB27_15
-; SSE2-NEXT: LBB27_16: ## %else20
+; SSE2-NEXT: jne LBB27_16
+; SSE2-NEXT: LBB27_17: ## %else20
; SSE2-NEXT: retq
-; SSE2-NEXT: LBB27_1: ## %cond.load
-; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: je LBB27_4
-; SSE2-NEXT: LBB27_3: ## %cond.load1
-; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: je LBB27_6
-; SSE2-NEXT: LBB27_5: ## %cond.load4
+; SSE2-NEXT: LBB27_6: ## %cond.load4
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]
; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: je LBB27_8
-; SSE2-NEXT: LBB27_7: ## %cond.load7
+; SSE2-NEXT: je LBB27_9
+; SSE2-NEXT: LBB27_8: ## %cond.load7
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: je LBB27_10
-; SSE2-NEXT: LBB27_9: ## %cond.load10
+; SSE2-NEXT: je LBB27_11
+; SSE2-NEXT: LBB27_10: ## %cond.load10
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: je LBB27_12
-; SSE2-NEXT: LBB27_11: ## %cond.load13
+; SSE2-NEXT: je LBB27_13
+; SSE2-NEXT: LBB27_12: ## %cond.load13
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[2,3]
; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: je LBB27_14
-; SSE2-NEXT: LBB27_13: ## %cond.load16
+; SSE2-NEXT: je LBB27_15
+; SSE2-NEXT: LBB27_14: ## %cond.load16
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: je LBB27_16
-; SSE2-NEXT: LBB27_15: ## %cond.load19
+; SSE2-NEXT: je LBB27_17
+; SSE2-NEXT: LBB27_16: ## %cond.load19
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
@@ -3492,62 +3485,61 @@ define <8 x i32> @load_v8i32_v8i1_zero(<8 x i1> %mask, ptr %addr) {
; SSE42-NEXT: psllw $15, %xmm0
; SSE42-NEXT: packsswb %xmm0, %xmm0
; SSE42-NEXT: pmovmskb %xmm0, %eax
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: testb $1, %al
; SSE42-NEXT: pxor %xmm1, %xmm1
-; SSE42-NEXT: jne LBB27_1
-; SSE42-NEXT: ## %bb.2: ## %else
+; SSE42-NEXT: testb $1, %al
+; SSE42-NEXT: je LBB27_1
+; SSE42-NEXT: ## %bb.2: ## %cond.load
+; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE42-NEXT: testb $2, %al
+; SSE42-NEXT: jne LBB27_4
+; SSE42-NEXT: jmp LBB27_5
+; SSE42-NEXT: LBB27_1:
+; SSE42-NEXT: pxor %xmm0, %xmm0
; SSE42-NEXT: testb $2, %al
-; SSE42-NEXT: jne LBB27_3
-; SSE42-NEXT: LBB27_4: ## %else2
+; SSE42-NEXT: je LBB27_5
+; SSE42-NEXT: LBB27_4: ## %cond.load1
+; SSE42-NEXT: pinsrd $1, 4(%rdi), %xmm0
+; SSE42-NEXT: LBB27_5: ## %else2
; SSE42-NEXT: testb $4, %al
-; SSE42-NEXT: jne LBB27_5
-; SSE42-NEXT: LBB27_6: ## %else5
+; SSE42-NEXT: jne LBB27_6
+; SSE42-NEXT: ## %bb.7: ## %else5
; SSE42-NEXT: testb $8, %al
-; SSE42-NEXT: jne LBB27_7
-; SSE42-NEXT: LBB27_8: ## %else8
+; SSE42-NEXT: jne LBB27_8
+; SSE42-NEXT: LBB27_9: ## %else8
; SSE42-NEXT: testb $16, %al
-; SSE42-NEXT: jne LBB27_9
-; SSE42-NEXT: LBB27_10: ## %else11
+; SSE42-NEXT: jne LBB27_10
+; SSE42-NEXT: LBB27_11: ## %else11
; SSE42-NEXT: testb $32, %al
-; SSE42-NEXT: jne LBB27_11
-; SSE42-NEXT: LBB27_12: ## %else14
+; SSE42-NEXT: jne LBB27_12
+; SSE42-NEXT: LBB27_13: ## %else14
; SSE42-NEXT: testb $64, %al
-; SSE42-NEXT: jne LBB27_13
-; SSE42-NEXT: LBB27_14: ## %else17
+; SSE42-NEXT: jne LBB27_14
+; SSE42-NEXT: LBB27_15: ## %else17
; SSE42-NEXT: testb $-128, %al
-; SSE42-NEXT: jne LBB27_15
-; SSE42-NEXT: LBB27_16: ## %else20
+; SSE42-NEXT: jne LBB27_16
+; SSE42-NEXT: LBB27_17: ## %else20
; SSE42-NEXT: retq
-; SSE42-NEXT: LBB27_1: ## %cond.load
-; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: testb $2, %al
-; SSE42-NEXT: je LBB27_4
-; SSE42-NEXT: LBB27_3: ## %cond.load1
-; SSE42-NEXT: pinsrd $1, 4(%rdi), %xmm0
-; SSE42-NEXT: testb $4, %al
-; SSE42-NEXT: je LBB27_6
-; SSE42-NEXT: LBB27_5: ## %cond.load4
+; SSE42-NEXT: LBB27_6: ## %cond.load4
; SSE42-NEXT: pinsrd $2, 8(%rdi), %xmm0
; SSE42-NEXT: testb $8, %al
-; SSE42-NEXT: je LBB27_8
-; SSE42-NEXT: LBB27_7: ## %cond.load7
+; SSE42-NEXT: je LBB27_9
+; SSE42-NEXT: LBB27_8: ## %cond.load7
; SSE42-NEXT: pinsrd $3, 12(%rdi), %xmm0
; SSE42-NEXT: testb $16, %al
-; SSE42-NEXT: je LBB27_10
-; SSE42-NEXT: LBB27_9: ## %cond.load10
+; SSE42-NEXT: je LBB27_11
+; SSE42-NEXT: LBB27_10: ## %cond.load10
; SSE42-NEXT: pinsrd $0, 16(%rdi), %xmm1
; SSE42-NEXT: testb $32, %al
-; SSE42-NEXT: je LBB27_12
-; SSE42-NEXT: LBB27_11: ## %cond.load13
+; SSE42-NEXT: je LBB27_13
+; SSE42-NEXT: LBB27_12: ## %cond.load13
; SSE42-NEXT: pinsrd $1, 20(%rdi), %xmm1
; SSE42-NEXT: testb $64, %al
-; SSE42-NEXT: je LBB27_14
-; SSE42-NEXT: LBB27_13: ## %cond.load16
+; SSE42-NEXT: je LBB27_15
+; SSE42-NEXT: LBB27_14: ## %cond.load16
; SSE42-NEXT: pinsrd $2, 24(%rdi), %xmm1
; SSE42-NEXT: testb $-128, %al
-; SSE42-NEXT: je LBB27_16
-; SSE42-NEXT: LBB27_15: ## %cond.load19
+; SSE42-NEXT: je LBB27_17
+; SSE42-NEXT: LBB27_16: ## %cond.load19
; SSE42-NEXT: pinsrd $3, 28(%rdi), %xmm1
; SSE42-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index 24390f2d852d3..fdfe11849623d 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -123,6 +123,8 @@
; CHECK-NEXT: X86 Fixup SetCC
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: X86 LEA Optimize
+; CHECK-NEXT: MachineDominator Tree Construction
+; CHECK-NEXT: X86 Optimize Vector Constants
; CHECK-NEXT: X86 Optimize Call Frame
; CHECK-NEXT: X86 Avoid Store Forwarding Block
; CHECK-NEXT: X86 Suppress APX features for relocation
diff --git a/llvm/test/CodeGen/X86/pr120093.ll b/llvm/test/CodeGen/X86/pr120093.ll
index 99bf2218c5538..4e405397559b6 100644
--- a/llvm/test/CodeGen/X86/pr120093.ll
+++ b/llvm/test/CodeGen/X86/pr120093.ll
@@ -5,10 +5,10 @@ define double @PR120093() {
; CHECK-LABEL: PR120093:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorpd %xmm0, %xmm0
+; CHECK-NEXT: movhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
; CHECK-NEXT: xorpd %xmm1, %xmm1
-; CHECK-NEXT: movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; CHECK-NEXT: cmpnltpd %xmm1, %xmm0
-; CHECK-NEXT: movmskpd %xmm0, %eax
+; CHECK-NEXT: cmpnltpd %xmm0, %xmm1
+; CHECK-NEXT: movmskpd %xmm1, %eax
; CHECK-NEXT: cmpl $3, %eax
; CHECK-NEXT: jne .LBB0_2
; CHECK-NEXT: # %bb.1: # %bb2
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..b5f98dbcbd70f 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -745,9 +745,8 @@ define <2 x i64> @trunc_v4i64_to_v4i16_return_v2i64(<4 x i64> %vec) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -948,9 +947,8 @@ define <8 x i16> @trunc_v4i64_to_v4i16_return_v8i16(<4 x i64> %vec) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vec_ss_load_fold.ll b/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
index e73d345d0fcd4..6b5f4da5ad087 100644
--- a/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
+++ b/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
@@ -32,57 +32,30 @@ define i16 @test1(float %f) nounwind {
; X64-NEXT: ## kill: def $ax killed $ax killed $eax
; X64-NEXT: retq
;
-; X86_AVX1-LABEL: test1:
-; X86_AVX1: ## %bb.0:
-; X86_AVX1-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86_AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86_AVX1-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X86_AVX1-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm0
-; X86_AVX1-NEXT: vcvttss2si %xmm0, %eax
-; X86_AVX1-NEXT: ## kill: def $ax killed $ax killed $eax
-; X86_AVX1-NEXT: retl
-;
-; X64_AVX1-LABEL: test1:
-; X64_AVX1: ## %bb.0:
-; X64_AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64_AVX1-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X64_AVX1-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm0
-; X64_AVX1-NEXT: vcvttss2si %xmm0, %eax
-; X64_AVX1-NEXT: ## kill: def $ax killed $ax killed $eax
-; X64_AVX1-NEXT: retq
-;
-; X86_AVX512-LABEL: test1:
-; X86_AVX512: ## %bb.0:
-; X86_AVX512-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86_AVX512-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86_AVX512-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X86_AVX512-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86_AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86_AVX512-NEXT: vmaxss %xmm1, %xmm0, %xmm0
-; X86_AVX512-NEXT: vcvttss2si %xmm0, %eax
-; X86_AVX512-NEXT: ## kill: def $ax killed $ax killed $eax
-; X86_AVX512-NEXT: retl
+; X86_AVX-LABEL: test1:
+; X86_AVX: ## %bb.0:
+; X86_AVX-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86_AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86_AVX-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86_AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X86_AVX-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86_AVX-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86_AVX-NEXT: vmaxss %xmm1, %xmm0, %xmm0
+; X86_AVX-NEXT: vcvttss2si %xmm0, %eax
+; X86_AVX-NEXT: ## kill: def $ax killed $ax killed $eax
+; X86_AVX-NEXT: retl
;
-; X64_AVX512-LABEL: test1:
-; X64_AVX512: ## %bb.0:
-; X64_AVX512-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64_AVX512-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; X64_AVX512-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64_AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64_AVX512-NEXT: vmaxss %xmm1, %xmm0, %xmm0
-; X64_AVX512-NEXT: vcvttss2si %xmm0, %eax
-; X64_AVX512-NEXT: ## kill: def $ax killed $ax killed $eax
-; X64_AVX512-NEXT: retq
+; X64_AVX-LABEL: test1:
+; X64_AVX: ## %bb.0:
+; X64_AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64_AVX-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64_AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X64_AVX-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64_AVX-NEXT: vminss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64_AVX-NEXT: vmaxss %xmm1, %xmm0, %xmm0
+; X64_AVX-NEXT: vcvttss2si %xmm0, %eax
+; X64_AVX-NEXT: ## kill: def $ax killed $ax killed $eax
+; X64_AVX-NEXT: retq
%tmp = insertelement <4 x float> undef, float %f, i32 0 ; <<4 x float>> [#uses=1]
%tmp10 = insertelement <4 x float> %tmp, float 0.000000e+00, i32 1 ; <<4 x float>> [#uses=1]
%tmp11 = insertelement <4 x float> %tmp10, float 0.000000e+00, i32 2 ; <<4 x float>> [#uses=1]
@@ -413,3 +386,8 @@ entry:
%3 = fadd <4 x float> %1, %2
ret <4 x float> %3
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; X64_AVX1: {{.*}}
+; X64_AVX512: {{.*}}
+; X86_AVX1: {{.*}}
+; X86_AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-constants-coalescing.mir b/llvm/test/CodeGen/X86/vector-constants-coalescing.mir
new file mode 100644
index 0000000000000..f6b786b1efcd6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-constants-coalescing.mir
@@ -0,0 +1,68 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f -run-pass=x86-optimize-vector-constants -o - %s | FileCheck %s
+
+---
+name: test_constants
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vr512 = AVX512_512_SET0
+ %1:vr512 = AVX512_512_SETALLONES
+
+ ; CHECK: %0:vr512 = AVX512_512_SET0
+ ; CHECK: %1:vr512 = AVX512_512_SETALLONES
+
+ ; Zeros coalescing
+ ; Same size and class:
+ %2:vr512 = AVX512_512_SET0
+ ; CHECK-NOT: %2:vr512 = AVX512_512_SET0
+
+ ; Smaller size:
+ %3:vr256 = AVX_SET0
+ ; CHECK: %3:vr256 = COPY %0.sub_ymm
+
+ ; Even smaller size:
+ %4:vr128 = V_SET0
+ ; CHECK: %4:vr128 = COPY %0.sub_xmm
+
+ ; Cross-class zeros (scalar XMM-backed vs vector):
+ %5:fr64 = FsFLD0SD
+ ; CHECK: %5:fr64 = COPY %0.sub_xmm
+
+ ; All-ones coalescing
+ ; Smaller size:
+ %6:vr256 = AVX2_SETALLONES
+ ; CHECK: %6:vr256 = COPY %1.sub_ymm
+
+ ; Even smaller size:
+ %7:vr128x = AVX512_128_SETALLONES
+ ; CHECK: %7:vr128x = COPY %1.sub_xmm
+
+ ; Ensure we don't cross-pollinate zeros and all-ones:
+ %8:vr128 = V_SET0
+ ; CHECK: %8:vr128 = COPY %0.sub_xmm
+...
+
+---
+name: test_nondominating
+tracksRegLiveness: true
+body: |
+ bb.0:
+ JCC_1 %bb.2, 4, implicit undef $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ ; CHECK-LABEL: bb.1:
+ ; CHECK: %0:vr128 = V_SET0
+ %0:vr128 = V_SET0
+ JMP_1 %bb.3
+
+ bb.2:
+ ; CHECK-LABEL: bb.2:
+ ; CHECK: %1:vr128 = V_SET0
+ ; Ensure non-dominating sibling block materialization is NOT replaced with %0
+ %1:vr128 = V_SET0
+ JMP_1 %bb.3
+
+ bb.3:
+ RET 0
+...
diff --git a/llvm/test/CodeGen/X86/vector-pcmp.ll b/llvm/test/CodeGen/X86/vector-pcmp.ll
index 69785909c434f..d986bdaed155f 100644
--- a/llvm/test/CodeGen/X86/vector-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vector-pcmp.ll
@@ -1633,8 +1633,7 @@ define <4 x i1> @is_positive_mask_v4i64_v4i1(<4 x i64> %x, <4 x i1> %y) {
; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX512F-NEXT: vpcmpgtq %ymm2, %ymm0, %k1
; AVX512F-NEXT: vptestmd %xmm1, %xmm1, %k1 {%k1}
-; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1} {z}
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
index 33901a0a71f27..d22b3f5494f2f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
@@ -2411,11 +2411,10 @@ define i8 @test_v32i8_v32i1(<32 x i8> %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $al killed $al killed $eax
@@ -2426,11 +2425,10 @@ define i8 @test_v32i8_v32i1(<32 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
@@ -2521,8 +2519,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $al killed $al killed $eax
; AVX2-NEXT: vzeroupper
@@ -2539,8 +2536,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: # kill: def $al killed $al killed $eax
; AVX512F-NEXT: vzeroupper
@@ -2557,8 +2553,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: # kill: def $al killed $al killed $eax
; AVX512VL-NEXT: vzeroupper
@@ -2759,8 +2754,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX2-NEXT: movl %ebp, %esp
@@ -2782,8 +2776,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX2-NEXT: vzeroupper
@@ -2805,8 +2798,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: # kill: def $al killed $al killed $eax
; AVX512F-NEXT: vzeroupper
@@ -2828,8 +2820,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: # kill: def $al killed $al killed $eax
; AVX512VL-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
index be606a3fa62da..c7c52bee1105e 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
@@ -940,7 +940,6 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovqw %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -1212,7 +1211,6 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -1636,7 +1634,6 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -2114,7 +2111,6 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
@@ -2864,8 +2860,7 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1
; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
index 5edea6d0fd68a..e73b4b289fcbd 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
@@ -3979,11 +3979,10 @@ define void @vec384_v24i16_to_v8i48_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm2, %ymm1
; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
-; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
+; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-SLOW-NEXT: vzeroupper
@@ -4293,11 +4292,10 @@ define void @vec384_v24i16_to_v4i96_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm2, %ymm1
; AVX512BW-SLOW-NEXT: vpsrld $16, %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]
-; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
+; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
+; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-SLOW-NEXT: vzeroupper
@@ -4474,11 +4472,10 @@ define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm2, %ymm1
; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
+; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
+; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-SLOW-NEXT: vzeroupper
@@ -4916,12 +4913,11 @@ define void @vec384_v12i32_to_v4i96_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
; AVX2-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rcx)
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX2-SLOW-NEXT: vzeroupper
@@ -5078,8 +5074,7 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
; AVX2-SLOW-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index a28c23bb8a61e..8393345203856 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -3775,9 +3775,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3795,9 +3794,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3834,9 +3832,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3872,7 +3869,6 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4093,7 +4089,6 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4200,9 +4195,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4220,9 +4214,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4259,9 +4252,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4297,7 +4289,6 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4446,7 +4437,6 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
; AVX512BW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4676,7 +4666,6 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4,5,6,7]
; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -5066,7 +5055,6 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4,5,6,7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
index 21a3df0456de1..44375ee33b84a 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -2999,12 +2999,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3047,12 +3046,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3063,12 +3061,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3331,12 +3328,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3379,12 +3375,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]
+; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3395,12 +3390,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm0
; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3717,7 +3711,6 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5,6,7]
; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6],ymm2[7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
More information about the llvm-commits
mailing list