[llvm] [AVX-512] make vpternlogq more aggressive for longer chains of bitmanipulations (PR #189971)
Julian Pokrovsky via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 2 05:16:10 PDT 2026
https://github.com/raventid updated https://github.com/llvm/llvm-project/pull/189971
>From 4ec75e67835f852719a4a33064257097faec15ce Mon Sep 17 00:00:00 2001
From: raventid <juliankul at gmail.com>
Date: Tue, 24 Mar 2026 14:37:20 +0800
Subject: [PATCH 1/3] [vpternlog] optimize more aggressevely
---
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 283 ++++++++++++++++--------
llvm/test/CodeGen/X86/vpternlog.ll | 78 ++++++-
2 files changed, 266 insertions(+), 95 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index be95168f2de00..1dc6c10aba4a6 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -32,6 +32,8 @@
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include <cstdint>
+#include <functional>
+#include <optional>
using namespace llvm;
@@ -4815,8 +4817,7 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
return true;
}
-// Try to match two logic ops to a VPTERNLOG.
-// FIXME: Handle more complex patterns that use an operand more than once?
+// Try to match logic trees to one or more VPTERNLOG operations.
bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
MVT NVT = N->getSimpleValueType(0);
@@ -4829,118 +4830,214 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
return false;
- auto getFoldableLogicOp = [](SDValue Op) {
- // Peek through single use bitcast.
- if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
- Op = Op.getOperand(0);
+ auto IsLogicOpcode = [](unsigned Opc) {
+ return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
+ Opc == X86ISD::ANDNP;
+ };
+
+ auto IsAllOnesXor = [](SDValue V) {
+ return V.getOpcode() == ISD::XOR &&
+ ISD::isBuildVectorAllOnes(V.getOperand(1).getNode());
+ };
+
+ auto PeelSingleUseBitcast = [](SDValue V) {
+ if (V.getOpcode() == ISD::BITCAST && V.hasOneUse())
+ return V.getOperand(0);
+ return V;
+ };
+
+ // Avoid consuming OR into a stand-alone VPTERNLOG if it is part of a
+ // higher-level A & ~(B | C) shape. Let the parent AND/ANDNP matcher absorb
+ // the whole pattern instead.
+ if (N->getOpcode() == ISD::OR && N->hasOneUse()) {
+ SDNode *User = *N->user_begin();
+ while (User->getOpcode() == ISD::BITCAST && User->hasOneUse())
+ User = *User->user_begin();
+
+ if (User->getOpcode() == ISD::XOR && User->hasOneUse() &&
+ (ISD::isBuildVectorAllOnes(User->getOperand(0).getNode()) ||
+ ISD::isBuildVectorAllOnes(User->getOperand(1).getNode()))) {
+ SDNode *NextUser = *User->user_begin();
+ while (NextUser->getOpcode() == ISD::BITCAST && NextUser->hasOneUse())
+ NextUser = *NextUser->user_begin();
+ unsigned NextOpc = NextUser->getOpcode();
+ if (NextOpc == ISD::AND || NextOpc == X86ISD::ANDNP)
+ return false;
+ }
+ }
- if (!Op.hasOneUse())
- return SDValue();
+ // Fast-path: A & ~(B | C) -> vpternlog(A, B, C, 0x10)
+ if (N->getOpcode() == ISD::AND) {
+ for (unsigned Idx = 0; Idx != 2; ++Idx) {
+ SDValue NotSide = N->getOperand(Idx);
+ SDValue A = N->getOperand(Idx ^ 1);
- unsigned Opc = Op.getOpcode();
- if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
- Opc == X86ISD::ANDNP)
- return Op;
+ SDValue NotSideNoCast = PeelSingleUseBitcast(NotSide);
+ if (!NotSideNoCast.hasOneUse() || !IsAllOnesXor(NotSideNoCast))
+ continue;
- return SDValue();
+ SDValue Inner = PeelSingleUseBitcast(NotSideNoCast.getOperand(0));
+ if (!Inner.hasOneUse() || Inner.getOpcode() != ISD::OR)
+ continue;
+
+ SDValue B = Inner.getOperand(0);
+ SDValue C = Inner.getOperand(1);
+ if (matchVPTERNLOG(N, N, Inner.getNode(), Inner.getNode(), A, B, C, 0x10))
+ return true;
+ }
+ }
+
+ struct LeafInfo {
+ SDValue Leaf;
+ SDNode *Parent;
+ uint8_t Magic;
};
- SDValue N0, N1, A, FoldableOp;
+ auto ComputeTernlog = [&](SDValue Root, SDNode *OpaqueSubtree,
+ SmallVectorImpl<LeafInfo> &Leaves,
+ uint8_t &ImmOut, bool &TooManyLeaves) {
+ TooManyLeaves = false;
- // Identify and (optionally) peel an outer NOT that wraps a pure logic tree
- auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
- if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
- ISD::isBuildVectorAllOnes(Op->getOperand(1).getNode())) {
- SDValue InnerOp = getFoldableLogicOp(Op->getOperand(0));
+ auto lookupLeaf = [&](SDValue Leaf) -> std::optional<uint8_t> {
+ for (const LeafInfo &L : Leaves)
+ if (L.Leaf == Leaf)
+ return L.Magic;
+ return std::nullopt;
+ };
- if (!InnerOp)
- return SDValue();
+ std::function<int(SDValue, SDNode *, bool)> ComputeRec =
+ [&](SDValue Op, SDNode *Parent, bool IsRoot) -> int {
+ if (Op.getNode() != OpaqueSubtree) {
+ // Peek through single-use bitcasts.
+ if (Op.getOpcode() == ISD::BITCAST && (IsRoot || Op.hasOneUse())) {
+ Parent = Op.getNode();
+ Op = Op.getOperand(0);
+ }
+
+ if ((IsRoot || Op.hasOneUse()) && IsAllOnesXor(Op)) {
+ int Inner = ComputeRec(Op.getOperand(0), Op.getNode(), false);
+ return Inner < 0 ? -1 : ((~Inner) & 0xFF);
+ }
- N0 = InnerOp.getOperand(0);
- N1 = InnerOp.getOperand(1);
- if ((FoldableOp = getFoldableLogicOp(N1))) {
- A = N0;
- return InnerOp;
+ if ((IsRoot || Op.hasOneUse()) && IsLogicOpcode(Op.getOpcode())) {
+ int L = ComputeRec(Op.getOperand(0), Op.getNode(), false);
+ int R = ComputeRec(Op.getOperand(1), Op.getNode(), false);
+ if (L < 0 || R < 0)
+ return -1;
+
+ switch (Op.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode");
+ case ISD::AND:
+ return (L & R) & 0xFF;
+ case ISD::OR:
+ return (L | R) & 0xFF;
+ case ISD::XOR:
+ return (L ^ R) & 0xFF;
+ case X86ISD::ANDNP:
+ return ((~L) & R) & 0xFF;
+ }
+ }
}
- if ((FoldableOp = getFoldableLogicOp(N0))) {
- A = N1;
- return InnerOp;
+
+ if (auto Existing = lookupLeaf(Op))
+ return *Existing;
+
+ if (Leaves.size() >= 3) {
+ TooManyLeaves = true;
+ return -1;
}
- }
- return SDValue();
- };
- bool PeeledOuterNot = false;
- SDNode *OriN = N;
- if (SDValue InnerOp = tryPeelOuterNotWrappingLogic(N)) {
- PeeledOuterNot = true;
- N = InnerOp.getNode();
- } else {
- N0 = N->getOperand(0);
- N1 = N->getOperand(1);
+ static constexpr uint8_t Magics[] = {0xF0, 0xCC, 0xAA};
+ uint8_t Magic = Magics[Leaves.size()];
+ Leaves.push_back({Op, Parent, Magic});
+ return Magic;
+ };
- if ((FoldableOp = getFoldableLogicOp(N1)))
- A = N0;
- else if ((FoldableOp = getFoldableLogicOp(N0)))
- A = N1;
- else
+ int Imm = ComputeRec(Root, Root.getNode(), true);
+ if (Imm < 0)
return false;
- }
+ ImmOut = static_cast<uint8_t>(Imm & 0xFF);
+ return true;
+ };
+
+ auto EmitFromLeaves = [&](SDNode *Root,
+ const SmallVectorImpl<LeafInfo> &InLeaves,
+ uint8_t Imm) {
+ assert(!InLeaves.empty() && "Expected at least one leaf");
+ SDValue A = InLeaves[0].Leaf;
+ SDNode *ParentA = InLeaves[0].Parent;
+ SDValue B = A;
+ SDNode *ParentB = ParentA;
+ SDValue C = A;
+ SDNode *ParentC = ParentA;
- SDValue B = FoldableOp.getOperand(0);
- SDValue C = FoldableOp.getOperand(1);
- SDNode *ParentA = N;
- SDNode *ParentB = FoldableOp.getNode();
- SDNode *ParentC = FoldableOp.getNode();
-
- // We can build the appropriate control immediate by performing the logic
- // operation we're matching using these constants for A, B, and C.
- uint8_t TernlogMagicA = 0xf0;
- uint8_t TernlogMagicB = 0xcc;
- uint8_t TernlogMagicC = 0xaa;
-
- // Some of the inputs may be inverted, peek through them and invert the
- // magic values accordingly.
- // TODO: There may be a bitcast before the xor that we should peek through.
- auto PeekThroughNot = [](SDValue &Op, SDNode *&Parent, uint8_t &Magic) {
- if (Op.getOpcode() == ISD::XOR && Op.hasOneUse() &&
- ISD::isBuildVectorAllOnes(Op.getOperand(1).getNode())) {
- Magic = ~Magic;
- Parent = Op.getNode();
- Op = Op.getOperand(0);
+ if (InLeaves.size() > 1) {
+ B = InLeaves[1].Leaf;
+ ParentB = InLeaves[1].Parent;
}
+ if (InLeaves.size() > 2) {
+ C = InLeaves[2].Leaf;
+ ParentC = InLeaves[2].Parent;
+ }
+
+ return matchVPTERNLOG(Root, ParentA, ParentB, ParentC, A, B, C, Imm);
};
- PeekThroughNot(A, ParentA, TernlogMagicA);
- PeekThroughNot(B, ParentB, TernlogMagicB);
- PeekThroughNot(C, ParentC, TernlogMagicC);
-
- uint8_t Imm;
- switch (FoldableOp.getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break;
- case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break;
- case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break;
- case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
+ SmallVector<LeafInfo, 3> Leaves;
+ uint8_t Imm = 0;
+ bool TooManyLeaves = false;
+ if (ComputeTernlog(SDValue(N, 0), /*OpaqueSubtree=*/nullptr, Leaves, Imm,
+ TooManyLeaves)) {
+ if (Leaves.size() < 2)
+ return false;
+ return EmitFromLeaves(N, Leaves, Imm);
}
- switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case X86ISD::ANDNP:
- if (A == N0)
- Imm &= ~TernlogMagicA;
- else
- Imm = ~(Imm) & TernlogMagicA;
- break;
- case ISD::AND: Imm &= TernlogMagicA; break;
- case ISD::OR: Imm |= TernlogMagicA; break;
- case ISD::XOR: Imm ^= TernlogMagicA; break;
- }
+ // Generic cascading for >3 leaves: keep one direct root operand as an opaque
+ // input leaf, then fold the remaining logic around it. This allows
+ // multi-level trees to be selected as chained VPTERNLOG operations.
+ if (TooManyLeaves) {
+ auto IsGoodOpaqueCandidate = [&](SDValue V) {
+ SDValue P = PeelSingleUseBitcast(V);
+ if (IsAllOnesXor(P))
+ P = PeelSingleUseBitcast(P.getOperand(0));
+ return IsLogicOpcode(P.getOpcode());
+ };
+
+ SmallVector<unsigned, 2> CandidateOrder;
+ if (IsGoodOpaqueCandidate(N->getOperand(0)))
+ CandidateOrder.push_back(0);
+ if (IsGoodOpaqueCandidate(N->getOperand(1)))
+ CandidateOrder.push_back(1);
+ if (CandidateOrder.empty())
+ return false;
+
+ // Prefer single-use subtrees first; they are better cascading anchors.
+ if (CandidateOrder.size() == 2 &&
+ !N->getOperand(CandidateOrder[0]).hasOneUse() &&
+ N->getOperand(CandidateOrder[1]).hasOneUse())
+ std::swap(CandidateOrder[0], CandidateOrder[1]);
- if (PeeledOuterNot)
- Imm = ~Imm;
+ for (unsigned Idx : CandidateOrder) {
+ SmallVector<LeafInfo, 3> CascadedLeaves;
+ uint8_t CascadedImm = 0;
+ bool CascadedTooManyLeaves = false;
+ SDNode *OpaqueSubtree = N->getOperand(Idx).getNode();
- return matchVPTERNLOG(OriN, ParentA, ParentB, ParentC, A, B, C, Imm);
+ if (!ComputeTernlog(SDValue(N, 0), OpaqueSubtree, CascadedLeaves,
+ CascadedImm, CascadedTooManyLeaves))
+ continue;
+
+ if (CascadedLeaves.size() < 2)
+ continue;
+
+ if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
+ return true;
+ }
+ }
+
+ return false;
}
/// If the high bits of an 'and' operand are known zero, try setting the
diff --git a/llvm/test/CodeGen/X86/vpternlog.ll b/llvm/test/CodeGen/X86/vpternlog.ll
index bd7478d3a82d5..100d31883823d 100644
--- a/llvm/test/CodeGen/X86/vpternlog.ll
+++ b/llvm/test/CodeGen/X86/vpternlog.ll
@@ -4,7 +4,7 @@
define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
; CHECK-LABEL: foo:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm2 | zmm1)
+; CHECK-NEXT: vpternlogq $1
; CHECK-NEXT: retq
%and.demorgan = or <8 x i64> %b, %a
%and3.demorgan = or <8 x i64> %and.demorgan, %c
@@ -15,7 +15,7 @@ define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
; CHECK-LABEL: xorbitcast:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm2 | zmm1)
+; CHECK-NEXT: vpternlogq $1
; CHECK-NEXT: retq
%or1 = or <64 x i8> %a, %b
%or2 = or <64 x i8> %or1, %c
@@ -23,3 +23,77 @@ define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
%xor = xor <8 x i64> %cast, splat (i64 -1)
ret <8 x i64> %xor
}
+
+define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d, <8 x i64> %e) {
+; CHECK-LABEL: foobar:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq $208
+; CHECK-NEXT: vpternlogq $252
+; CHECK-NEXT: vpandnq
+; CHECK-NEXT: retq
+ %nb = xor <8 x i64> %b, splat (i64 -1)
+ %or = or <8 x i64> %nb, %c
+ %foo = and <8 x i64> %or, %a
+ %de = or <8 x i64> %d, %e
+ %nde = xor <8 x i64> %de, splat (i64 -1)
+ %bar = and <8 x i64> %foo, %nde
+ ret <8 x i64> %bar
+}
+
+define <8 x i64> @or_not_and_guard(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d) {
+; CHECK-LABEL: or_not_and_guard:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq $16
+; CHECK-NOT: vpternlogq $252
+; CHECK: retq
+ %or = or <8 x i64> %b, %c
+ %not_or = xor <8 x i64> %or, splat (i64 -1)
+ %lhs = and <8 x i64> %a, %not_or
+ %res = and <8 x i64> %lhs, %d
+ ret <8 x i64> %res
+}
+
+define <8 x i64> @depth4_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d, <8 x i64> %e) {
+; CHECK-LABEL: depth4_chain:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq
+; CHECK: retq
+ %t0 = xor <8 x i64> %a, %b
+ %t1 = or <8 x i64> %t0, %c
+ %t2 = and <8 x i64> %t1, %d
+ %t3 = xor <8 x i64> %t2, %e
+ ret <8 x i64> %t3
+}
+
+define <8 x i64> @depth5_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; CHECK-LABEL: depth5_chain:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq
+; CHECK: retq
+ %t0 = and <8 x i64> %a, %b
+ %t1 = xor <8 x i64> %t0, %c
+ %t2 = or <8 x i64> %t1, %d
+ %t3 = and <8 x i64> %t2, %e
+ %t4 = xor <8 x i64> %t3, %f
+ ret <8 x i64> %t4
+}
+
+define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; CHECK-LABEL: balanced_depth:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq
+; CHECK: vpternlogq
+; CHECK: vpternlogq
+; CHECK: retq
+ %l0 = or <8 x i64> %a, %b
+ %l1 = xor <8 x i64> %c, %d
+ %l2 = and <8 x i64> %l0, %l1
+ %r0 = xor <8 x i64> %e, %f
+ %res = or <8 x i64> %l2, %r0
+ ret <8 x i64> %res
+}
>From ecc91eba380937db5e8d1ae26913155f441ec792 Mon Sep 17 00:00:00 2001
From: raventid <juliankul at gmail.com>
Date: Wed, 1 Apr 2026 22:46:46 +0800
Subject: [PATCH 2/3] [AVX-512] make vpternlogq more aggressive for longer
chains of bitmanipulations
---
llvm/lib/Target/X86/X86FixupInstTuning.cpp | 133 +++++++++++++
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 221 ++++++++++++++++++++-
llvm/test/CodeGen/X86/vpternlog.ll | 123 +++++++++++-
3 files changed, 469 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/X86/X86FixupInstTuning.cpp b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
index af3c3af38e681..c766da744a581 100644
--- a/llvm/lib/Target/X86/X86FixupInstTuning.cpp
+++ b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
@@ -21,6 +21,7 @@
//
//===----------------------------------------------------------------------===//
+#include "MCTargetDesc/X86BaseInfo.h"
#include "X86.h"
#include "X86InstrInfo.h"
#include "X86RegisterInfo.h"
@@ -37,8 +38,135 @@ using namespace llvm;
#define DEBUG_TYPE "x86-fixup-inst-tuning"
STATISTIC(NumInstChanges, "Number of instructions changes");
+STATISTIC(NumVPTERNLOGNotFolds,
+ "Number of all-ones + XOR-mem fused into VPTERNLOG NOT");
namespace {
+
+/// Return the VPTERNLOG-rmi opcode for the given XOR-mem opcode, or 0 if there
+/// is no corresponding opcode. We use the Q (64-bit element) variant for the
+/// VPTERNLOG so that the memory operand can be folded with the larger element
+/// granularity – element type is irrelevant for a bitwise NOT.
+static unsigned getVPTERNLOGForXORrm(unsigned XorOpc) {
+ switch (XorOpc) {
+ default:
+ return 0;
+ case X86::VPXORQZrm:
+ case X86::VPXORDZrm:
+ return X86::VPTERNLOGQZrmi;
+ case X86::VPXORQZ256rm:
+ case X86::VPXORDZ256rm:
+ return X86::VPTERNLOGQZ256rmi;
+ case X86::VPXORQZ128rm:
+ case X86::VPXORDZ128rm:
+ return X86::VPTERNLOGQZ128rmi;
+ }
+}
+
+/// Return true if \p MI is a VPTERNLOG-rri that materializes all-ones
+/// (immediate 0xFF) with all source operands marked as undef.
+static bool isVPTERNLOGAllOnes(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ default:
+ return false;
+ case X86::VPTERNLOGDZrri:
+ case X86::VPTERNLOGQZrri:
+ case X86::VPTERNLOGDZ256rri:
+ case X86::VPTERNLOGQZ256rri:
+ case X86::VPTERNLOGDZ128rri:
+ case X86::VPTERNLOGQZ128rri:
+ break;
+ }
+ // The last operand is the immediate; it must be 0xFF (all-ones).
+ const MachineOperand &ImmOp = MI.getOperand(MI.getNumOperands() - 1);
+ return ImmOp.isImm() && (ImmOp.getImm() & 0xFF) == 0xFF;
+}
+
+/// Try to fuse an all-ones materialization followed by a vector XOR-from-memory
+/// into a single VPTERNLOG NOT-from-memory:
+///
+/// $dst = VPTERNLOGDZrri undef $dst, undef $dst, undef $dst, 255
+/// $dst = VPXORQZrm killed $dst, <mem>
+///
+/// becomes:
+///
+/// $dst = VPTERNLOGQZrmi undef $dst, undef $dst, <mem>, 0x55
+///
+/// The immediate 0x55 = ~C (where C = src3 = memory operand), which is
+/// independent of src1 and src2.
+static bool tryFuseNotFromMem(const X86InstrInfo *TII, MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator &I) {
+ MachineInstr &XorMI = *I;
+ unsigned TernlogOpc = getVPTERNLOGForXORrm(XorMI.getOpcode());
+ if (!TernlogOpc)
+ return false;
+
+ // The XOR-rm layout: $dst(tied=$src1), $src1, base, scale, index, disp, seg.
+ Register DstReg = XorMI.getOperand(0).getReg();
+
+ // Walk backward to find the all-ones materialization. Skip debug and
+ // position-independent instructions, but stop at any real instruction that
+ // touches DstReg.
+ MachineBasicBlock::iterator PrevIt = I;
+ if (PrevIt == MBB.begin())
+ return false;
+
+ MachineInstr *AllOnesMI = nullptr;
+ for (--PrevIt;; --PrevIt) {
+ MachineInstr &Prev = *PrevIt;
+
+ if (Prev.isDebugInstr()) {
+ if (PrevIt == MBB.begin())
+ return false;
+ continue;
+ }
+
+ if (isVPTERNLOGAllOnes(Prev) && Prev.getOperand(0).getReg() == DstReg) {
+ AllOnesMI = &Prev;
+ break;
+ }
+
+ // Any other instruction that reads or writes DstReg blocks the fold.
+ return false;
+ }
+
+ if (!AllOnesMI)
+ return false;
+
+ // Verify that the all-ones defines only DstReg and has no other users
+ // between itself and the XOR. Since they are adjacent (modulo debug instrs)
+ // and both write DstReg, this is guaranteed.
+
+ LLVM_DEBUG(dbgs() << "Fusing VPTERNLOG NOT-from-memory:\n"
+ << " " << *AllOnesMI << " " << XorMI);
+
+ // Build: $dst = VPTERNLOGQZrmi undef $dst, undef $dst, <mem>, 0x0F
+ // The XOR-rm operands: 0=dst, 1=src1, 2..6=mem(base,scale,index,disp,seg)
+ MachineInstrBuilder MIB =
+ BuildMI(MBB, I, XorMI.getDebugLoc(), TII->get(TernlogOpc), DstReg)
+ .addReg(DstReg, RegState::Undef) // src1 (tied, don't care)
+ .addReg(DstReg, RegState::Undef); // src2 (don't care)
+
+ // Copy the 5 memory addressing operands from the XOR.
+ for (unsigned J = 2; J < 2 + X86::AddrNumOperands; ++J)
+ MIB.add(XorMI.getOperand(J));
+
+ MIB.addImm(0x55); // imm = ~C where C = src3 = memory operand
+
+ // Preserve mem-refs from the XOR.
+ MIB.setMemRefs(XorMI.memoperands());
+
+ LLVM_DEBUG(dbgs() << " -> " << *MIB);
+
+ // Erase the two old instructions.
+ AllOnesMI->eraseFromParent();
+ I = MIB.getInstr()->getIterator();
+ XorMI.eraseFromParent();
+
+ ++NumVPTERNLOGNotFolds;
+ return true;
+}
+
class X86FixupInstTuningImpl {
public:
bool runOnMachineFunction(MachineFunction &MF);
@@ -683,6 +811,11 @@ bool X86FixupInstTuningImpl::runOnMachineFunction(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
+ // Try fusing all-ones + XOR-mem → VPTERNLOG NOT-mem first.
+ if (tryFuseNotFromMem(TII, MBB, I)) {
+ Changed = true;
+ continue;
+ }
if (processInstruction(MF, MBB, I)) {
++NumInstChanges;
Changed = true;
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 1dc6c10aba4a6..17c7d7dfe92a5 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -4835,6 +4835,10 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
Opc == X86ISD::ANDNP;
};
+ auto IsAllOnesVec = [](SDValue V) {
+ return ISD::isBuildVectorAllOnes(V.getNode());
+ };
+
auto IsAllOnesXor = [](SDValue V) {
return V.getOpcode() == ISD::XOR &&
ISD::isBuildVectorAllOnes(V.getOperand(1).getNode());
@@ -4846,6 +4850,47 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
return V;
};
+ std::function<bool(SDValue, unsigned)> IsHomogeneousAssociativeTree =
+ [&](SDValue V, unsigned Opc) {
+ V = PeelSingleUseBitcast(V);
+ if (V.getOpcode() != Opc)
+ return true;
+ if (!V.hasOneUse())
+ return false;
+ return IsHomogeneousAssociativeTree(V.getOperand(0), Opc) &&
+ IsHomogeneousAssociativeTree(V.getOperand(1), Opc);
+ };
+
+ auto IsLoadLike = [](SDValue V) {
+ return isa<LoadSDNode>(V.getNode()) ||
+ V.getOpcode() == X86ISD::VBROADCAST_LOAD;
+ };
+
+ // Fast-path: X ^ -1 -> ~X.
+ //
+ // Use X for all three VPTERNLOG inputs and select an immediate that yields
+ // ~X when A == B == C == X (imm bit0 = 1, bit7 = 0; other bits are don't
+ // care). This avoids introducing undef register operands.
+ //
+ // Keep this for register-like X only. For load-like X, this can cause an
+ // extra move/load before a folded-load VPTERNLOG form, which is usually not
+ // profitable.
+ if (N->getOpcode() == ISD::XOR) {
+ for (unsigned Idx = 0; Idx != 2; ++Idx) {
+ if (!IsAllOnesVec(N->getOperand(Idx)))
+ continue;
+
+ SDValue X = N->getOperand(Idx ^ 1);
+ SDValue XNoCast = PeelSingleUseBitcast(X);
+ if (IsLogicOpcode(XNoCast.getOpcode()) || IsAllOnesXor(XNoCast) ||
+ IsLoadLike(XNoCast))
+ continue;
+
+ if (matchVPTERNLOG(N, N, N, N, X, X, X, 0x01))
+ return true;
+ }
+ }
+
// Avoid consuming OR into a stand-alone VPTERNLOG if it is part of a
// higher-level A & ~(B | C) shape. Let the parent AND/ANDNP matcher absorb
// the whole pattern instead.
@@ -4866,6 +4911,26 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
}
}
+ // Avoid consuming xor(logic_op, -1) (i.e. NOT of a logic sub-tree) into a
+ // stand-alone VPTERNLOG when the parent is also a logic op. The parent's
+ // ComputeTernlog will fold the NOT directly into the truth-table immediate,
+ // producing fewer instructions overall.
+ if (N->getOpcode() == ISD::XOR && N->hasOneUse() &&
+ (ISD::isBuildVectorAllOnes(N->getOperand(0).getNode()) ||
+ ISD::isBuildVectorAllOnes(N->getOperand(1).getNode()))) {
+ SDValue Inner = ISD::isBuildVectorAllOnes(N->getOperand(1).getNode())
+ ? N->getOperand(0)
+ : N->getOperand(1);
+ SDValue InnerNoCast = PeelSingleUseBitcast(Inner);
+ if (IsLogicOpcode(InnerNoCast.getOpcode())) {
+ SDNode *User = *N->user_begin();
+ while (User->getOpcode() == ISD::BITCAST && User->hasOneUse())
+ User = *User->user_begin();
+ if (IsLogicOpcode(User->getOpcode()))
+ return false;
+ }
+ }
+
// Fast-path: A & ~(B | C) -> vpternlog(A, B, C, 0x10)
if (N->getOpcode() == ISD::AND) {
for (unsigned Idx = 0; Idx != 2; ++Idx) {
@@ -4894,8 +4959,8 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
};
auto ComputeTernlog = [&](SDValue Root, SDNode *OpaqueSubtree,
- SmallVectorImpl<LeafInfo> &Leaves,
- uint8_t &ImmOut, bool &TooManyLeaves) {
+ SmallVectorImpl<LeafInfo> &Leaves, uint8_t &ImmOut,
+ bool &TooManyLeaves) {
TooManyLeaves = false;
auto lookupLeaf = [&](SDValue Leaf) -> std::optional<uint8_t> {
@@ -4989,7 +5054,13 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
bool TooManyLeaves = false;
if (ComputeTernlog(SDValue(N, 0), /*OpaqueSubtree=*/nullptr, Leaves, Imm,
TooManyLeaves)) {
- if (Leaves.size() < 2)
+ if (Leaves.empty())
+ return false;
+ // A single-leaf load folded into VPTERNLOG causes a redundant explicit
+ // load (for the tied src1=dst) plus a folded load, doubling memory
+ // traffic. Bail out and let default lowering handle it (e.g.
+ // SETALLONES + VPXORQ mem for NOT-of-load).
+ if (Leaves.size() == 1 && IsLoadLike(PeelSingleUseBitcast(Leaves[0].Leaf)))
return false;
return EmitFromLeaves(N, Leaves, Imm);
}
@@ -4998,6 +5069,45 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
// input leaf, then fold the remaining logic around it. This allows
// multi-level trees to be selected as chained VPTERNLOG operations.
if (TooManyLeaves) {
+ bool IsAssocOp = N->getOpcode() == ISD::OR || N->getOpcode() == ISD::AND ||
+ N->getOpcode() == ISD::XOR;
+ bool IsHomogeneousAssoc = IsAssocOp && IsHomogeneousAssociativeTree(
+ SDValue(N, 0), N->getOpcode());
+
+ // For homogeneous associative trees, prefer choosing an opaque subtree
+ // from one level below the root-side logic node so we can expose two fresh
+ // siblings and form 3-input VPTERNLOG combines (e.g. OR reductions as
+ // repeated imm=254), rather than creating long 2-input $252/$250 chains.
+ if (IsHomogeneousAssoc) {
+ for (unsigned RootIdx = 0; RootIdx != 2; ++RootIdx) {
+ SDValue Side = PeelSingleUseBitcast(N->getOperand(RootIdx));
+ if (!Side.hasOneUse() || Side.getOpcode() != N->getOpcode())
+ continue;
+
+ for (unsigned ChildIdx = 0; ChildIdx != 2; ++ChildIdx) {
+ SDValue Child = PeelSingleUseBitcast(Side.getOperand(ChildIdx));
+ if (!Child.hasOneUse() || Child.getOpcode() != N->getOpcode())
+ continue;
+
+ SmallVector<LeafInfo, 3> AssocLeaves;
+ uint8_t AssocImm = 0;
+ bool AssocTooManyLeaves = false;
+ if (!ComputeTernlog(SDValue(N, 0), Child.getNode(), AssocLeaves,
+ AssocImm, AssocTooManyLeaves))
+ continue;
+
+ if (AssocLeaves.size() < 2)
+ continue;
+
+ if (EmitFromLeaves(N, AssocLeaves, AssocImm))
+ return true;
+ }
+ }
+ // Fall through to generic cascading — for balanced trees all
+ // grandchildren may be leaves, so the child-as-opaque strategy below
+ // can still produce a valid 3-input combine.
+ }
+
auto IsGoodOpaqueCandidate = [&](SDValue V) {
SDValue P = PeelSingleUseBitcast(V);
if (IsAllOnesXor(P))
@@ -5019,6 +5129,25 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
N->getOperand(CandidateOrder[1]).hasOneUse())
std::swap(CandidateOrder[0], CandidateOrder[1]);
+ // Collect all opaque subtree candidates: direct children and, if a direct
+ // child is itself a logic op, its grandchildren. Trying grandchildren
+ // allows the root VPTERNLOG to absorb more distinct operands (3 instead
+ // of 2), which produces tighter cascades. Example:
+ //
+ // xor With child "and" opaque: 2 leaves (and, e)
+ // / \ With grandchild "or" opaque: 3 leaves (or, d, e)
+ // and e → saves one instruction in the cascade.
+ // / \
+ // or d
+ //
+ // Try direct children first. Only explore grandchildren when all direct
+ // children produce ≤2 leaves (i.e. a degenerate 2-input fold that wastes
+ // a VPTERNLOG slot) AND the opaque subtree itself has >3 leaves, meaning
+ // a single VPTERNLOG cannot handle it. When the opaque child fits in one
+ // VPTERNLOG (≤3 leaves), going deeper just reshuffles the split without
+ // saving instructions.
+ bool TriedDirect = false;
+ bool NeedsGrandchild = false;
for (unsigned Idx : CandidateOrder) {
SmallVector<LeafInfo, 3> CascadedLeaves;
uint8_t CascadedImm = 0;
@@ -5029,11 +5158,91 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
CascadedImm, CascadedTooManyLeaves))
continue;
- if (CascadedLeaves.size() < 2)
+ if (CascadedLeaves.empty())
continue;
- if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
- return true;
+ // If the direct child yields a 3-leaf fold, emit it right away — this
+ // is already optimal for this level.
+ if (CascadedLeaves.size() == 3) {
+ if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
+ return true;
+ }
+ TriedDirect = true;
+
+ // Check if the opaque subtree itself has >3 leaves: if so, it cannot
+ // be handled by a single VPTERNLOG, and going one level deeper may
+ // help reduce the total instruction count.
+ if (CascadedLeaves.size() <= 2) {
+ SmallVector<LeafInfo, 3> SubLeaves;
+ uint8_t SubImm = 0;
+ bool SubTooMany = false;
+ if (!ComputeTernlog(SDValue(N->getOperand(Idx)),
+ /*OpaqueSubtree=*/nullptr, SubLeaves, SubImm,
+ SubTooMany) &&
+ SubTooMany)
+ NeedsGrandchild = true;
+ }
+ }
+
+ // Direct children only yielded ≤2-leaf folds and the opaque subtree has
+ // >3 leaves (can't fit in one VPTERNLOG). Try grandchildren — making a
+ // deeper subtree opaque exposes more leaves at the root level, reducing
+ // the total instruction count.
+ if (NeedsGrandchild) {
+ for (unsigned Idx : CandidateOrder) {
+ SDValue Child = N->getOperand(Idx);
+ SDValue ChildNoCast = PeelSingleUseBitcast(Child);
+ if (!ChildNoCast.hasOneUse() || !IsLogicOpcode(ChildNoCast.getOpcode()))
+ continue;
+
+ for (unsigned GIdx = 0; GIdx != 2; ++GIdx) {
+ SDValue GChild = ChildNoCast.getOperand(GIdx);
+ SDValue GChildNoCast = PeelSingleUseBitcast(GChild);
+
+ // Skip NOT-wrappers (xor X, -1): ComputeTernlog already folds NOT
+ // into the truth table, so cutting at a NOT boundary just pushes
+ // the NOT into a separate instruction without saving anything.
+ if (IsAllOnesXor(GChildNoCast))
+ continue;
+
+ if (!IsLogicOpcode(GChildNoCast.getOpcode()))
+ continue;
+
+ SmallVector<LeafInfo, 3> CascadedLeaves;
+ uint8_t CascadedImm = 0;
+ bool CascadedTooManyLeaves = false;
+
+ if (!ComputeTernlog(SDValue(N, 0), GChild.getNode(), CascadedLeaves,
+ CascadedImm, CascadedTooManyLeaves))
+ continue;
+
+ if (CascadedLeaves.size() < 3)
+ continue;
+
+ if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
+ return true;
+ }
+ }
+ }
+
+ // Fall back to direct-child opaque with ≤2 leaves if nothing else worked.
+ if (TriedDirect) {
+ for (unsigned Idx : CandidateOrder) {
+ SmallVector<LeafInfo, 3> CascadedLeaves;
+ uint8_t CascadedImm = 0;
+ bool CascadedTooManyLeaves = false;
+ SDNode *OpaqueSubtree = N->getOperand(Idx).getNode();
+
+ if (!ComputeTernlog(SDValue(N, 0), OpaqueSubtree, CascadedLeaves,
+ CascadedImm, CascadedTooManyLeaves))
+ continue;
+
+ if (CascadedLeaves.empty())
+ continue;
+
+ if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
+ return true;
+ }
}
}
diff --git a/llvm/test/CodeGen/X86/vpternlog.ll b/llvm/test/CodeGen/X86/vpternlog.ll
index 100d31883823d..4f9e384bd85a0 100644
--- a/llvm/test/CodeGen/X86/vpternlog.ll
+++ b/llvm/test/CodeGen/X86/vpternlog.ll
@@ -29,8 +29,7 @@ define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
; CHECK-LABEL: foobar:
; CHECK: # %bb.0:
; CHECK-NEXT: vpternlogq $208
-; CHECK-NEXT: vpternlogq $252
-; CHECK-NEXT: vpandnq
+; CHECK-NEXT: vpternlogq $16
; CHECK-NEXT: retq
%nb = xor <8 x i64> %b, splat (i64 -1)
%or = or <8 x i64> %nb, %c
@@ -97,3 +96,123 @@ define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
%res = or <8 x i64> %l2, %r0
ret <8 x i64> %res
}
+
+define <8 x i64> @flip(ptr %x) {
+; CHECK-LABEL: flip:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq $85, (%rdi), %zmm0, %zmm0
+; CHECK-NEXT: retq
+ %ld = load <8 x i64>, ptr %x, align 64
+ %not = xor <8 x i64> %ld, splat (i64 -1)
+ ret <8 x i64> %not
+}
+
+define dso_local <8 x i64> @fubar(<8 x i64> %0, <8 x i64> %1, <8 x i64> %2,
+ <8 x i64> %3, <8 x i64> %4, <8 x i64> %5,
+ <8 x i64> %6, <8 x i64> %7, <8 x i64> %8)
+ local_unnamed_addr {
+; CHECK-LABEL: fubar:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK-NOT: vpternlogq $252
+; CHECK-NOT: vpternlogq $250
+; CHECK: retq
+Entry:
+ %9 = or <8 x i64> %1, %0
+ %10 = or <8 x i64> %9, %2
+ %11 = or <8 x i64> %10, %3
+ %12 = or <8 x i64> %11, %4
+ %13 = or <8 x i64> %12, %5
+ %14 = or <8 x i64> %13, %6
+ %15 = or <8 x i64> %14, %7
+ %16 = or <8 x i64> %15, %8
+ ret <8 x i64> %16
+}
+
+define dso_local <8 x i64> @baz(<8 x i64> %0, <8 x i64> %1, <8 x i64> %2,
+ <8 x i64> %3, <8 x i64> %4, <8 x i64> %5,
+ <8 x i64> %6, <8 x i64> %7, <8 x i64> %8)
+ local_unnamed_addr {
+; CHECK-LABEL: baz:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK: vpternlogq $254
+; CHECK: retq
+Entry:
+ %9 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %0,
+ <8 x i64> %1,
+ <8 x i64> %2,
+ i32 254)
+ %10 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %3,
+ <8 x i64> %4,
+ <8 x i64> %5,
+ i32 254)
+ %11 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %6,
+ <8 x i64> %7,
+ <8 x i64> %8,
+ i32 254)
+ %12 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %9,
+ <8 x i64> %10,
+ <8 x i64> %11,
+ i32 254)
+ ret <8 x i64> %12
+}
+
+declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>,
+ <8 x i64>, i32 immarg)
+
+; 256-bit vector case — exercises VLX path.
+define <4 x i64> @foo_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
+; CHECK-LABEL: foo_256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: retq
+ %or1 = or <4 x i64> %b, %a
+ %or2 = or <4 x i64> %or1, %c
+ %not = xor <4 x i64> %or2, splat (i64 -1)
+ ret <4 x i64> %not
+}
+
+; 128-bit vector case — exercises VLX path.
+define <2 x i64> @foo_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: foo_128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: retq
+ %or1 = or <2 x i64> %b, %a
+ %or2 = or <2 x i64> %or1, %c
+ %not = xor <2 x i64> %or2, splat (i64 -1)
+ ret <2 x i64> %not
+}
+
+; Balanced OR tree — tests fallthrough from homogeneous-associative to generic.
+define <8 x i64> @balanced_or4(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
+ <8 x i64> %d) {
+; CHECK-LABEL: balanced_or4:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq $254
+; CHECK-NOT: vporq
+; CHECK: retq
+ %t1 = or <8 x i64> %a, %b
+ %t2 = or <8 x i64> %c, %d
+ %res = or <8 x i64> %t1, %t2
+ ret <8 x i64> %res
+}
+
+; Multi-use operand — %a used in both ORs. hasOneUse() on the shared
+; value must not block the combine.
+define <8 x i64> @shared_operand(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
+; CHECK-LABEL: shared_operand:
+; CHECK: # %bb.0:
+; CHECK: vpternlogq
+; CHECK-NEXT: retq
+ %or1 = or <8 x i64> %a, %b
+ %or2 = or <8 x i64> %a, %c
+ %res = and <8 x i64> %or1, %or2
+ ret <8 x i64> %res
+}
>From 9aad9778b015184c52d5913114bdd099f8e8e427 Mon Sep 17 00:00:00 2001
From: raventid <juliankul at gmail.com>
Date: Tue, 2 Jun 2026 20:15:26 +0800
Subject: [PATCH 3/3] feat: simplified design for a more aggressive vpternlog
---
llvm/lib/Target/X86/X86FixupInstTuning.cpp | 133 --
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 499 ++---
llvm/test/CodeGen/X86/avx512-cvt.ll | 21 +-
llvm/test/CodeGen/X86/avx512-logic.ll | 2 +-
llvm/test/CodeGen/X86/avx512fp16-arith.ll | 28 +-
llvm/test/CodeGen/X86/avx512vl-logic.ll | 10 +-
llvm/test/CodeGen/X86/bitcast-vector-bool.ll | 2 +-
llvm/test/CodeGen/X86/clmul-vector-256.ll | 84 +-
llvm/test/CodeGen/X86/clmul-vector-512.ll | 478 ++---
llvm/test/CodeGen/X86/combine-fround.ll | 24 +-
llvm/test/CodeGen/X86/combine-or-shuffle.ll | 8 +-
.../X86/elementwise-store-of-scalar-splat.ll | 12 +-
.../CodeGen/X86/fminimumnum-fmaximumnum.ll | 16 +-
llvm/test/CodeGen/X86/fp-round.ll | 74 +-
llvm/test/CodeGen/X86/fshl-splat-undef.ll | 6 +-
llvm/test/CodeGen/X86/gfni-shifts.ll | 18 +-
llvm/test/CodeGen/X86/icmp-pow2-diff.ll | 15 +-
.../CodeGen/X86/min-legal-vector-width.ll | 12 +-
llvm/test/CodeGen/X86/pmul.ll | 4 +-
.../CodeGen/X86/prefer-avx256-wide-mul.ll | 2 +-
llvm/test/CodeGen/X86/sat-add.ll | 10 +-
llvm/test/CodeGen/X86/select-big-integer.ll | 12 +-
.../CodeGen/X86/srem-seteq-vec-nonsplat.ll | 12 +-
.../subvectorwise-store-of-vector-splat.ll | 1618 ++++++++++++-----
.../X86/urem-seteq-vec-tautological.ll | 5 +-
llvm/test/CodeGen/X86/vector-fshr-256.ll | 2 +-
llvm/test/CodeGen/X86/vector-fshr-512.ll | 4 +-
.../vector-interleaved-load-i16-stride-6.ll | 8 +-
.../vector-interleaved-load-i8-stride-6.ll | 64 +-
.../vector-interleaved-load-i8-stride-7.ll | 80 +-
.../vector-interleaved-store-i16-stride-3.ll | 32 +-
.../vector-interleaved-store-i16-stride-5.ll | 40 +-
.../vector-interleaved-store-i16-stride-7.ll | 16 +-
.../vector-interleaved-store-i8-stride-5.ll | 16 +-
.../vector-interleaved-store-i8-stride-7.ll | 1102 ++++++-----
.../CodeGen/X86/vector-reduce-fmaximum.ll | 43 +-
llvm/test/CodeGen/X86/vector-rotate-128.ll | 8 +-
llvm/test/CodeGen/X86/vector-rotate-256.ll | 8 +-
llvm/test/CodeGen/X86/vector-rotate-512.ll | 24 +-
.../test/CodeGen/X86/vector-shift-ashr-256.ll | 4 +-
.../test/CodeGen/X86/vector-shift-ashr-512.ll | 4 +-
llvm/test/CodeGen/X86/vpternlog.ll | 165 +-
.../zero_extend_vector_inreg_of_broadcast.ll | 66 +-
...d_vector_inreg_of_broadcast_from_memory.ll | 66 +-
44 files changed, 2559 insertions(+), 2298 deletions(-)
diff --git a/llvm/lib/Target/X86/X86FixupInstTuning.cpp b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
index c766da744a581..af3c3af38e681 100644
--- a/llvm/lib/Target/X86/X86FixupInstTuning.cpp
+++ b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
@@ -21,7 +21,6 @@
//
//===----------------------------------------------------------------------===//
-#include "MCTargetDesc/X86BaseInfo.h"
#include "X86.h"
#include "X86InstrInfo.h"
#include "X86RegisterInfo.h"
@@ -38,135 +37,8 @@ using namespace llvm;
#define DEBUG_TYPE "x86-fixup-inst-tuning"
STATISTIC(NumInstChanges, "Number of instructions changes");
-STATISTIC(NumVPTERNLOGNotFolds,
- "Number of all-ones + XOR-mem fused into VPTERNLOG NOT");
namespace {
-
-/// Return the VPTERNLOG-rmi opcode for the given XOR-mem opcode, or 0 if there
-/// is no corresponding opcode. We use the Q (64-bit element) variant for the
-/// VPTERNLOG so that the memory operand can be folded with the larger element
-/// granularity – element type is irrelevant for a bitwise NOT.
-static unsigned getVPTERNLOGForXORrm(unsigned XorOpc) {
- switch (XorOpc) {
- default:
- return 0;
- case X86::VPXORQZrm:
- case X86::VPXORDZrm:
- return X86::VPTERNLOGQZrmi;
- case X86::VPXORQZ256rm:
- case X86::VPXORDZ256rm:
- return X86::VPTERNLOGQZ256rmi;
- case X86::VPXORQZ128rm:
- case X86::VPXORDZ128rm:
- return X86::VPTERNLOGQZ128rmi;
- }
-}
-
-/// Return true if \p MI is a VPTERNLOG-rri that materializes all-ones
-/// (immediate 0xFF) with all source operands marked as undef.
-static bool isVPTERNLOGAllOnes(const MachineInstr &MI) {
- switch (MI.getOpcode()) {
- default:
- return false;
- case X86::VPTERNLOGDZrri:
- case X86::VPTERNLOGQZrri:
- case X86::VPTERNLOGDZ256rri:
- case X86::VPTERNLOGQZ256rri:
- case X86::VPTERNLOGDZ128rri:
- case X86::VPTERNLOGQZ128rri:
- break;
- }
- // The last operand is the immediate; it must be 0xFF (all-ones).
- const MachineOperand &ImmOp = MI.getOperand(MI.getNumOperands() - 1);
- return ImmOp.isImm() && (ImmOp.getImm() & 0xFF) == 0xFF;
-}
-
-/// Try to fuse an all-ones materialization followed by a vector XOR-from-memory
-/// into a single VPTERNLOG NOT-from-memory:
-///
-/// $dst = VPTERNLOGDZrri undef $dst, undef $dst, undef $dst, 255
-/// $dst = VPXORQZrm killed $dst, <mem>
-///
-/// becomes:
-///
-/// $dst = VPTERNLOGQZrmi undef $dst, undef $dst, <mem>, 0x55
-///
-/// The immediate 0x55 = ~C (where C = src3 = memory operand), which is
-/// independent of src1 and src2.
-static bool tryFuseNotFromMem(const X86InstrInfo *TII, MachineBasicBlock &MBB,
- MachineBasicBlock::iterator &I) {
- MachineInstr &XorMI = *I;
- unsigned TernlogOpc = getVPTERNLOGForXORrm(XorMI.getOpcode());
- if (!TernlogOpc)
- return false;
-
- // The XOR-rm layout: $dst(tied=$src1), $src1, base, scale, index, disp, seg.
- Register DstReg = XorMI.getOperand(0).getReg();
-
- // Walk backward to find the all-ones materialization. Skip debug and
- // position-independent instructions, but stop at any real instruction that
- // touches DstReg.
- MachineBasicBlock::iterator PrevIt = I;
- if (PrevIt == MBB.begin())
- return false;
-
- MachineInstr *AllOnesMI = nullptr;
- for (--PrevIt;; --PrevIt) {
- MachineInstr &Prev = *PrevIt;
-
- if (Prev.isDebugInstr()) {
- if (PrevIt == MBB.begin())
- return false;
- continue;
- }
-
- if (isVPTERNLOGAllOnes(Prev) && Prev.getOperand(0).getReg() == DstReg) {
- AllOnesMI = &Prev;
- break;
- }
-
- // Any other instruction that reads or writes DstReg blocks the fold.
- return false;
- }
-
- if (!AllOnesMI)
- return false;
-
- // Verify that the all-ones defines only DstReg and has no other users
- // between itself and the XOR. Since they are adjacent (modulo debug instrs)
- // and both write DstReg, this is guaranteed.
-
- LLVM_DEBUG(dbgs() << "Fusing VPTERNLOG NOT-from-memory:\n"
- << " " << *AllOnesMI << " " << XorMI);
-
- // Build: $dst = VPTERNLOGQZrmi undef $dst, undef $dst, <mem>, 0x0F
- // The XOR-rm operands: 0=dst, 1=src1, 2..6=mem(base,scale,index,disp,seg)
- MachineInstrBuilder MIB =
- BuildMI(MBB, I, XorMI.getDebugLoc(), TII->get(TernlogOpc), DstReg)
- .addReg(DstReg, RegState::Undef) // src1 (tied, don't care)
- .addReg(DstReg, RegState::Undef); // src2 (don't care)
-
- // Copy the 5 memory addressing operands from the XOR.
- for (unsigned J = 2; J < 2 + X86::AddrNumOperands; ++J)
- MIB.add(XorMI.getOperand(J));
-
- MIB.addImm(0x55); // imm = ~C where C = src3 = memory operand
-
- // Preserve mem-refs from the XOR.
- MIB.setMemRefs(XorMI.memoperands());
-
- LLVM_DEBUG(dbgs() << " -> " << *MIB);
-
- // Erase the two old instructions.
- AllOnesMI->eraseFromParent();
- I = MIB.getInstr()->getIterator();
- XorMI.eraseFromParent();
-
- ++NumVPTERNLOGNotFolds;
- return true;
-}
-
class X86FixupInstTuningImpl {
public:
bool runOnMachineFunction(MachineFunction &MF);
@@ -811,11 +683,6 @@ bool X86FixupInstTuningImpl::runOnMachineFunction(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
- // Try fusing all-ones + XOR-mem → VPTERNLOG NOT-mem first.
- if (tryFuseNotFromMem(TII, MBB, I)) {
- Changed = true;
- continue;
- }
if (processInstruction(MF, MBB, I)) {
++NumInstChanges;
Changed = true;
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 17c7d7dfe92a5..d27e67f8c67db 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -33,7 +33,6 @@
#include "llvm/Support/MathExtras.h"
#include <cstdint>
#include <functional>
-#include <optional>
using namespace llvm;
@@ -4698,8 +4697,17 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
SDNode *ParentB, SDNode *ParentC,
SDValue A, SDValue B, SDValue C,
uint8_t Imm) {
- assert(A.isOperandOf(ParentA) && B.isOperandOf(ParentB) &&
- C.isOperandOf(ParentC) && "Incorrect parent node");
+ // Unused operand slots may be padded with an IMPLICIT_DEF (e.g. when a tree
+ // folds to a VPTERNLOG with fewer than three distinct inputs); padding has no
+ // parent node.
+ auto IsPad = [](SDValue V) {
+ return V.isMachineOpcode() &&
+ V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
+ };
+ assert((IsPad(A) || A.isOperandOf(ParentA)) &&
+ (IsPad(B) || B.isOperandOf(ParentB)) &&
+ (IsPad(C) || C.isOperandOf(ParentC)) && "Incorrect parent node");
+ (void)IsPad;
auto tryFoldLoadOrBCast =
[this](SDNode *Root, SDNode *P, SDValue &L, SDValue &Base, SDValue &Scale,
@@ -4817,7 +4825,17 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
return true;
}
-// Try to match logic trees to one or more VPTERNLOG operations.
+// Try to match a tree of bitwise logic ops to one or more VPTERNLOG ops.
+//
+// VPTERNLOG implements an arbitrary 3-input bitwise function via an 8-bit
+// truth-table immediate. We evaluate the logic sub-DAG rooted at N
+// symbolically: each of the (up to three) distinct leaf inputs is seeded with a
+// canonical truth-table column (0xF0/0xCC/0xAA) and AND/OR/XOR/ANDNP/NOT are
+// folded bitwise over those seeds, yielding the control immediate. Operand
+// reuse, inverted inputs and arbitrary nesting depth all fall out of the
+// recursion. Trees with more than three distinct leaves are split by leaving
+// one subtree "opaque" (it selects into its own VPTERNLOG when isel reaches it)
+// and folding the rest around it.
bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
MVT NVT = N->getSimpleValueType(0);
@@ -4830,422 +4848,169 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
return false;
- auto IsLogicOpcode = [](unsigned Opc) {
+ auto IsLogic = [](unsigned Opc) {
return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
Opc == X86ISD::ANDNP;
};
-
- auto IsAllOnesVec = [](SDValue V) {
- return ISD::isBuildVectorAllOnes(V.getNode());
- };
-
- auto IsAllOnesXor = [](SDValue V) {
+ auto IsNot = [](SDValue V) {
return V.getOpcode() == ISD::XOR &&
ISD::isBuildVectorAllOnes(V.getOperand(1).getNode());
};
-
- auto PeelSingleUseBitcast = [](SDValue V) {
+ auto PeelBitcast = [](SDValue V) {
if (V.getOpcode() == ISD::BITCAST && V.hasOneUse())
return V.getOperand(0);
return V;
};
-
- std::function<bool(SDValue, unsigned)> IsHomogeneousAssociativeTree =
- [&](SDValue V, unsigned Opc) {
- V = PeelSingleUseBitcast(V);
- if (V.getOpcode() != Opc)
- return true;
- if (!V.hasOneUse())
- return false;
- return IsHomogeneousAssociativeTree(V.getOperand(0), Opc) &&
- IsHomogeneousAssociativeTree(V.getOperand(1), Opc);
- };
-
auto IsLoadLike = [](SDValue V) {
return isa<LoadSDNode>(V.getNode()) ||
V.getOpcode() == X86ISD::VBROADCAST_LOAD;
};
- // Fast-path: X ^ -1 -> ~X.
- //
- // Use X for all three VPTERNLOG inputs and select an immediate that yields
- // ~X when A == B == C == X (imm bit0 = 1, bit7 = 0; other bits are don't
- // care). This avoids introducing undef register operands.
- //
- // Keep this for register-like X only. For load-like X, this can cause an
- // extra move/load before a folded-load VPTERNLOG form, which is usually not
- // profitable.
- if (N->getOpcode() == ISD::XOR) {
- for (unsigned Idx = 0; Idx != 2; ++Idx) {
- if (!IsAllOnesVec(N->getOperand(Idx)))
- continue;
-
- SDValue X = N->getOperand(Idx ^ 1);
- SDValue XNoCast = PeelSingleUseBitcast(X);
- if (IsLogicOpcode(XNoCast.getOpcode()) || IsAllOnesXor(XNoCast) ||
- IsLoadLike(XNoCast))
- continue;
-
- if (matchVPTERNLOG(N, N, N, N, X, X, X, 0x01))
- return true;
- }
- }
-
- // Avoid consuming OR into a stand-alone VPTERNLOG if it is part of a
- // higher-level A & ~(B | C) shape. Let the parent AND/ANDNP matcher absorb
- // the whole pattern instead.
- if (N->getOpcode() == ISD::OR && N->hasOneUse()) {
- SDNode *User = *N->user_begin();
- while (User->getOpcode() == ISD::BITCAST && User->hasOneUse())
- User = *User->user_begin();
-
- if (User->getOpcode() == ISD::XOR && User->hasOneUse() &&
- (ISD::isBuildVectorAllOnes(User->getOperand(0).getNode()) ||
- ISD::isBuildVectorAllOnes(User->getOperand(1).getNode()))) {
- SDNode *NextUser = *User->user_begin();
- while (NextUser->getOpcode() == ISD::BITCAST && NextUser->hasOneUse())
- NextUser = *NextUser->user_begin();
- unsigned NextOpc = NextUser->getOpcode();
- if (NextOpc == ISD::AND || NextOpc == X86ISD::ANDNP)
- return false;
- }
- }
-
- // Avoid consuming xor(logic_op, -1) (i.e. NOT of a logic sub-tree) into a
- // stand-alone VPTERNLOG when the parent is also a logic op. The parent's
- // ComputeTernlog will fold the NOT directly into the truth-table immediate,
- // producing fewer instructions overall.
- if (N->getOpcode() == ISD::XOR && N->hasOneUse() &&
- (ISD::isBuildVectorAllOnes(N->getOperand(0).getNode()) ||
- ISD::isBuildVectorAllOnes(N->getOperand(1).getNode()))) {
- SDValue Inner = ISD::isBuildVectorAllOnes(N->getOperand(1).getNode())
- ? N->getOperand(0)
- : N->getOperand(1);
- SDValue InnerNoCast = PeelSingleUseBitcast(Inner);
- if (IsLogicOpcode(InnerNoCast.getOpcode())) {
- SDNode *User = *N->user_begin();
- while (User->getOpcode() == ISD::BITCAST && User->hasOneUse())
- User = *User->user_begin();
- if (IsLogicOpcode(User->getOpcode()))
- return false;
- }
- }
-
- // Fast-path: A & ~(B | C) -> vpternlog(A, B, C, 0x10)
- if (N->getOpcode() == ISD::AND) {
- for (unsigned Idx = 0; Idx != 2; ++Idx) {
- SDValue NotSide = N->getOperand(Idx);
- SDValue A = N->getOperand(Idx ^ 1);
-
- SDValue NotSideNoCast = PeelSingleUseBitcast(NotSide);
- if (!NotSideNoCast.hasOneUse() || !IsAllOnesXor(NotSideNoCast))
- continue;
-
- SDValue Inner = PeelSingleUseBitcast(NotSideNoCast.getOperand(0));
- if (!Inner.hasOneUse() || Inner.getOpcode() != ISD::OR)
- continue;
-
- SDValue B = Inner.getOperand(0);
- SDValue C = Inner.getOperand(1);
- if (matchVPTERNLOG(N, N, Inner.getNode(), Inner.getNode(), A, B, C, 0x10))
- return true;
- }
- }
-
- struct LeafInfo {
- SDValue Leaf;
+ struct Leaf {
+ SDValue V;
SDNode *Parent;
- uint8_t Magic;
};
- auto ComputeTernlog = [&](SDValue Root, SDNode *OpaqueSubtree,
- SmallVectorImpl<LeafInfo> &Leaves, uint8_t &ImmOut,
- bool &TooManyLeaves) {
- TooManyLeaves = false;
-
- auto lookupLeaf = [&](SDValue Leaf) -> std::optional<uint8_t> {
- for (const LeafInfo &L : Leaves)
- if (L.Leaf == Leaf)
- return L.Magic;
- return std::nullopt;
- };
-
- std::function<int(SDValue, SDNode *, bool)> ComputeRec =
+ // Symbolically evaluate the tree rooted at Root. Opaque, if non-null, is
+ // treated as a leaf even when it is a logic op (used for cascading). On
+ // success fills Leaves (the distinct inputs in seed order, at most three),
+ // Imm and NumOps (the number of logic/NOT nodes folded - a profitability
+ // signal), and returns true. Returns false when more than three distinct
+ // leaves are required. Single-use is required to fold a node; bitcasts are
+ // peeled.
+ auto Evaluate = [&](SDValue Root, SDNode *Opaque,
+ SmallVectorImpl<Leaf> &Leaves, uint8_t &Imm,
+ unsigned &NumOps) -> bool {
+ static constexpr uint8_t Seeds[] = {0xF0, 0xCC, 0xAA};
+ NumOps = 0;
+ std::function<int(SDValue, SDNode *, bool)> Eval =
[&](SDValue Op, SDNode *Parent, bool IsRoot) -> int {
- if (Op.getNode() != OpaqueSubtree) {
- // Peek through single-use bitcasts.
+ if (Op.getNode() != Opaque) {
if (Op.getOpcode() == ISD::BITCAST && (IsRoot || Op.hasOneUse())) {
Parent = Op.getNode();
Op = Op.getOperand(0);
}
-
- if ((IsRoot || Op.hasOneUse()) && IsAllOnesXor(Op)) {
- int Inner = ComputeRec(Op.getOperand(0), Op.getNode(), false);
- return Inner < 0 ? -1 : ((~Inner) & 0xFF);
+ if ((IsRoot || Op.hasOneUse()) && IsNot(Op)) {
+ ++NumOps;
+ int Inner = Eval(Op.getOperand(0), Op.getNode(), false);
+ return Inner < 0 ? -1 : (~Inner & 0xFF);
}
-
- if ((IsRoot || Op.hasOneUse()) && IsLogicOpcode(Op.getOpcode())) {
- int L = ComputeRec(Op.getOperand(0), Op.getNode(), false);
- int R = ComputeRec(Op.getOperand(1), Op.getNode(), false);
+ if ((IsRoot || Op.hasOneUse()) && IsLogic(Op.getOpcode())) {
+ ++NumOps;
+ int L = Eval(Op.getOperand(0), Op.getNode(), false);
+ int R = Eval(Op.getOperand(1), Op.getNode(), false);
if (L < 0 || R < 0)
return -1;
-
switch (Op.getOpcode()) {
- default:
- llvm_unreachable("Unexpected opcode");
case ISD::AND:
- return (L & R) & 0xFF;
+ return L & R;
case ISD::OR:
- return (L | R) & 0xFF;
+ return L | R;
case ISD::XOR:
return (L ^ R) & 0xFF;
case X86ISD::ANDNP:
- return ((~L) & R) & 0xFF;
+ return ~L & R;
+ default:
+ llvm_unreachable("Checked by IsLogic");
}
}
}
-
- if (auto Existing = lookupLeaf(Op))
- return *Existing;
-
- if (Leaves.size() >= 3) {
- TooManyLeaves = true;
+ // Leaf: reuse the seed of an identical input, else allocate a new one.
+ for (unsigned I = 0, E = Leaves.size(); I != E; ++I)
+ if (Leaves[I].V == Op)
+ return Seeds[I];
+ if (Leaves.size() >= 3)
return -1;
- }
-
- static constexpr uint8_t Magics[] = {0xF0, 0xCC, 0xAA};
- uint8_t Magic = Magics[Leaves.size()];
- Leaves.push_back({Op, Parent, Magic});
- return Magic;
+ uint8_t Seed = Seeds[Leaves.size()];
+ Leaves.push_back({Op, Parent});
+ return Seed;
};
-
- int Imm = ComputeRec(Root, Root.getNode(), true);
- if (Imm < 0)
+ int Result = Eval(Root, Root.getNode(), /*IsRoot=*/true);
+ if (Result < 0)
return false;
- ImmOut = static_cast<uint8_t>(Imm & 0xFF);
+ Imm = Result & 0xFF;
return true;
};
- auto EmitFromLeaves = [&](SDNode *Root,
- const SmallVectorImpl<LeafInfo> &InLeaves,
- uint8_t Imm) {
- assert(!InLeaves.empty() && "Expected at least one leaf");
- SDValue A = InLeaves[0].Leaf;
- SDNode *ParentA = InLeaves[0].Parent;
- SDValue B = A;
- SDNode *ParentB = ParentA;
- SDValue C = A;
- SDNode *ParentC = ParentA;
-
- if (InLeaves.size() > 1) {
- B = InLeaves[1].Leaf;
- ParentB = InLeaves[1].Parent;
- }
- if (InLeaves.size() > 2) {
- C = InLeaves[2].Leaf;
- ParentC = InLeaves[2].Parent;
- }
+ // Emit one VPTERNLOG from up to three leaves. Unused operand slots are
+ // padded with undef so that, e.g., a NOT of a single memory operand folds the
+ // load with no false dependency on the other inputs.
+ auto Emit = [&](ArrayRef<Leaf> Leaves, uint8_t Imm) {
+ SDValue Undef(
+ CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SDLoc(N), NVT), 0);
+ SDValue Ops[3] = {Undef, Undef, Undef};
+ SDNode *Parents[3] = {N, N, N};
+ for (unsigned I = 0, E = Leaves.size(); I != E; ++I) {
+ Ops[I] = Leaves[I].V;
+ Parents[I] = Leaves[I].Parent;
+ }
+ return matchVPTERNLOG(N, Parents[0], Parents[1], Parents[2], Ops[0], Ops[1],
+ Ops[2], Imm);
+ };
- return matchVPTERNLOG(Root, ParentA, ParentB, ParentC, A, B, C, Imm);
+ // A cover is profitable when it folds at least two logic ops into one
+ // VPTERNLOG, or when it is a NOT of a single memory operand (which saves
+ // materializing an all-ones vector). A lone binary op (and/or/xor/andn) is
+ // left to its dedicated, cheaper instruction.
+ auto Profitable = [&](ArrayRef<Leaf> Leaves, unsigned NumOps) {
+ if (NumOps >= 2)
+ return true;
+ return Leaves.size() == 1 && IsLoadLike(PeelBitcast(Leaves[0].V));
};
- SmallVector<LeafInfo, 3> Leaves;
+ // Whole tree fits in a single VPTERNLOG (<= 3 distinct leaves).
+ SmallVector<Leaf, 3> Leaves;
uint8_t Imm = 0;
- bool TooManyLeaves = false;
- if (ComputeTernlog(SDValue(N, 0), /*OpaqueSubtree=*/nullptr, Leaves, Imm,
- TooManyLeaves)) {
- if (Leaves.empty())
+ unsigned NumOps = 0;
+ if (Evaluate(SDValue(N, 0), /*Opaque=*/nullptr, Leaves, Imm, NumOps)) {
+ if (!Profitable(Leaves, NumOps))
return false;
- // A single-leaf load folded into VPTERNLOG causes a redundant explicit
- // load (for the tied src1=dst) plus a folded load, doubling memory
- // traffic. Bail out and let default lowering handle it (e.g.
- // SETALLONES + VPXORQ mem for NOT-of-load).
- if (Leaves.size() == 1 && IsLoadLike(PeelSingleUseBitcast(Leaves[0].Leaf)))
- return false;
- return EmitFromLeaves(N, Leaves, Imm);
+ return Emit(Leaves, Imm);
}
- // Generic cascading for >3 leaves: keep one direct root operand as an opaque
- // input leaf, then fold the remaining logic around it. This allows
- // multi-level trees to be selected as chained VPTERNLOG operations.
- if (TooManyLeaves) {
- bool IsAssocOp = N->getOpcode() == ISD::OR || N->getOpcode() == ISD::AND ||
- N->getOpcode() == ISD::XOR;
- bool IsHomogeneousAssoc = IsAssocOp && IsHomogeneousAssociativeTree(
- SDValue(N, 0), N->getOpcode());
-
- // For homogeneous associative trees, prefer choosing an opaque subtree
- // from one level below the root-side logic node so we can expose two fresh
- // siblings and form 3-input VPTERNLOG combines (e.g. OR reductions as
- // repeated imm=254), rather than creating long 2-input $252/$250 chains.
- if (IsHomogeneousAssoc) {
- for (unsigned RootIdx = 0; RootIdx != 2; ++RootIdx) {
- SDValue Side = PeelSingleUseBitcast(N->getOperand(RootIdx));
- if (!Side.hasOneUse() || Side.getOpcode() != N->getOpcode())
- continue;
-
- for (unsigned ChildIdx = 0; ChildIdx != 2; ++ChildIdx) {
- SDValue Child = PeelSingleUseBitcast(Side.getOperand(ChildIdx));
- if (!Child.hasOneUse() || Child.getOpcode() != N->getOpcode())
- continue;
-
- SmallVector<LeafInfo, 3> AssocLeaves;
- uint8_t AssocImm = 0;
- bool AssocTooManyLeaves = false;
- if (!ComputeTernlog(SDValue(N, 0), Child.getNode(), AssocLeaves,
- AssocImm, AssocTooManyLeaves))
- continue;
-
- if (AssocLeaves.size() < 2)
- continue;
-
- if (EmitFromLeaves(N, AssocLeaves, AssocImm))
- return true;
- }
- }
- // Fall through to generic cascading — for balanced trees all
- // grandchildren may be leaves, so the child-as-opaque strategy below
- // can still produce a valid 3-input combine.
- }
-
- auto IsGoodOpaqueCandidate = [&](SDValue V) {
- SDValue P = PeelSingleUseBitcast(V);
- if (IsAllOnesXor(P))
- P = PeelSingleUseBitcast(P.getOperand(0));
- return IsLogicOpcode(P.getOpcode());
- };
-
- SmallVector<unsigned, 2> CandidateOrder;
- if (IsGoodOpaqueCandidate(N->getOperand(0)))
- CandidateOrder.push_back(0);
- if (IsGoodOpaqueCandidate(N->getOperand(1)))
- CandidateOrder.push_back(1);
- if (CandidateOrder.empty())
- return false;
-
- // Prefer single-use subtrees first; they are better cascading anchors.
- if (CandidateOrder.size() == 2 &&
- !N->getOperand(CandidateOrder[0]).hasOneUse() &&
- N->getOperand(CandidateOrder[1]).hasOneUse())
- std::swap(CandidateOrder[0], CandidateOrder[1]);
-
- // Collect all opaque subtree candidates: direct children and, if a direct
- // child is itself a logic op, its grandchildren. Trying grandchildren
- // allows the root VPTERNLOG to absorb more distinct operands (3 instead
- // of 2), which produces tighter cascades. Example:
- //
- // xor With child "and" opaque: 2 leaves (and, e)
- // / \ With grandchild "or" opaque: 3 leaves (or, d, e)
- // and e → saves one instruction in the cascade.
- // / \
- // or d
- //
- // Try direct children first. Only explore grandchildren when all direct
- // children produce ≤2 leaves (i.e. a degenerate 2-input fold that wastes
- // a VPTERNLOG slot) AND the opaque subtree itself has >3 leaves, meaning
- // a single VPTERNLOG cannot handle it. When the opaque child fits in one
- // VPTERNLOG (≤3 leaves), going deeper just reshuffles the split without
- // saving instructions.
- bool TriedDirect = false;
- bool NeedsGrandchild = false;
- for (unsigned Idx : CandidateOrder) {
- SmallVector<LeafInfo, 3> CascadedLeaves;
- uint8_t CascadedImm = 0;
- bool CascadedTooManyLeaves = false;
- SDNode *OpaqueSubtree = N->getOperand(Idx).getNode();
-
- if (!ComputeTernlog(SDValue(N, 0), OpaqueSubtree, CascadedLeaves,
- CascadedImm, CascadedTooManyLeaves))
- continue;
-
- if (CascadedLeaves.empty())
- continue;
-
- // If the direct child yields a 3-leaf fold, emit it right away — this
- // is already optimal for this level.
- if (CascadedLeaves.size() == 3) {
- if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
- return true;
- }
- TriedDirect = true;
-
- // Check if the opaque subtree itself has >3 leaves: if so, it cannot
- // be handled by a single VPTERNLOG, and going one level deeper may
- // help reduce the total instruction count.
- if (CascadedLeaves.size() <= 2) {
- SmallVector<LeafInfo, 3> SubLeaves;
- uint8_t SubImm = 0;
- bool SubTooMany = false;
- if (!ComputeTernlog(SDValue(N->getOperand(Idx)),
- /*OpaqueSubtree=*/nullptr, SubLeaves, SubImm,
- SubTooMany) &&
- SubTooMany)
- NeedsGrandchild = true;
- }
+ // More than three leaves: cascade. Collect the single-use logic subtrees
+ // that could be made opaque (NOTs are transparent - cutting at one would just
+ // spill it to a separate instruction).
+ SmallVector<SDNode *, 8> Candidates;
+ std::function<void(SDValue)> Collect = [&](SDValue V) {
+ V = PeelBitcast(V);
+ if (!V.hasOneUse())
+ return;
+ if (IsNot(V)) {
+ Collect(V.getOperand(0));
+ return;
}
-
- // Direct children only yielded ≤2-leaf folds and the opaque subtree has
- // >3 leaves (can't fit in one VPTERNLOG). Try grandchildren — making a
- // deeper subtree opaque exposes more leaves at the root level, reducing
- // the total instruction count.
- if (NeedsGrandchild) {
- for (unsigned Idx : CandidateOrder) {
- SDValue Child = N->getOperand(Idx);
- SDValue ChildNoCast = PeelSingleUseBitcast(Child);
- if (!ChildNoCast.hasOneUse() || !IsLogicOpcode(ChildNoCast.getOpcode()))
- continue;
-
- for (unsigned GIdx = 0; GIdx != 2; ++GIdx) {
- SDValue GChild = ChildNoCast.getOperand(GIdx);
- SDValue GChildNoCast = PeelSingleUseBitcast(GChild);
-
- // Skip NOT-wrappers (xor X, -1): ComputeTernlog already folds NOT
- // into the truth table, so cutting at a NOT boundary just pushes
- // the NOT into a separate instruction without saving anything.
- if (IsAllOnesXor(GChildNoCast))
- continue;
-
- if (!IsLogicOpcode(GChildNoCast.getOpcode()))
- continue;
-
- SmallVector<LeafInfo, 3> CascadedLeaves;
- uint8_t CascadedImm = 0;
- bool CascadedTooManyLeaves = false;
-
- if (!ComputeTernlog(SDValue(N, 0), GChild.getNode(), CascadedLeaves,
- CascadedImm, CascadedTooManyLeaves))
- continue;
-
- if (CascadedLeaves.size() < 3)
- continue;
-
- if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
- return true;
- }
- }
+ if (IsLogic(V.getOpcode())) {
+ Candidates.push_back(V.getNode());
+ Collect(V.getOperand(0));
+ Collect(V.getOperand(1));
}
-
- // Fall back to direct-child opaque with ≤2 leaves if nothing else worked.
- if (TriedDirect) {
- for (unsigned Idx : CandidateOrder) {
- SmallVector<LeafInfo, 3> CascadedLeaves;
- uint8_t CascadedImm = 0;
- bool CascadedTooManyLeaves = false;
- SDNode *OpaqueSubtree = N->getOperand(Idx).getNode();
-
- if (!ComputeTernlog(SDValue(N, 0), OpaqueSubtree, CascadedLeaves,
- CascadedImm, CascadedTooManyLeaves))
- continue;
-
- if (CascadedLeaves.empty())
- continue;
-
- if (EmitFromLeaves(N, CascadedLeaves, CascadedImm))
- return true;
- }
+ };
+ Collect(N->getOperand(0));
+ Collect(N->getOperand(1));
+
+ // Pick the cut that lets the root fold the most leaves (a tighter cascade),
+ // breaking ties towards folding more ops.
+ SmallVector<Leaf, 3> BestLeaves;
+ uint8_t BestImm = 0;
+ unsigned BestNumOps = 0;
+ for (SDNode *S : Candidates) {
+ SmallVector<Leaf, 3> CutLeaves;
+ uint8_t CutImm = 0;
+ unsigned CutNumOps = 0;
+ if (!Evaluate(SDValue(N, 0), S, CutLeaves, CutImm, CutNumOps))
+ continue;
+ if (CutLeaves.size() > BestLeaves.size() ||
+ (CutLeaves.size() == BestLeaves.size() && CutNumOps > BestNumOps)) {
+ BestLeaves = std::move(CutLeaves);
+ BestImm = CutImm;
+ BestNumOps = CutNumOps;
}
}
+ if (!BestLeaves.empty() && Profitable(BestLeaves, BestNumOps))
+ return Emit(BestLeaves, BestImm);
+
return false;
}
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index 76c87900b04d2..a8a906b69e9cb 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -350,8 +350,8 @@ define <4 x float> @ulto4f32_nneg(<4 x i64> %a) {
define <8 x double> @ulto8f64(<8 x i64> %a) {
; NODQ-LABEL: ulto8f64:
; NODQ: # %bb.0:
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm0
; NODQ-NEXT: vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; NODQ-NEXT: vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
@@ -374,17 +374,16 @@ define <8 x double> @ulto8f64(<8 x i64> %a) {
define <16 x double> @ulto16f64(<16 x i64> %a) {
; NODQ-LABEL: ulto16f64:
; NODQ: # %bb.0:
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT: vmovdqa64 %zmm3, %zmm4
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm0 & zmm2)
-; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm0
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm4
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm0 = (zmm0 & zmm3) | zmm2
; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm5 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
-; NODQ-NEXT: vporq %zmm5, %zmm0, %zmm0
+; NODQ-NEXT: vporq %zmm5, %zmm4, %zmm4
; NODQ-NEXT: vbroadcastsd {{.*#+}} zmm6 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
-; NODQ-NEXT: vsubpd %zmm6, %zmm0, %zmm0
-; NODQ-NEXT: vaddpd %zmm0, %zmm4, %zmm0
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm1 & zmm2)
+; NODQ-NEXT: vsubpd %zmm6, %zmm4, %zmm4
+; NODQ-NEXT: vaddpd %zmm4, %zmm0, %zmm0
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & zmm1) | zmm2
; NODQ-NEXT: vpsrlq $32, %zmm1, %zmm1
; NODQ-NEXT: vporq %zmm5, %zmm1, %zmm1
; NODQ-NEXT: vsubpd %zmm6, %zmm1, %zmm1
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index bdcc524545fb1..03cf284b2f26c 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -856,7 +856,7 @@ entry:
define <16 x i32> @ternlog_and_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
; ALL-LABEL: ternlog_and_andn:
; ALL: ## %bb.0:
-; ALL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm2 & zmm1 & ~zmm0
+; ALL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 & zmm2 & ~zmm0
; ALL-NEXT: retq
%a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <16 x i32> %y, %a
diff --git a/llvm/test/CodeGen/X86/avx512fp16-arith.ll b/llvm/test/CodeGen/X86/avx512fp16-arith.ll
index 9f74316f3c918..5114bcd98b236 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-arith.ll
@@ -339,9 +339,9 @@ declare half @llvm.copysign.f16(half, half)
define half @fround(half %x) {
; CHECK-LABEL: fround:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; CHECK-NEXT: vaddsh %xmm2, %xmm0, %xmm0
; CHECK-NEXT: vrndscalesh $11, %xmm0, %xmm0, %xmm0
; CHECK-NEXT: retq
@@ -394,9 +394,9 @@ declare <8 x half> @llvm.copysign.v8f16(<8 x half>, <8 x half>)
define <8 x half> @roundv8f16(<8 x half> %x) {
; CHECK-LABEL: roundv8f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; CHECK-NEXT: vaddph %xmm2, %xmm0, %xmm0
; CHECK-NEXT: vrndscaleph $11, %xmm0, %xmm0
; CHECK-NEXT: retq
@@ -449,9 +449,9 @@ declare <16 x half> @llvm.copysign.v16f16(<16 x half>, <16 x half>)
define <16 x half> @roundv16f16(<16 x half> %x) {
; CHECK-LABEL: roundv16f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ymm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm0) | ymm1
; CHECK-NEXT: vaddph %ymm2, %ymm0, %ymm0
; CHECK-NEXT: vrndscaleph $11, %ymm0, %ymm0
; CHECK-NEXT: retq
@@ -504,9 +504,9 @@ declare <32 x half> @llvm.copysign.v32f16(<32 x half>, <32 x half>)
define <32 x half> @roundv32f16(<32 x half> %x) {
; CHECK-LABEL: roundv32f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm0 & zmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm2 = (zmm2 & zmm0) | zmm1
; CHECK-NEXT: vaddph %zmm2, %zmm0, %zmm0
; CHECK-NEXT: vrndscaleph $11, %zmm0, %zmm0
; CHECK-NEXT: retq
@@ -659,7 +659,7 @@ entry:
define <8 x half> @ceil_sh(<8 x half> %x, <8 x half> %y) nounwind {
; CHECK-LABEL: ceil_sh:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vrndscalesh $10, %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vrndscalesh $10, %xmm0, %xmm1, %xmm0
; CHECK-NEXT: retq
%s = extractelement <8 x half> %x, i32 0
%call = call half @llvm.ceil.f16(half %s)
@@ -695,7 +695,7 @@ declare half @llvm.rint.f16(half %s)
define <8 x half> @nearbyint_sh(<8 x half> %x, <8 x half> %y) nounwind {
; CHECK-LABEL: nearbyint_sh:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vrndscalesh $12, %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vrndscalesh $12, %xmm0, %xmm1, %xmm0
; CHECK-NEXT: retq
%s = extractelement <8 x half> %x, i32 0
%call = call half @llvm.nearbyint.f16(half %s)
diff --git a/llvm/test/CodeGen/X86/avx512vl-logic.ll b/llvm/test/CodeGen/X86/avx512vl-logic.ll
index c1ae0e36c2c0d..8c95f219d6c48 100644
--- a/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -958,7 +958,7 @@ entry:
define <4 x i32> @ternlog_and_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_and_andn:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm2 & xmm1 & ~xmm0
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 & xmm2 & ~xmm0
; CHECK-NEXT: retq
%a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <4 x i32> %y, %a
@@ -999,14 +999,10 @@ define <4 x i32> @ternlog_and_orn_2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
ret <4 x i32> %c
}
-; FIXME: This should be a single vpternlog, but we accidentally match the xor -1
-; as the second binary op instead of the and.
define <4 x i32> @ternlog_orn_and(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_orn_and:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpand %xmm2, %xmm1, %xmm1
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 | (xmm0 ^ xmm3)
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = (xmm2 & xmm1) | ~xmm0
; CHECK-NEXT: retq
%a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <4 x i32> %y, %z
@@ -1328,7 +1324,7 @@ define <4 x i32> @ternlog_andn_or(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
define <4 x i32> @ternlog_andn_or_2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_andn_or_2:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & ~(xmm1 | xmm2)
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & ~(xmm2 | xmm1)
; CHECK-NEXT: retq
%a = or <4 x i32> %y, %z
%b = xor <4 x i32> %a, <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index f0688482fbeb8..4005e60f039cb 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -1252,7 +1252,7 @@ define i1 @trunc_v128i8_cmp(<128 x i8> %a0) nounwind {
; AVX512-LABEL: trunc_v128i8_cmp:
; AVX512: # %bb.0:
; AVX512-NEXT: vpbroadcastb {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm2 & zmm1
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm1 & zmm2
; AVX512-NEXT: vpcmpneqd %zmm2, %zmm0, %k0
; AVX512-NEXT: kortestw %k0, %k0
; AVX512-NEXT: setne %al
diff --git a/llvm/test/CodeGen/X86/clmul-vector-256.ll b/llvm/test/CodeGen/X86/clmul-vector-256.ll
index 3c0929f70560a..32b610e526988 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-256.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-256.ll
@@ -206,56 +206,56 @@ define <32 x i8> @clmul_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm3 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm4 & ymm3)
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpxor %ymm2, %ymm3, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm4, %ymm3, %ymm3
; AVX512-NEXT: vpandn %ymm1, %ymm4, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 | ymm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm3)
+; AVX512-NEXT: vpxor %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <32 x i8> @llvm.clmul.v32i8(<32 x i8> %a, <32 x i8> %b)
ret <32 x i8> %res
@@ -888,24 +888,24 @@ define <32 x i8> @clmulr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpand %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm7, %ymm5, %ymm7
; AVX512-NEXT: vpxor %ymm4, %ymm7, %ymm4
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512-NEXT: vpand %ymm7, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512-NEXT: vpand %ymm2, %ymm9, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm4 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512-NEXT: vpand %ymm2, %ymm11, %ymm12
; AVX512-NEXT: vpmullw %ymm5, %ymm12, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpand %ymm2, %ymm8, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512-NEXT: vpand %ymm2, %ymm13, %ymm2
; AVX512-NEXT: vpmullw %ymm2, %ymm5, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ ymm12 ^ ymm10
@@ -1882,24 +1882,24 @@ define <32 x i8> @clmulh_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpand %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm7, %ymm5, %ymm7
; AVX512-NEXT: vpxor %ymm4, %ymm7, %ymm4
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512-NEXT: vpand %ymm7, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512-NEXT: vpand %ymm2, %ymm9, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm4 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512-NEXT: vpand %ymm2, %ymm11, %ymm12
; AVX512-NEXT: vpmullw %ymm5, %ymm12, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpand %ymm2, %ymm8, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512-NEXT: vpand %ymm2, %ymm13, %ymm2
; AVX512-NEXT: vpmullw %ymm2, %ymm5, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ ymm12 ^ ymm10
diff --git a/llvm/test/CodeGen/X86/clmul-vector-512.ll b/llvm/test/CodeGen/X86/clmul-vector-512.ll
index f337d9177eb80..a4f592ebf2c5f 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-512.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-512.ll
@@ -19,13 +19,12 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm2, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm6 & zmm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & zmm6)
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -33,27 +32,13 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
-; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
-; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpxorq %zmm4, %zmm5, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -61,27 +46,27 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpxorq %zmm5, %zmm4, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -89,27 +74,41 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
+; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpxorq %zmm5, %zmm4, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm5
; AVX512-NEXT: vpmullw %ymm5, %ymm2, %ymm6
; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-NEXT: vpandq %zmm3, %zmm6, %zmm6
; AVX512-NEXT: vpandn %ymm1, %ymm3, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
@@ -117,7 +116,8 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm2, %ymm1
; AVX512-NEXT: vpsllw $8, %ymm1, %ymm1
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm0 | zmm6)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm6)
+; AVX512-NEXT: vpxorq %zmm0, %zmm4, %zmm0
; AVX512-NEXT: retq
%res = call <64 x i8> @llvm.clmul.v64i8(<64 x i8> %a, <64 x i8> %b)
ret <64 x i8> %res
@@ -495,78 +495,78 @@ define <64 x i8> @clmulr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
-; AVX512VL-NEXT: vpor %ymm6, %ymm5, %ymm8
+; AVX512VL-NEXT: vpor %ymm6, %ymm5, %ymm10
; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm11, %ymm5
; AVX512VL-NEXT: vpand %ymm2, %ymm5, %ymm5
; AVX512VL-NEXT: vpshufb %ymm5, %ymm3, %ymm12
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
; AVX512VL-NEXT: vpandq %ymm16, %ymm12, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm8, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm10, %ymm8
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm10
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm9 & ymm5)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; AVX512VL-NEXT: vpandq %ymm17, %ymm12, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm9, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
-; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm8, %ymm9
-; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm14
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm10 ^ (ymm14 | ymm13)
+; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm10, %ymm7
+; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm9
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpandq %ymm17, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm8, %ymm10, %ymm13
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm10, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm13)
+; AVX512VL-NEXT: vpxor %ymm9, %ymm8, %ymm13
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; AVX512VL-NEXT: vpandq %ymm18, %ymm12, %ymm10
-; AVX512VL-NEXT: vpmullw %ymm10, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm10, %ymm5, %ymm10
-; AVX512VL-NEXT: vpmaddubsw %ymm10, %ymm8, %ymm10
-; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm14 ^ (ymm15 | ymm13)
+; AVX512VL-NEXT: vpandq %ymm18, %ymm12, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm9, %ymm10, %ymm14
+; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
+; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm10, %ymm9
+; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm15
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm14)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm19, %ymm12, %ymm12
-; AVX512VL-NEXT: vpmullw %ymm12, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm10, %ymm14
; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
-; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm8, %ymm12
+; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm10, %ymm12
; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm15 ^ (ymm12 | ymm13)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm5 & ymm14)
+; AVX512VL-NEXT: vpxor %ymm12, %ymm15, %ymm12
+; AVX512VL-NEXT: vpxor %ymm12, %ymm13, %ymm14
; AVX512VL-NEXT: vpand %ymm2, %ymm11, %ymm11
-; AVX512VL-NEXT: vpshufb %ymm11, %ymm4, %ymm14
+; AVX512VL-NEXT: vpshufb %ymm11, %ymm4, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT: vpand %ymm11, %ymm14, %ymm13
-; AVX512VL-NEXT: vpmullw %ymm13, %ymm8, %ymm15
-; AVX512VL-NEXT: vpand %ymm5, %ymm15, %ymm15
+; AVX512VL-NEXT: vpand %ymm6, %ymm11, %ymm12
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm10, %ymm13
+; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
+; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm10, %ymm12
+; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm15
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm13)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpand %ymm6, %ymm12, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm13, %ymm10, %ymm7
; AVX512VL-NEXT: vpandn %ymm13, %ymm5, %ymm13
-; AVX512VL-NEXT: vpmaddubsw %ymm13, %ymm8, %ymm13
+; AVX512VL-NEXT: vpmaddubsw %ymm13, %ymm10, %ymm13
; AVX512VL-NEXT: vpsllw $8, %ymm13, %ymm13
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm12 ^ (ymm13 | ymm15)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512VL-NEXT: vpand %ymm12, %ymm14, %ymm15
-; AVX512VL-NEXT: vpmullw %ymm15, %ymm8, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm13, %ymm15, %ymm7
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpand %ymm6, %ymm13, %ymm15
+; AVX512VL-NEXT: vpmullw %ymm15, %ymm10, %ymm8
; AVX512VL-NEXT: vpandn %ymm15, %ymm5, %ymm15
-; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm8, %ymm15
+; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm10, %ymm15
; AVX512VL-NEXT: vpsllw $8, %ymm15, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm13 ^ (ymm15 | ymm6)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512VL-NEXT: vpand %ymm13, %ymm14, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm7
-; AVX512VL-NEXT: vpand %ymm5, %ymm7, %ymm7
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm15, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm14, %ymm7
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpand %ymm6, %ymm15, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm8
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm10, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm15 ^ (ymm6 | ymm7)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512VL-NEXT: vpand %ymm15, %ymm14, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm8, %ymm14
-; AVX512VL-NEXT: vpand %ymm5, %ymm14, %ymm14
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm8
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 | ymm14)
-; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm10
+; AVX512VL-NEXT: vpand %ymm2, %ymm10, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm14
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
@@ -578,66 +578,66 @@ define <64 x i8> @clmulr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
; AVX512VL-NEXT: vpandq %ymm16, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm9 & ymm5)
-; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm0, %ymm9, %ymm10
-; AVX512VL-NEXT: vpand %ymm5, %ymm10, %ymm10
-; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
-; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm0, %ymm9
-; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm9
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm7 ^ (ymm9 | ymm10)
-; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm10
-; AVX512VL-NEXT: vpand %ymm5, %ymm10, %ymm10
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm9 ^ (ymm7 | ymm10)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
; AVX512VL-NEXT: vpandq %ymm19, %ymm6, %ymm6
; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm9)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm9)
-; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm9)
-; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm9)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm1, %ymm15, %ymm1
-; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512VL-NEXT: vpandn %ymm1, %ymm5, %ymm1
; AVX512VL-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm7 ^ (ymm0 | ymm6)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm0, %ymm6, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm14, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsrlw $4, %ymm8, %ymm4
+; AVX512VL-NEXT: vpsrlw $4, %ymm10, %ymm4
; AVX512VL-NEXT: vpand %ymm2, %ymm4, %ymm4
; AVX512VL-NEXT: vpshufb %ymm4, %ymm3, %ymm4
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
@@ -840,54 +840,54 @@ define <32 x i16> @clmulr_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
; AVX512VL-NEXT: vpandq %ymm22, %ymm7, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpxor %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm24, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm11
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm25, %ymm7, %ymm12
; AVX512VL-NEXT: vpmullw %ymm6, %ymm12, %ymm12
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm11
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm8
; AVX512VL-NEXT: vpshufb %ymm8, %ymm5, %ymm8
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpandq %ymm26, %ymm8, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpandq %ymm27, %ymm8, %ymm14
; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm14
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 ^ ymm12 ^ ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm12, %ymm8, %ymm13
; AVX512VL-NEXT: vpmullw %ymm6, %ymm13, %ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512VL-NEXT: vpand %ymm13, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 ^ ymm14 ^ ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpand %ymm7, %ymm14, %ymm15
; AVX512VL-NEXT: vpmullw %ymm6, %ymm15, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
; AVX512VL-NEXT: vpand %ymm7, %ymm15, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm11
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 ^ ymm9 ^ ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
; AVX512VL-NEXT: vpandq %ymm16, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
; AVX512VL-NEXT: vpandq %ymm17, %ymm7, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 ^ ymm11 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
; AVX512VL-NEXT: vpandq %ymm19, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
; AVX512VL-NEXT: vpandq %ymm20, %ymm8, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm7 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
; AVX512VL-NEXT: vpandq %ymm21, %ymm8, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm23 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm23 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
; AVX512VL-NEXT: vpandq %ymm23, %ymm8, %ymm8
; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm6
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 ^ ymm10 ^ ymm7
@@ -1365,78 +1365,78 @@ define <64 x i8> @clmulh_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm5
; AVX512VL-NEXT: vpand %ymm2, %ymm5, %ymm5
; AVX512VL-NEXT: vpshufb %ymm5, %ymm4, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
; AVX512VL-NEXT: vpandq %ymm16, %ymm10, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm9
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm8 & ymm5)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512VL-NEXT: vpandq %ymm17, %ymm10, %ymm8
; AVX512VL-NEXT: vpmullw %ymm8, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm14, %ymm8
-; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm9 ^ (ymm12 | ymm11)
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm11)
+; AVX512VL-NEXT: vpxor %ymm9, %ymm8, %ymm11
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm18, %ymm10, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm9, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
+; AVX512VL-NEXT: vpmullw %ymm9, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm14, %ymm9
; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm13
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm12 ^ (ymm13 | ymm11)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm5 & ymm12)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpand %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpmullw %ymm10, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
+; AVX512VL-NEXT: vpmullw %ymm10, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm10, %ymm5, %ymm10
; AVX512VL-NEXT: vpmaddubsw %ymm10, %ymm14, %ymm10
-; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 | ymm11)
+; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm10
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm5 & ymm12)
+; AVX512VL-NEXT: vpxor %ymm10, %ymm13, %ymm10
+; AVX512VL-NEXT: vpxor %ymm10, %ymm11, %ymm13
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm10
; AVX512VL-NEXT: vpshufb %ymm10, %ymm3, %ymm15
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm10 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpand %ymm10, %ymm15, %ymm11
-; AVX512VL-NEXT: vpmullw %ymm11, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm11, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm11, %ymm5, %ymm11
; AVX512VL-NEXT: vpmaddubsw %ymm11, %ymm14, %ymm11
; AVX512VL-NEXT: vpsllw $8, %ymm11, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm12 ^ (ymm6 | ymm13)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm12)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpand %ymm11, %ymm15, %ymm12
-; AVX512VL-NEXT: vpmullw %ymm12, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm14, %ymm7
; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm14, %ymm12
-; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm13)
+; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm12
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm12, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512VL-NEXT: vpand %ymm12, %ymm15, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm14, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm13)
+; AVX512VL-NEXT: vpand %ymm12, %ymm15, %ymm7
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
+; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
+; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
+; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm13, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm13, %ymm15, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm15
-; AVX512VL-NEXT: vpand %ymm5, %ymm15, %ymm15
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm15)
-; AVX512VL-NEXT: vpand %ymm2, %ymm7, %ymm6
-; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
-; AVX512VL-NEXT: vpsrlw $4, %ymm7, %ymm7
-; AVX512VL-NEXT: vpand %ymm2, %ymm7, %ymm7
-; AVX512VL-NEXT: vpshufb %ymm7, %ymm4, %ymm7
-; AVX512VL-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm7
+; AVX512VL-NEXT: vpshufb %ymm7, %ymm3, %ymm7
+; AVX512VL-NEXT: vpsrlw $4, %ymm6, %ymm6
+; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
+; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
+; AVX512VL-NEXT: vpor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpsrlw $1, %ymm6, %ymm14
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm6
@@ -1450,62 +1450,62 @@ define <64 x i8> @clmulh_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
; AVX512VL-NEXT: vpandq %ymm16, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm15
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm15 & ymm5)
-; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm15
-; AVX512VL-NEXT: vpmullw %ymm0, %ymm15, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm15, %ymm5, %ymm15
-; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm0, %ymm15
-; AVX512VL-NEXT: vpsllw $8, %ymm15, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 | ymm8)
-; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm15 ^ (ymm7 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm15
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm15)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm15
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm15)
; AVX512VL-NEXT: vpand %ymm6, %ymm9, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm3, %ymm1
; AVX512VL-NEXT: vpand %ymm1, %ymm10, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm8)
-; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm8)
-; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm1
-; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512VL-NEXT: vpandn %ymm1, %ymm5, %ymm1
; AVX512VL-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm7 ^ (ymm0 | ymm6)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm0, %ymm6, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm3, %ymm1
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
@@ -1710,54 +1710,54 @@ define <32 x i16> @clmulh_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
; AVX512VL-NEXT: vpandq %ymm23, %ymm7, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpxor %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm24, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm11
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm25, %ymm7, %ymm12
; AVX512VL-NEXT: vpmullw %ymm6, %ymm12, %ymm12
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm11
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm8
; AVX512VL-NEXT: vpshufb %ymm8, %ymm4, %ymm8
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpandq %ymm26, %ymm8, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpandq %ymm27, %ymm8, %ymm14
; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm14
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 ^ ymm12 ^ ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm12, %ymm8, %ymm13
; AVX512VL-NEXT: vpmullw %ymm6, %ymm13, %ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512VL-NEXT: vpand %ymm13, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 ^ ymm14 ^ ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpand %ymm7, %ymm14, %ymm15
; AVX512VL-NEXT: vpmullw %ymm6, %ymm15, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
; AVX512VL-NEXT: vpand %ymm7, %ymm15, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm11
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 ^ ymm9 ^ ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
; AVX512VL-NEXT: vpandq %ymm16, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
; AVX512VL-NEXT: vpandq %ymm17, %ymm7, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 ^ ymm11 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
; AVX512VL-NEXT: vpandq %ymm18, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
; AVX512VL-NEXT: vpandq %ymm19, %ymm8, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm7 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
; AVX512VL-NEXT: vpandq %ymm20, %ymm8, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
; AVX512VL-NEXT: vpandq %ymm21, %ymm8, %ymm8
; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm6
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 ^ ymm10 ^ ymm7
diff --git a/llvm/test/CodeGen/X86/combine-fround.ll b/llvm/test/CodeGen/X86/combine-fround.ll
index 42dbaf234dbc7..f88055fc591e1 100644
--- a/llvm/test/CodeGen/X86/combine-fround.ll
+++ b/llvm/test/CodeGen/X86/combine-fround.ll
@@ -45,9 +45,9 @@ define <4 x double> @concat_round_v4f64_v2f64(<2 x double> %a0, <2 x double> %a1
; AVX512-LABEL: concat_round_v4f64_v2f64:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm0) | m64bcst
; AVX512-NEXT: vaddpd %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -98,9 +98,9 @@ define <8 x float> @concat_round_v8f32_v4f32(<4 x float> %a0, <4 x float> %a1) {
; AVX512-LABEL: concat_round_v8f32_v4f32:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 | (ymm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm2 = (ymm2 & ymm0) | m32bcst
; AVX512-NEXT: vaddps %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -178,9 +178,9 @@ define <8 x double> @concat_round_v8f64_v2f64(<2 x double> %a0, <2 x double> %a1
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -262,9 +262,9 @@ define <16 x float> @concat_round_v16f32_v4f32(<4 x float> %a0, <4 x float> %a1,
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -335,9 +335,9 @@ define <8 x double> @concat_round_v8f64_v4f64(<4 x double> %a0, <4 x double> %a1
; AVX512-LABEL: concat_round_v8f64_v4f64:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = (zmm2 & zmm0) | m64bcst
; AVX512-NEXT: vaddpd %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -404,9 +404,9 @@ define <16 x float> @concat_round_v16f32_v8f32(<8 x float> %a0, <8 x float> %a1)
; AVX512-LABEL: concat_round_v16f32_v8f32:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm2 = zmm2 | (zmm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm2 = (zmm2 & zmm0) | m32bcst
; AVX512-NEXT: vaddps %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/combine-or-shuffle.ll b/llvm/test/CodeGen/X86/combine-or-shuffle.ll
index 38ea796c0fcb0..2e39cf6e2fe4f 100644
--- a/llvm/test/CodeGen/X86/combine-or-shuffle.ll
+++ b/llvm/test/CodeGen/X86/combine-or-shuffle.ll
@@ -802,8 +802,8 @@ define <2 x i64> @or_and_v2i64(<2 x i64> %a0) {
;
; AVX512-LABEL: or_and_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm1 = [7,7]
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 & (xmm0 | m64bcst)
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst & (xmm0 | xmm1)
; AVX512-NEXT: retq
%1 = and <2 x i64> %a0, <i64 7, i64 7>
%2 = or <2 x i64> %1, <i64 3, i64 3>
@@ -831,8 +831,8 @@ define <4 x i32> @or_and_v4i32(<4 x i32> %a0) {
;
; AVX512-LABEL: or_and_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [3,3,15,7]
-; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 | mem)
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [3,2,4294967295,2]
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = mem & (xmm0 | xmm1)
; AVX512-NEXT: retq
%1 = and <4 x i32> %a0, <i32 1, i32 3, i32 5, i32 7>
%2 = or <4 x i32> %1, <i32 3, i32 2, i32 15, i32 2>
diff --git a/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll b/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
index c3822b3f3021f..5ac83ca7a4f13 100644
--- a/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
+++ b/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
@@ -651,8 +651,7 @@ define void @vec256_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec256_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovdqa %xmm0, 16(%rsi)
; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
; AVX512-NEXT: retq
@@ -1092,8 +1091,7 @@ define void @vec384_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec384_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
; AVX512-NEXT: vmovdqa %xmm0, 16(%rsi)
; AVX512-NEXT: vmovdqa %xmm0, 32(%rsi)
@@ -1658,8 +1656,7 @@ define void @vec512_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec512_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
; AVX512-NEXT: vzeroupper
@@ -1731,8 +1728,7 @@ define void @vec512_i256(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec512_i256:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX512-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512-NEXT: vmovdqa %ymm0, 32(%rsi)
; AVX512-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 8abd68701676c..4f343bdd141ab 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -1970,7 +1970,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vcvtph2ps %xmm2, %xmm4
; AVX512BF16-NEXT: vmaxss %xmm3, %xmm4, %xmm5
; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 & (xmm4 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 & (xmm2 | xmm4)
; AVX512BF16-NEXT: vcmpunordss %xmm3, %xmm3, %k1
; AVX512BF16-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm5, %xmm3
@@ -1979,7 +1979,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vshufps {{.*#+}} xmm5 = xmm0[3,3,3,3]
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -1989,7 +1989,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -1998,7 +1998,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vshufpd {{.*#+}} xmm6 = xmm0[1,0]
; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2009,7 +2009,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrlq $48, %xmm0, %xmm5
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -2018,7 +2018,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2026,7 +2026,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vcvtph2ps %xmm1, %xmm5
; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2035,7 +2035,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm0
; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm0 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm0)
; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512BF16-NEXT: vmovss %xmm0, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm0
diff --git a/llvm/test/CodeGen/X86/fp-round.ll b/llvm/test/CodeGen/X86/fp-round.ll
index 8595b63fc8107..ea9179f6cc61e 100644
--- a/llvm/test/CodeGen/X86/fp-round.ll
+++ b/llvm/test/CodeGen/X86/fp-round.ll
@@ -51,8 +51,8 @@ define half @round_f16(half %h) {
; AVX512F-LABEL: round_f16:
; AVX512F: # %bb.0: # %entry
; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vcvtps2ph $4, %xmm0, %xmm0
@@ -60,9 +60,9 @@ define half @round_f16(half %h) {
;
; AVX512FP16-LABEL: round_f16:
; AVX512FP16: ## %bb.0: ## %entry
-; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; AVX512FP16-NEXT: vaddsh %xmm2, %xmm0, %xmm0
; AVX512FP16-NEXT: vrndscalesh $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -97,16 +97,16 @@ define float @round_f32(float %x) {
;
; AVX512F-LABEL: round_f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512FP16-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -141,16 +141,16 @@ define double @round_f64(double %x) {
;
; AVX512F-LABEL: round_f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512F-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundsd $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512FP16-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundsd $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -207,16 +207,16 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
;
; AVX512F-LABEL: round_v4f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundps $11, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v4f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512FP16-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundps $11, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -261,16 +261,16 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
;
; AVX512F-LABEL: round_v2f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512F-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundpd $11, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v2f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundpd $11, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -355,16 +355,16 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
;
; AVX512F-LABEL: round_v8f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm1 = (ymm1 & ymm0) | m32bcst
; AVX512F-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v8f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} ymm1 = (ymm1 & ymm0) | m32bcst
; AVX512FP16-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512FP16-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512FP16-NEXT: retq
@@ -425,16 +425,16 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
;
; AVX512F-LABEL: round_v4f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ymm0) | m64bcst
; AVX512F-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v4f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ymm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512FP16-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512FP16-NEXT: retq
@@ -581,16 +581,16 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
;
; AVX512F-LABEL: round_v16f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512F-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512F-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v16f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512FP16-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512FP16-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512FP16-NEXT: retq
@@ -689,16 +689,16 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
;
; AVX512F-LABEL: round_v8f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512F-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512F-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v8f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512FP16-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512FP16-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/fshl-splat-undef.ll b/llvm/test/CodeGen/X86/fshl-splat-undef.ll
index a2d0345077b87..09fc9d48bbd10 100644
--- a/llvm/test/CodeGen/X86/fshl-splat-undef.ll
+++ b/llvm/test/CodeGen/X86/fshl-splat-undef.ll
@@ -21,10 +21,10 @@ define void @test_fshl(<8 x i64> %lo, <8 x i64> %hi, ptr %arr) {
; CHECK-LABEL: test_fshl:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT: vpsllq $12, %zmm1, %zmm1
; CHECK-NEXT: vpsrlq $52, %zmm0, %zmm0
-; CHECK-NEXT: vpternlogq $168, {{\.?LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm0
-; CHECK-NEXT: vmovdqa64 %zmm0, (%eax)
+; CHECK-NEXT: vpsllq $12, %zmm1, %zmm1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm1 = mem & (zmm1 | zmm0)
+; CHECK-NEXT: vmovdqa64 %zmm1, (%eax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retl
entry:
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index 638d88481f071..a3b7b37f7a860 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -1187,9 +1187,9 @@ define <32 x i8> @splatvar_ashr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; GFNIAVX512VL-LABEL: splatvar_ashr_v32i8:
; GFNIAVX512VL: # %bb.0:
; GFNIAVX512VL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; GFNIAVX512VL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; GFNIAVX512VL-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -2482,29 +2482,29 @@ define <64 x i8> @splatvar_ashr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; GFNIAVX512VL-LABEL: splatvar_ashr_v64i8:
; GFNIAVX512VL: # %bb.0:
-; GFNIAVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; GFNIAVX512VL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm3 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
+; GFNIAVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm3, %ymm3
; GFNIAVX512VL-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpsrlw $8, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpbroadcastb %xmm4, %ymm4
-; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 & ymm4)
-; GFNIAVX512VL-NEXT: vpsubb %ymm3, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & ymm4)
+; GFNIAVX512VL-NEXT: vpsubb %ymm2, %ymm3, %ymm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & ymm4)
-; GFNIAVX512VL-NEXT: vpsubb %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & ymm4)
+; GFNIAVX512VL-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; GFNIAVX512VL-NEXT: retq
;
; GFNIAVX512BW-LABEL: splatvar_ashr_v64i8:
; GFNIAVX512BW: # %bb.0:
; GFNIAVX512BW-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; GFNIAVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
index dada1726be424..3d2538b485ff5 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
@@ -149,9 +149,9 @@ define <8 x i1> @andnot_ne_v8i16_todo_no_splat(<8 x i16> %x) nounwind {
; AVX512-LABEL: andnot_ne_v8i16_todo_no_splat:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16_todo_no_splat:
@@ -184,7 +184,7 @@ define <8 x i1> @andnot_ne_v8i16(<8 x i16> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16:
@@ -215,9 +215,9 @@ define <16 x i1> @andnot_ne_v16i8_fail_max_not_n1(<16 x i8> %x) nounwind {
; AVX512-LABEL: andnot_ne_v16i8_fail_max_not_n1:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8_fail_max_not_n1:
@@ -250,7 +250,7 @@ define <16 x i1> @andnot_ne_v16i8(<16 x i8> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8:
@@ -307,9 +307,8 @@ define <8 x i1> @addand_ne_v8i16_fail(<8 x i16> %x) nounwind {
; AVX512-LABEL: addand_ne_v8i16_fail:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $86, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: addand_ne_v8i16_fail:
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index aa2dd00237b07..35e9c63db77c0 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -894,12 +894,12 @@ define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm3, %ymm5, %ymm3
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm1, %ymm1
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm5 & ymm4)
; CHECK-SKX-NOVBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm3
; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm2, %ymm5, %ymm2
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm0, %ymm0
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm3)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-SKX-NOVBMI-NEXT: vzeroupper
@@ -937,12 +937,12 @@ define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-AVX512-NEXT: vpandn %ymm3, %ymm5, %ymm3
; CHECK-AVX512-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpsllw $8, %ymm1, %ymm1
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm5 & ymm4)
; CHECK-AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm3
; CHECK-AVX512-NEXT: vpandn %ymm2, %ymm5, %ymm2
; CHECK-AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm3)
; CHECK-AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-AVX512-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-AVX512-NEXT: vzeroupper
@@ -985,7 +985,7 @@ define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-SKX-NOVBMI-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %zmm0, %zmm0
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT: vzeroupper
; CHECK-SKX-NOVBMI-NEXT: retq
@@ -1012,7 +1012,7 @@ define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-AVX512-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; CHECK-AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vpsllw $8, %zmm0, %zmm0
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; CHECK-AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 625d18bcf4edf..8fbef0d45c41f 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -956,7 +956,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind {
; AVX512F-NEXT: vpmaddubsw %ymm1, %ymm3, %ymm1
; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm4 & zmm5)
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm5 & zmm4)
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: mul_v64i8:
@@ -966,7 +966,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind {
; AVX512BW-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; AVX512BW-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0
-; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; AVX512BW-NEXT: retq
entry:
%A = mul <64 x i8> %i, %j
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll b/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
index c9e48f817fb44..a8bf9f0190fed 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
@@ -63,7 +63,7 @@ define <32 x i8> @test_mul_32i8(<32 x i8> %a, <32 x i8> %b) {
; AVX256BW-NEXT: vpandn %ymm1, %ymm3, %ymm1
; AVX256BW-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX256BW-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX256BW-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm3)
+; AVX256BW-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm2)
; AVX256BW-NEXT: retq
;
; AVX512BWVL-LABEL: test_mul_32i8:
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index 69e6ff7770ebe..9984532b94224 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -874,12 +874,11 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
;
; AVX512-LABEL: unsigned_sat_variable_v16i8_using_cmp_notval:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm3
+; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~xmm1
; AVX512-NEXT: vpminub %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm3 | (xmm0 ^ xmm2)
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 | ~xmm0
; AVX512-NEXT: retq
%noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
%a = add <16 x i8> %x, %y
@@ -981,12 +980,11 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
;
; AVX512-LABEL: unsigned_sat_variable_v8i16_using_cmp_notval:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm3
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~xmm1
; AVX512-NEXT: vpminuw %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm3 | (xmm0 ^ xmm2)
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 | ~xmm0
; AVX512-NEXT: retq
%noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
%a = add <8 x i16> %x, %y
diff --git a/llvm/test/CodeGen/X86/select-big-integer.ll b/llvm/test/CodeGen/X86/select-big-integer.ll
index b7f4a57b01338..72c5e4725b419 100644
--- a/llvm/test/CodeGen/X86/select-big-integer.ll
+++ b/llvm/test/CodeGen/X86/select-big-integer.ll
@@ -71,8 +71,7 @@ define void @test_not_i128(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i128:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor (%rsi), %xmm0, %xmm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512VL-NEXT: negb %dl
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqa32 (%rdi), %xmm0 {%k1}
@@ -166,8 +165,7 @@ define void @test_not_i256(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX512VL-NEXT: vpxor (%rsi), %ymm0, %ymm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
; AVX512VL-NEXT: negb %dl
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqu32 (%rdi), %ymm0 {%k1}
@@ -284,8 +282,7 @@ define void @test_not_i512(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512F-LABEL: test_not_i512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = -1
-; AVX512F-NEXT: vpxord (%rsi), %zmm0, %zmm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = ~mem
; AVX512F-NEXT: negl %edx
; AVX512F-NEXT: kmovw %edx, %k1
; AVX512F-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1}
@@ -294,8 +291,7 @@ define void @test_not_i512(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i512:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = -1
-; AVX512VL-NEXT: vpxord (%rsi), %zmm0, %zmm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = ~mem
; AVX512VL-NEXT: negl %edx
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1}
diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
index 4e343af8e824d..87d0da008e0dd 100644
--- a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
@@ -2474,14 +2474,12 @@ define <32 x i1> @pr51133(<32 x i8> %x, <32 x i8> %y) {
; CHECK-AVX512VL-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
; CHECK-AVX512VL-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm3
; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm2, %ymm2
-; CHECK-AVX512VL-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
-; CHECK-AVX512VL-NEXT: vpandn %ymm3, %ymm2, %ymm2
; CHECK-AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; CHECK-AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; CHECK-AVX512VL-NEXT: vpcmpgtb %ymm4, %ymm0, %ymm0
-; CHECK-AVX512VL-NEXT: vpandn %ymm0, %ymm3, %ymm3
-; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm1, %ymm0
-; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ~ymm0 & (ymm2 | ymm3)
+; CHECK-AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; CHECK-AVX512VL-NEXT: vpcmpgtb %ymm3, %ymm0, %ymm0
+; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = (ymm2 & mem) ^ (ymm0 | mem)
+; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm1, %ymm1
+; CHECK-AVX512VL-NEXT: vpandn %ymm0, %ymm1, %ymm0
; CHECK-AVX512VL-NEXT: retq
%rem = srem <32 x i8> %x, <i8 13, i8 5, i8 19, i8 34, i8 2, i8 8, i8 2, i8 88, i8 62, i8 62, i8 5, i8 7, i8 97, i8 2, i8 3, i8 60, i8 3, i8 87, i8 7, i8 6, i8 84, i8 -128, i8 127, i8 56, i8 114, i8 1, i8 50, i8 7, i8 2, i8 8, i8 97, i8 117>
%cmp = icmp ne <32 x i8> %rem, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
index 21915a0d50a77..9c58016fbd2b1 100644
--- a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
+++ b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
@@ -252,14 +252,22 @@ define void @vec128_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -319,14 +327,22 @@ define void @vec128_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -519,14 +535,22 @@ define void @vec128_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -822,15 +846,24 @@ define void @vec256_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -916,15 +949,24 @@ define void @vec256_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1124,14 +1166,31 @@ define void @vec256_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1166,14 +1225,31 @@ define void @vec256_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -1254,15 +1330,24 @@ define void @vec256_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1394,14 +1479,31 @@ define void @vec256_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1431,14 +1533,31 @@ define void @vec256_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -1626,14 +1745,31 @@ define void @vec256_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1793,14 +1929,31 @@ define void @vec256_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1946,16 +2099,26 @@ define void @vec384_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2067,16 +2230,26 @@ define void @vec384_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2311,15 +2484,34 @@ define void @vec384_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2359,15 +2551,34 @@ define void @vec384_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -3150,21 +3361,52 @@ define void @vec384_v3i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v3i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v3i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v3i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
store <3 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3281,24 +3523,55 @@ define void @vec384_v3f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v3f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
- %in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
- %in.subvec.int = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
- %in.subvec = bitcast <3 x i32> %in.subvec.int to <3 x float>
+; AVX1-LABEL: vec384_v3f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v3f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
+ %in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
+ %in.subvec.int = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
+ %in.subvec = bitcast <3 x i32> %in.subvec.int to <3 x float>
store <3 x float> %in.subvec, ptr %out.subvec.ptr, align 64
%out.subvec0.ptr = getelementptr <3 x float>, ptr %out.vec.ptr, i64 0
store <3 x float> %in.subvec, ptr %out.subvec0.ptr, align 64
@@ -3360,19 +3633,32 @@ define void @vec384_v3i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v3i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v3i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <3 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1>
store <3 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3432,19 +3718,32 @@ define void @vec384_v3f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v3f64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v3f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <3 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1>
%in.subvec = bitcast <3 x i64> %in.subvec.int to <3 x double>
@@ -3543,16 +3842,26 @@ define void @vec384_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3710,15 +4019,34 @@ define void @vec384_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3752,15 +4080,34 @@ define void @vec384_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -4127,21 +4474,52 @@ define void @vec384_v6i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v6i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v6i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v6i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <6 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <6 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4205,19 +4583,32 @@ define void @vec384_v6i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v6i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v6i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <6 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
store <6 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4277,19 +4668,32 @@ define void @vec384_v6f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v6f32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v6f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <6 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <6 x i32> %in.subvec.int to <6 x float>
@@ -4510,15 +4914,34 @@ define void @vec384_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4633,21 +5056,52 @@ define void @vec384_v12i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v12i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v12i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v12i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v12i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <12 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <12 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <12 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4711,19 +5165,32 @@ define void @vec384_v12i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v12i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v12i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v12i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <12 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <12 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <12 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4901,15 +5368,34 @@ define void @vec384_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4971,19 +5457,32 @@ define void @vec384_v24i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v24i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v24i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v24i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <24 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <24 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <24 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5163,8 +5662,7 @@ define void @vec512_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512F-LABEL: vec512_v2i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsi)
; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
@@ -5174,8 +5672,7 @@ define void @vec512_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512BW-LABEL: vec512_v2i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512BW-NEXT: vpextrw $0, %xmm0, (%rsi)
; AVX512BW-NEXT: vpbroadcastw %xmm0, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -5331,8 +5828,7 @@ define void @vec512_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
;
; AVX512-LABEL: vec512_v2i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovd %xmm0, (%rsi)
; AVX512-NEXT: vpbroadcastd %xmm0, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -5599,16 +6095,37 @@ define void @vec512_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5653,16 +6170,37 @@ define void @vec512_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -5717,18 +6255,43 @@ define void @vec512_v2i128(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; SSE2-NEXT: movdqa %xmm1, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2i128:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm1
-; AVX-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, 16(%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: vmovdqa %xmm1, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2i128:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm1
+; AVX1-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rsi)
+; AVX1-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: vmovdqa %xmm1, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2i128:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm1
+; AVX2-ONLY-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2i128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~mem
+; AVX512-NEXT: vmovdqa %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i128>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i128> %in.subvec.not, <i128 -1, i128 -1>
store <2 x i128> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5857,8 +6420,7 @@ define void @vec512_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512-LABEL: vec512_v4i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovd %xmm0, (%rsi)
; AVX512-NEXT: vpbroadcastd %xmm0, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -6043,16 +6605,37 @@ define void @vec512_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6090,16 +6673,37 @@ define void @vec512_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -6165,15 +6769,24 @@ define void @vec512_v4i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v4i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v4i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1, i64 -1>
store <4 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6234,15 +6847,24 @@ define void @vec512_v4f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v4f64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v4f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1, i64 -1>
%in.subvec = bitcast <4 x i64> %in.subvec.int to <4 x double>
@@ -6492,16 +7114,37 @@ define void @vec512_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6588,15 +7231,24 @@ define void @vec512_v8i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v8i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
store <8 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6679,15 +7331,24 @@ define void @vec512_v8f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v8f32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v8f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <8 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <8 x i32> %in.subvec.int to <8 x float>
@@ -6898,16 +7559,37 @@ define void @vec512_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7078,15 +7760,24 @@ define void @vec512_v16i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v16i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v16i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v16i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <16 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7393,15 +8084,24 @@ define void @vec512_v32i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v32i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v32i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <32 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <32 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <32 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7412,4 +8112,6 @@ define void @vec512_v32i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; AVX2: {{.*}}
; SSSE3: {{.*}}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
index e5b19a5efe6f2..9a5c7def87e81 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
@@ -118,9 +118,8 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
; CHECK-AVX512VL: # %bb.0:
; CHECK-AVX512VL-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; CHECK-AVX512VL-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst | (xmm0 ^ xmm1)
+; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst | ~xmm0
; CHECK-AVX512VL-NEXT: retq
%urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
%cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 5804cb782bfbd..17f4bacf00472 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1693,7 +1693,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
; AVX512VL-NEXT: vpsllw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [128,64,32,16,8,4,2,1,128,1,2,4,8,16,32,64,128,64,32,16,8,4,2,1,128,1,2,4,8,16,32,64]
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | ymm1 | ymm2
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | ymm2 | ymm1
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: constant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 4257bcc0e3f99..37e13fad829ea 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -923,7 +923,7 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
; AVX512F-NEXT: vpmullw %ymm4, %ymm2, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
; AVX512F-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm3
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm1
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: constant_funnnel_v64i8:
@@ -965,7 +965,7 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
; AVX512VL-NEXT: vpmullw %ymm4, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm3
+; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm1
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: constant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
index dc8a9ed4a4ccc..3b18d09277afa 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
@@ -2613,7 +2613,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm2
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm11)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm11 & ymm2)
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm6
; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
@@ -2837,7 +2837,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm11)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm11 & ymm2)
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm6
; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
@@ -5490,7 +5490,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm7, %xmm7
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3],xmm9[4],xmm7[5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm7 & ymm11)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm11 & ymm7)
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm4
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm1, %xmm1
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3],xmm4[4],xmm1[5,6,7]
@@ -6039,7 +6039,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm8, %xmm8
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1],xmm8[2,3],xmm9[4],xmm8[5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm8 & ymm13)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm13 & ymm8)
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm1, %xmm1
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6,7]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index 763b8a67edaf7..108a324fb4802 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -3782,7 +3782,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -3798,7 +3798,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -3815,7 +3815,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512-NEXT: vmovdqa %ymm0, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -3829,7 +3829,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -3892,7 +3892,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -3908,7 +3908,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512-FCP-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -3925,7 +3925,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm12
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -3939,7 +3939,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512-FCP-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -4002,7 +4002,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -4018,7 +4018,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512DQ-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -4035,7 +4035,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm12
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -4049,7 +4049,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512DQ-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -4112,7 +4112,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -4128,7 +4128,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -4145,7 +4145,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm12
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -4159,7 +4159,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -7471,7 +7471,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7482,7 +7482,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -7539,7 +7539,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -7556,7 +7556,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -7725,7 +7725,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7736,7 +7736,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512-FCP-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -7793,7 +7793,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -7810,7 +7810,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -7979,7 +7979,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512DQ-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7990,7 +7990,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512DQ-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -8047,7 +8047,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512DQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512DQ-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -8064,7 +8064,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -8233,7 +8233,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -8244,7 +8244,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -8301,7 +8301,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -8318,7 +8318,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 09d00795e4cc9..2cc0087a80e28 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -5690,7 +5690,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0,1,2],ymm10[3],ymm15[4,5],ymm10[6],ymm15[7,8,9,10],ymm10[11],ymm15[12,13],ymm10[14],ymm15[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm8 & ymm17)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm8)
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm3 ^ (ymm14 & (ymm2 ^ ymm3))
; AVX512-NEXT: vextracti128 $1, %ymm14, %xmm8
; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u],zero,zero,zero,xmm8[5,12],zero,zero,xmm8[1,8,15,u,u,u,u]
@@ -5729,7 +5729,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm1 ^ ymm9))
; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2],ymm10[3],ymm14[4,5,6],ymm10[7,8],ymm14[9,10],ymm10[11],ymm14[12,13,14],ymm10[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm17)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm8)
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm14[0],ymm12[1,2,3,4,5,6,7],ymm14[8],ymm12[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm8, %ymm21
@@ -5755,7 +5755,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm1 ^ ymm9))
; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2,3],ymm10[4],ymm14[5,6],ymm10[7,8],ymm14[9,10,11],ymm10[12],ymm14[13,14],ymm10[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm17)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm8)
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm14[0],ymm12[1,2,3,4,5,6,7],ymm14[8],ymm12[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa %ymm11, %ymm8
@@ -5780,7 +5780,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm9 ^ ymm1))
; AVX512-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm10[1],ymm15[2,3],ymm10[4],ymm15[5,6,7,8],ymm10[9],ymm15[10,11],ymm10[12],ymm15[13,14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm8 & ymm17)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm8)
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm13 = ymm2 ^ (ymm13 & (ymm3 ^ ymm2))
@@ -5802,7 +5802,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm9 ^ ymm1))
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm10[1],ymm0[2,3,4],ymm10[5],ymm0[6,7,8],ymm10[9],ymm0[10,11,12],ymm10[13],ymm0[14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -5890,7 +5890,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm14 = ymm14[0,1,2],ymm8[3],ymm14[4,5],ymm8[6],ymm14[7,8,9,10],ymm8[11],ymm14[12,13],ymm8[14],ymm14[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm3 ^ (ymm13 & (ymm2 ^ ymm3))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm13, %xmm6
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u,u,u]
@@ -5929,7 +5929,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2],ymm8[3],ymm15[4,5,6],ymm8[7,8],ymm15[9,10],ymm8[11],ymm15[12,13,14],ymm8[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm20
@@ -5955,7 +5955,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2,3],ymm8[4],ymm15[5,6],ymm8[7,8],ymm15[9,10,11],ymm8[12],ymm15[13,14],ymm8[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm21
@@ -5981,7 +5981,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm7 ^ ymm1))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm8[1],ymm15[2,3],ymm8[4],ymm15[5,6,7,8],ymm8[9],ymm15[10,11],ymm8[12],ymm15[13,14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm2 ^ (ymm11 & (ymm3 ^ ymm2))
@@ -6003,7 +6003,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm7 ^ ymm1))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm8[1],ymm0[2,3,4],ymm8[5],ymm0[6,7,8],ymm8[9],ymm0[10,11,12],ymm8[13],ymm0[14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -6097,7 +6097,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0,1,2],ymm10[3],ymm15[4,5],ymm10[6],ymm15[7,8,9,10],ymm10[11],ymm15[12,13],ymm10[14],ymm15[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm8 & ymm17)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm8)
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm3 ^ (ymm14 & (ymm2 ^ ymm3))
; AVX512DQ-NEXT: vextracti128 $1, %ymm14, %xmm8
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u],zero,zero,zero,xmm8[5,12],zero,zero,xmm8[1,8,15,u,u,u,u]
@@ -6136,7 +6136,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm1 ^ ymm9))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2],ymm10[3],ymm14[4,5,6],ymm10[7,8],ymm14[9,10],ymm10[11],ymm14[12,13,14],ymm10[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm17)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm8)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm14[0],ymm12[1,2,3,4,5,6,7],ymm14[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm8, %ymm21
@@ -6162,7 +6162,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm1 ^ ymm9))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2,3],ymm10[4],ymm14[5,6],ymm10[7,8],ymm14[9,10,11],ymm10[12],ymm14[13,14],ymm10[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm17)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm8)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm14[0],ymm12[1,2,3,4,5,6,7],ymm14[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa %ymm11, %ymm8
@@ -6187,7 +6187,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm9 ^ ymm1))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm10[1],ymm15[2,3],ymm10[4],ymm15[5,6,7,8],ymm10[9],ymm15[10,11],ymm10[12],ymm15[13,14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm8 & ymm17)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm8)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm13 = ymm2 ^ (ymm13 & (ymm3 ^ ymm2))
@@ -6209,7 +6209,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm9 ^ ymm1))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm10[1],ymm0[2,3,4],ymm10[5],ymm0[6,7,8],ymm10[9],ymm0[10,11,12],ymm10[13],ymm0[14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -6297,7 +6297,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm14 = ymm14[0,1,2],ymm8[3],ymm14[4,5],ymm8[6],ymm14[7,8,9,10],ymm8[11],ymm14[12,13],ymm8[14],ymm14[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm3 ^ (ymm13 & (ymm2 ^ ymm3))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm13, %xmm6
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u,u,u]
@@ -6336,7 +6336,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2],ymm8[3],ymm15[4,5,6],ymm8[7,8],ymm15[9,10],ymm8[11],ymm15[12,13,14],ymm8[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm20
@@ -6362,7 +6362,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2,3],ymm8[4],ymm15[5,6],ymm8[7,8],ymm15[9,10,11],ymm8[12],ymm15[13,14],ymm8[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm21
@@ -6388,7 +6388,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm7 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm8[1],ymm15[2,3],ymm8[4],ymm15[5,6,7,8],ymm8[9],ymm15[10,11],ymm8[12],ymm15[13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm13[1,2,3,4,5,6,7],ymm15[8],ymm13[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm2 ^ (ymm11 & (ymm3 ^ ymm2))
@@ -6410,7 +6410,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm7 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm8[1],ymm0[2,3,4],ymm8[5],ymm0[6,7,8],ymm8[9],ymm0[10,11,12],ymm8[13],ymm0[14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -11795,7 +11795,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm11[3],ymm2[4,5],ymm11[6],ymm2[7,8,9,10],ymm11[11],ymm2[12,13],ymm11[14],ymm2[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpmovsxdq {{.*#+}} ymm15 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ymm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm15 & ymm0)
; AVX512-NEXT: vmovdqa %ymm5, %ymm0
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm29 ^ (ymm0 & (ymm21 ^ ymm29))
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
@@ -11831,7 +11831,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm19 ^ (ymm2 & (ymm31 ^ ymm19))
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0],ymm2[1,2],ymm11[3],ymm2[4,5,6],ymm11[7,8],ymm2[9,10],ymm11[11],ymm2[12,13,14],ymm11[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm0 & ymm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm15 & ymm0)
; AVX512-NEXT: vmovdqa %ymm15, %ymm11
; AVX512-NEXT: vmovdqa %ymm7, %ymm0
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm29 ^ (ymm0 & (ymm21 ^ ymm29))
@@ -12004,13 +12004,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm8[0],ymm3[1,2,3],ymm8[4],ymm3[5,6],ymm8[7,8],ymm3[9,10,11],ymm8[12],ymm3[13,14],ymm8[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vmovdqa %ymm11, %ymm1
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm0 & ymm11)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm11 & ymm0)
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5,6,7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13,14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm2 & ymm1)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm1 & ymm2)
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm8[1],ymm14[2,3,4],ymm8[5],ymm14[6,7,8],ymm8[9],ymm14[10,11,12],ymm8[13],ymm14[14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm1)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm1 & ymm6)
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm29 ^ (ymm12 & (ymm21 ^ ymm29))
; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm12[u,u,2,9],zero,zero,zero,xmm12[5,12],zero,zero,xmm12[u,u,u,u,u]
; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm2
@@ -12243,7 +12243,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0,1,2],ymm6[3],ymm8[4,5],ymm6[6],ymm8[7,8,9,10],ymm6[11],ymm8[12,13],ymm6[14],ymm8[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} ymm25 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm25 & ymm7)
; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm7
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm30 ^ (ymm7 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm7, %xmm13
@@ -12293,7 +12293,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm31 ^ (ymm7 & (ymm27 ^ ymm31))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0],ymm7[1,2],ymm6[3],ymm7[4,5,6],ymm6[7,8],ymm7[9,10],ymm6[11],ymm7[12,13,14],ymm6[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm3 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm3)
; AVX512-FCP-NEXT: vmovdqa %ymm1, %ymm3
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm30 ^ (ymm3 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm8
@@ -12406,13 +12406,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpor %xmm4, %xmm10, %xmm4
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0],ymm7[1,2,3],ymm6[4],ymm7[5,6],ymm6[7,8],ymm7[9,10,11],ymm6[12],ymm7[13,14],ymm6[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm3 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm25 & ymm3)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0],ymm6[1],ymm13[2,3],ymm6[4],ymm13[5,6,7,8],ymm6[9],ymm13[10,11],ymm6[12],ymm13[13,14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm0 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm0)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm6[1],ymm9[2,3,4],ymm6[5],ymm9[6,7,8],ymm6[9],ymm9[10,11,12],ymm6[13],ymm9[14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm4 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm25 & ymm4)
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm14 ^ (ymm1 & (ymm2 ^ ymm14))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm30 ^ (ymm8 & (ymm28 ^ ymm30))
@@ -12618,7 +12618,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm11[3],ymm2[4,5],ymm11[6],ymm2[7,8,9,10],ymm11[11],ymm2[12,13],ymm11[14],ymm2[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vpmovsxdq {{.*#+}} ymm27 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ymm27)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm27 & ymm0)
; AVX512DQ-NEXT: vmovdqa64 %ymm29, %ymm0
; AVX512DQ-NEXT: vmovdqa64 %ymm25, %ymm15
; AVX512DQ-NEXT: vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12657,7 +12657,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm28 ^ (ymm2 & (ymm31 ^ ymm28))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0],ymm2[1,2],ymm11[3],ymm2[4,5,6],ymm11[7,8],ymm2[9,10],ymm11[11],ymm2[12,13,14],ymm11[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm0 & ymm27)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm27 & ymm0)
; AVX512DQ-NEXT: vmovdqa64 %ymm27, %ymm11
; AVX512DQ-NEXT: vmovdqa %ymm7, %ymm0
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm19 ^ (ymm0 & (ymm15 ^ ymm19))
@@ -12829,13 +12829,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1,2,3],ymm4[4],ymm2[5,6],ymm4[7,8],ymm2[9,10,11],ymm4[12],ymm2[13,14],ymm4[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vmovdqa %ymm11, %ymm1
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm29 & ymm11)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm11 & ymm29)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm4[1],ymm9[2,3],ymm4[4],ymm9[5,6,7,8],ymm4[9],ymm9[10,11],ymm4[12],ymm9[13,14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm0 & ymm1)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm1 & ymm0)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm4[1],ymm14[2,3,4],ymm4[5],ymm14[6,7,8],ymm4[9],ymm14[10,11,12],ymm4[13],ymm14[14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm3 & ymm1)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm1 & ymm3)
; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm6 ^ (ymm5 & (ymm13 ^ ymm6))
@@ -13073,7 +13073,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm20, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} ymm23 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm7 & ymm23)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm23 & ymm7)
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm7
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm30 ^ (ymm7 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
@@ -13123,7 +13123,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm31 ^ (ymm7 & (ymm27 ^ ymm31))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0],ymm7[1,2],ymm0[3],ymm7[4,5,6],ymm0[7,8],ymm7[9,10],ymm0[11],ymm7[12,13,14],ymm0[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm3 & ymm23)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm23 & ymm3)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm10, %ymm3
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm30 ^ (ymm3 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm13
@@ -13239,13 +13239,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm8[1,2,3],ymm4[4],ymm8[5,6],ymm4[7,8],ymm8[9,10,11],ymm4[12],ymm8[13,14],ymm4[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm13 & ymm23)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm23 & ymm13)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm15[0],ymm4[1],ymm15[2,3],ymm4[4],ymm15[5,6,7,8],ymm4[9],ymm15[10,11],ymm4[12],ymm15[13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm3 & ymm23)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm23 & ymm3)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm4[1],ymm9[2,3,4],ymm4[5],ymm9[6,7,8],ymm4[9],ymm9[10,11,12],ymm4[13],ymm9[14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm16 & ymm23)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm23 & ymm16)
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm14 ^ (ymm10 & (ymm2 ^ ymm14))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm30 ^ (ymm7 & (ymm28 ^ ymm30))
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
index 7dbff047e4f87..57d667563f8d4 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
@@ -3100,7 +3100,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512-NEXT: vprold $16, %xmm10, %xmm11
; AVX512-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3145,7 +3145,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3164,7 +3164,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3183,7 +3183,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512-NEXT: vprold $16, %xmm24, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3244,7 +3244,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512-FCP-NEXT: vprold $16, %xmm10, %xmm11
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3289,7 +3289,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3308,7 +3308,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3327,7 +3327,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512-FCP-NEXT: vprold $16, %xmm24, %xmm0
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3388,7 +3388,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512DQ-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512DQ-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512DQ-NEXT: vprold $16, %xmm10, %xmm11
; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3433,7 +3433,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3452,7 +3452,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512DQ-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3471,7 +3471,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512DQ-NEXT: vprold $16, %xmm24, %xmm0
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3532,7 +3532,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512DQ-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512DQ-FCP-NEXT: vprold $16, %xmm10, %xmm11
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3577,7 +3577,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3596,7 +3596,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512DQ-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3615,7 +3615,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512DQ-FCP-NEXT: vprold $16, %xmm24, %xmm0
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index 3311a311c8e46..98487632ebf5c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -3409,7 +3409,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,1,1,1]
; AVX512-NEXT: vpandnq %ymm11, %ymm20, %ymm11
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm11, %zmm11
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm13 & zmm20)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm20 & zmm13)
; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm0
; AVX512-NEXT: vpbroadcastq 8(%rdi), %xmm2
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
@@ -3535,7 +3535,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,1]
; AVX512-FCP-NEXT: vpandnq %ymm3, %ymm17, %ymm3
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm12 & zmm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm17 & zmm12)
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm12
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm12[0,1,2,1,4,5,6,5]
; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm15
@@ -3716,7 +3716,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,1,1,1]
; AVX512DQ-NEXT: vpandnq %ymm11, %ymm20, %ymm11
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm11, %zmm11
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm13 & zmm20)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm20 & zmm13)
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm0
; AVX512DQ-NEXT: vpbroadcastq 8(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
@@ -3842,7 +3842,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,1]
; AVX512DQ-FCP-NEXT: vpandnq %ymm3, %ymm17, %ymm3
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm12 & zmm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm17 & zmm12)
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm12[0,1,2,1,4,5,6,5]
; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm15
@@ -6987,8 +6987,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpandnq 80(%r8){1to4}, %ymm23, %ymm27
; AVX512-NEXT: vinserti64x4 $1, %ymm27, %zmm8, %zmm8
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm29 & zmm1)
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm22 = zmm22 | (zmm16 & zmm1)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm1 & zmm29)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm22 = zmm22 | (zmm1 & zmm16)
; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm11, %zmm11
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
; AVX512-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm11 # 64-byte Folded Reload
@@ -7020,8 +7020,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpternlogq {{.*#+}} zmm18 = zmm21 ^ (zmm30 & (zmm18 ^ zmm21))
; AVX512-NEXT: vpternlogq {{.*#+}} zmm7 = zmm20 ^ (zmm30 & (zmm7 ^ zmm20))
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm18 & zmm3)
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm7 & zmm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm3 & zmm7)
; AVX512-NEXT: vmovdqa64 %zmm8, 384(%r9)
; AVX512-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512-NEXT: vmovdqa64 %zmm4, 256(%r9)
@@ -7303,15 +7303,15 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm20 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm20 = mem ^ (zmm8 & (zmm20 ^ mem))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm10 & zmm7)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm20 & zmm7)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm7 & zmm10)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm7 & zmm20)
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm28 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm28 = mem ^ (zmm2 & (zmm28 ^ mem))
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm19 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm19 = mem ^ (zmm2 & (zmm19 ^ mem))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm28 & zmm2)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm19 & zmm2)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm2 & zmm28)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm2 & zmm19)
; AVX512-FCP-NEXT: vpbroadcastq 64(%r8), %ymm2
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm2, %zmm2
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
@@ -7596,8 +7596,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpandnq 80(%r8){1to4}, %ymm23, %ymm27
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm27, %zmm8, %zmm8
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm29 & zmm1)
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm22 = zmm22 | (zmm16 & zmm1)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm1 & zmm29)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm22 = zmm22 | (zmm1 & zmm16)
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm14, %zmm11, %zmm11
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm11 # 64-byte Folded Reload
@@ -7629,8 +7629,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm18 = zmm21 ^ (zmm30 & (zmm18 ^ zmm21))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm7 = zmm20 ^ (zmm30 & (zmm7 ^ zmm20))
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm18 & zmm3)
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm7 & zmm3)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm3 & zmm7)
; AVX512DQ-NEXT: vmovdqa64 %zmm8, 384(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm4, 256(%r9)
@@ -7912,15 +7912,15 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm20 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm20 = mem ^ (zmm8 & (zmm20 ^ mem))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm10 & zmm7)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm20 & zmm7)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm7 & zmm10)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm7 & zmm20)
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm28 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm28 = mem ^ (zmm2 & (zmm28 ^ mem))
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm19 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm19 = mem ^ (zmm2 & (zmm19 ^ mem))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm28 & zmm2)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm19 & zmm2)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm2 & zmm28)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm2 & zmm19)
; AVX512DQ-FCP-NEXT: vpbroadcastq 64(%r8), %ymm2
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm2, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index a4299ffe7a736..fa94786e85c3e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -12660,9 +12660,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
; AVX512-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm7 & ymm12)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm12 & ymm7)
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm11)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm11 & ymm6)
; AVX512-NEXT: vextracti64x4 $1, %zmm7, %ymm6
; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
@@ -13324,12 +13324,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
; AVX512-FCP-NEXT: vmovdqa 96(%r8), %ymm5
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm5[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[16,17,u,u,u,u],zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm12 & ymm13)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm13 & ymm12)
; AVX512-FCP-NEXT: vmovdqa 96(%r9), %ymm6
; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm4
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm11 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm16 & ymm11)
; AVX512-FCP-NEXT: vmovdqa64 %zmm16, %zmm20
; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm12, %ymm11
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
@@ -13987,9 +13987,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm7 & ymm12)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm12 & ymm7)
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm11)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm11 & ymm6)
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm7, %ymm6
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
@@ -14651,12 +14651,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %ymm5
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm5[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[16,17,u,u,u,u],zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm12 & ymm13)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm13 & ymm12)
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %ymm6
; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm4
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm11 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm16 & ymm11)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, %zmm20
; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm12, %ymm11
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index ad9db98711a62..5748d9f3bb430 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -1481,7 +1481,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512BW-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512BW-NEXT: vzeroupper
@@ -1511,7 +1511,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512BW-FCP-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512BW-FCP-NEXT: vzeroupper
@@ -1549,7 +1549,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512DQ-BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512DQ-BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512DQ-BW-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512DQ-BW-NEXT: vzeroupper
@@ -1579,7 +1579,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512DQ-BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512DQ-BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
@@ -4793,7 +4793,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vinserti64x4 $1, %ymm22, %zmm7, %zmm7
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm7 = zmm3 ^ (zmm8 & (zmm7 ^ zmm3))
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm7 & zmm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm15 & zmm7)
; AVX512-NEXT: vpermq {{.*#+}} zmm3 = zmm24[2,2,3,3,6,6,7,7]
; AVX512-NEXT: vpermq {{.*#+}} zmm7 = zmm26[2,2,3,3,6,6,7,7]
; AVX512-NEXT: vporq %zmm3, %zmm7, %zmm3
@@ -4971,7 +4971,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm23, %zmm4, %zmm4
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm5 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm5 & (zmm4 ^ zmm2))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm4 & zmm10)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm10 & zmm4)
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm26[2,2,3,3]
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm25, %zmm2
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm28[2,2,3,3]
@@ -5135,7 +5135,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm22, %zmm7, %zmm7
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm7 = zmm3 ^ (zmm8 & (zmm7 ^ zmm3))
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm7 & zmm15)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm15 & zmm7)
; AVX512DQ-NEXT: vpermq {{.*#+}} zmm3 = zmm24[2,2,3,3,6,6,7,7]
; AVX512DQ-NEXT: vpermq {{.*#+}} zmm7 = zmm26[2,2,3,3,6,6,7,7]
; AVX512DQ-NEXT: vporq %zmm3, %zmm7, %zmm3
@@ -5313,7 +5313,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm23, %zmm4, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm5 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm5 & (zmm4 ^ zmm2))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm4 & zmm10)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm10 & zmm4)
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm26[2,2,3,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm25, %zmm2
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm28[2,2,3,3]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 80151358c8316..9638cce12fb1d 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -8725,71 +8725,73 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
;
; AVX512-FCP-LABEL: store_i8_stride7_vf64:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: subq $1448, %rsp # imm = 0x5A8
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm8
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero,zero,zero,zero,ymm8[18]
+; AVX512-FCP-NEXT: subq $1416, %rsp # imm = 0x588
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm7
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
+; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm21
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm7
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm0
-; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm14
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm14, %ymm1
-; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
+; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm9
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm9[14],zero,zero,zero,zero,zero,zero,ymm9[15],zero,zero,zero,zero,zero,zero,ymm9[16],zero,zero,zero,zero,zero,zero,ymm9[17],zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm6
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[0,1,0,1,14],zero,ymm6[14,15,0,1,14,15],zero,ymm6[13,14,15,16,17,16],zero,ymm6[30,31,30,31,16,17],zero,ymm6[31,28,29,30,31]
+; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm20
-; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm9
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[13,u,u,u,u,u],zero,ymm9[14,u,u,u,u,u],zero,ymm9[15,u,u,u,u,u],zero,ymm9[16,u,u,u,u,u],zero,ymm9[17,u,u,u]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm16
+; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm8
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
+; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm11
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
-; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm15
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
+; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm15, %ymm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm17
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero,zero,ymm5[27],zero,ymm5[25]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm11[23,u,u,u],zero,ymm11[26],zero,ymm11[24,u,u,u],zero,ymm11[27],zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[23],zero,zero,zero,zero,ymm11[26],zero,ymm11[24],zero,zero,zero,zero,ymm11[27],zero,ymm11[25]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm15[23,u,u,u],zero,ymm15[26],zero,ymm15[24,u,u,u],zero,ymm15[27],zero
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm4
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm10
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128]
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm1
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
-; AVX512-FCP-NEXT: # ymm10 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm10, %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
+; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
+; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[25],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm10[23],zero,ymm10[21,22,23,26],zero,ymm10[24],zero,ymm10[28,29,26,27]
; AVX512-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm3
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
-; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm12
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
+; AVX512-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm12
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm2
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128]
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm13
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
@@ -8797,300 +8799,295 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm15
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm0
+; AVX512-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
; AVX512-FCP-NEXT: # ymm12 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm15, %ymm13
-; AVX512-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm15[2,3,2,3],zmm13[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm13
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[2,3,2,3],zmm13[2,3,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm21
-; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm13
-; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm8
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm7, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
+; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm7
+; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm7
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm14, %ymm19
-; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm8
-; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm17
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 %ymm9, %ymm24
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rax), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm6
+; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm5
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [4,0,6,0,4,0,6,7,0,17,0,17,0,16,16,0]
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm8
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT: vpermi2d %zmm10, %zmm6, %zmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm7
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT: vpermi2d %zmm9, %zmm5, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm14
-; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
-; AVX512-FCP-NEXT: vmovdqa64 %ymm9, %ymm20
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm7, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm12, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm19
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm7
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm7
-; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm27
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm7, %xmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm13
+; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u],zero,xmm0[7],zero,xmm0[5,u,u,u],zero,xmm0[8],zero,xmm0[6,u,u]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm29
+; AVX512-FCP-NEXT: vpor %xmm5, %xmm6, %xmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm6
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm9
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm9, %xmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm14, %xmm8
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm9
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
+; AVX512-FCP-NEXT: vpor %xmm8, %xmm9, %xmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,5,5,6]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm8
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm18
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm17
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
-; AVX512-FCP-NEXT: vpermi2d %zmm6, %zmm8, %zmm0
+; AVX512-FCP-NEXT: vpermi2d %zmm8, %zmm7, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm0
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm9
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm23
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm25
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm16
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm8
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm8, %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
+; AVX512-FCP-NEXT: vporq %xmm7, %xmm8, %xmm31
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,ymm4[14],zero,zero,zero,zero,zero,zero,ymm4[15],zero,zero,zero,zero,zero,zero,ymm4[16],zero,zero,zero,zero,zero,zero,ymm4[17],zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[0,1,0,1,14],zero,ymm10[14,15,0,1,14,15],zero,ymm10[13,14,15,16,17,16],zero,ymm10[30,31,30,31,16,17],zero,ymm10[31,28,29,30,31]
+; AVX512-FCP-NEXT: vpor %ymm7, %ymm8, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm8
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[18],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm10[18,19,20,21],zero,ymm10[19],zero,ymm10[25,26,27,22],zero,ymm10[20],zero
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero,zero,ymm15[18]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[0,1,14],zero,ymm11[12,13,0,1,14,15],zero,ymm11[3,12,13,2,3,16],zero,ymm11[30,31,28,29,16,17],zero,ymm11[31,18,19,28,29,18],zero
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
+; AVX512-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm15, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm1
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm11[14],zero,zero,zero,zero,zero,zero,ymm11[15],zero,zero,zero,zero,zero,zero,ymm11[16],zero,zero,zero,zero,zero,zero,ymm11[17],zero,zero,zero,zero,zero,zero,ymm11[18]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm1
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm4
-; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm12
-; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm8
-; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm2
-; AVX512-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm2
+; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm11
-; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm2
-; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm10
-; AVX512-FCP-NEXT: vmovdqa64 %xmm18, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm18
-; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm7
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm11[u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u]
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm29, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm17, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm1
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm17
+; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm10
+; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
+; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm8
; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm5
-; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-FCP-NEXT: vporq %xmm3, %xmm4, %xmm26
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512-FCP-NEXT: vporq %xmm3, %xmm4, %xmm29
+; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero,ymm2[27],zero,ymm2[25]
+; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm1[23,u,u,u],zero,ymm1[26],zero,ymm1[24,u,u,u],zero,ymm1[27],zero
; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm25
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm4
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm23
; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm1
-; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm24
-; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm23
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm22
-; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm14[23],zero,ymm14[23,24,25,26],zero,ymm14[24],zero,ymm14[30,31]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm14[18],zero,ymm14[20,21,20,21],zero,ymm14[19],zero,ymm14[19,20,21,22],zero
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
+; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm22
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm2[18,19,20,21],zero,ymm2[19],zero,ymm2[25,26,27,22],zero,ymm2[20],zero
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm21
+; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm12[23],zero,ymm12[23,24,25,26],zero,ymm12[24],zero,ymm12[30,31]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm12[18],zero,ymm12[20,21,20,21],zero,ymm12[19],zero,ymm12[19,20,21,22],zero
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,6]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm0
; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm29
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm14
-; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm15 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
+; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm3
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm5
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm10
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3],xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm1
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm1
; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3],xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm6[8],xmm11[9],xmm6[9],xmm11[10],xmm6[10],xmm11[11],xmm6[11],xmm11[12],xmm6[12],xmm11[13],xmm6[13],xmm11[14],xmm6[14],xmm11[15],xmm6[15]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm13[8],xmm4[9],xmm13[9],xmm4[10],xmm13[10],xmm4[11],xmm13[11],xmm4[12],xmm13[12],xmm4[13],xmm13[13],xmm4[14],xmm13[14],xmm4[15],xmm13[15]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm13
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm3[0,1,0,1],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm15[8],xmm2[8],xmm15[9],xmm2[9],xmm15[10],xmm2[10],xmm15[11],xmm2[11],xmm15[12],xmm2[12],xmm15[13],xmm2[13],xmm15[14],xmm2[14],xmm15[15],xmm2[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm4
-; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm3, %xmm3
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm2[0,1,0,1],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm14
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm26 = zmm0[0,1,0,1],zmm3[0,1,0,1]
; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm9[8],xmm1[8],xmm9[9],xmm1[9],xmm9[10],xmm1[10],xmm9[11],xmm1[11],xmm9[12],xmm1[12],xmm9[13],xmm1[13],xmm9[14],xmm1[14],xmm9[15],xmm1[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm28 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm28, %zmm15
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm0
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT: vpermd %ymm13, %ymm28, %ymm13
-; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3],xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm6 = zmm6[0,1,0,1],mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm26[0,1,0,1]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm24[2,3,2,3],zmm25[2,3,2,3]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm22[2,3,2,3],zmm23[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm12 & (zmm7 ^ zmm5))
-; AVX512-FCP-NEXT: vporq %zmm20, %zmm19, %zmm5
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm5))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm31[0,1,0,1],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm1, %zmm9
+; AVX512-FCP-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm0, %ymm2
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm1
+; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm1 = zmm1[0,1,0,1],mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm7 = zmm7[0,1,0,1],mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm2
+; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm29[0,1,0,1]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm23[2,3,2,3],zmm25[2,3,2,3]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm21[2,3,2,3],zmm22[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm10 & (zmm8 ^ zmm6))
+; AVX512-FCP-NEXT: vporq %zmm19, %zmm18, %zmm6
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm6))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm8))
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa64 %zmm15, 128(%rax)
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm7 = zmm7[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm8 = zmm8[2,3,2,3],mem[2,3,2,3]
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm13[0,1,0,1]
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm5))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm12 & (zmm5 ^ zmm7))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm17 ^ (zmm12 & (zmm31 ^ zmm17))
-; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm21 ^ (zmm7 & (zmm16 ^ zmm21))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm8))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm26 = zmm16 ^ (zmm10 & (zmm26 ^ zmm16))
+; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm20 ^ (zmm8 & (zmm12 ^ zmm20))
+; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: # ymm10 = mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm10
; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm11 = mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: # ymm12 = mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm11 ^ (zmm7 & (zmm12 ^ zmm11))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm2, %zmm11
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm11 = zmm10 ^ (zmm8 & (zmm11 ^ zmm10))
; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm4))
-; AVX512-FCP-NEXT: vinserti64x4 $1, (%rsp), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm3 & mem)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm7))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm5))
+; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm8 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm4))
+; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & mem) | zmm24
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm8))
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm4 ^ (mem & (zmm30 ^ zmm4))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm16))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm1 ^ (mem & (zmm9 ^ zmm1))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm31))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm12))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm18 ^ (mem & (zmm2 ^ zmm18))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm6))
-; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rax)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm2 ^ (mem & (zmm28 ^ zmm2))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (mem & (zmm28 ^ zmm12))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm2 ^ (mem & (zmm31 ^ zmm2))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm26))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm2 = zmm2[0,1,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm11))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm17 ^ (mem & (zmm5 ^ zmm17))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm7))
+; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 320(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 256(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm30, 192(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 384(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm14, 64(%rax)
-; AVX512-FCP-NEXT: addq $1448, %rsp # imm = 0x5A8
+; AVX512-FCP-NEXT: vmovdqa64 %zmm31, 256(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm28, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 384(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 64(%rax)
+; AVX512-FCP-NEXT: addq $1416, %rsp # imm = 0x588
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -9452,71 +9449,73 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
;
; AVX512DQ-FCP-LABEL: store_i8_stride7_vf64:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: subq $1448, %rsp # imm = 0x5A8
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm8
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero,zero,zero,zero,ymm8[18]
+; AVX512DQ-FCP-NEXT: subq $1416, %rsp # imm = 0x588
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm21
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm14, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
+; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm9
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm9[14],zero,zero,zero,zero,zero,zero,ymm9[15],zero,zero,zero,zero,zero,zero,ymm9[16],zero,zero,zero,zero,zero,zero,ymm9[17],zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm6
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[0,1,0,1,14],zero,ymm6[14,15,0,1,14,15],zero,ymm6[13,14,15,16,17,16],zero,ymm6[30,31,30,31,16,17],zero,ymm6[31,28,29,30,31]
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm20
-; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm9
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[13,u,u,u,u,u],zero,ymm9[14,u,u,u,u,u],zero,ymm9[15,u,u,u,u,u],zero,ymm9[16,u,u,u,u,u],zero,ymm9[17,u,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm16
+; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm8
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
+; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm11
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
-; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm15
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
+; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm17
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero,zero,ymm5[27],zero,ymm5[25]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm11[23,u,u,u],zero,ymm11[26],zero,ymm11[24,u,u,u],zero,ymm11[27],zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[23],zero,zero,zero,zero,ymm11[26],zero,ymm11[24],zero,zero,zero,zero,ymm11[27],zero,ymm11[25]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm15[23,u,u,u],zero,ymm15[26],zero,ymm15[24,u,u,u],zero,ymm15[27],zero
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm10
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128]
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm1
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
-; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
+; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512DQ-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[25],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm10[23],zero,ymm10[21,22,23,26],zero,ymm10[24],zero,ymm10[28,29,26,27]
; AVX512DQ-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm3
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
-; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
+; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm2
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128]
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512DQ-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
@@ -9524,300 +9523,295 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
; AVX512DQ-FCP-NEXT: # ymm12 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm15, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm15[2,3,2,3],zmm13[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm13
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[2,3,2,3],zmm13[2,3,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm21
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm8
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm7, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm7
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm14, %ymm19
-; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm17
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, %ymm24
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm6
+; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm5
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [4,0,6,0,4,0,6,7,0,17,0,17,0,16,16,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm8
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm10, %zmm6, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm7
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm9, %zmm5, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, %ymm20
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm7, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm12, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm19
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm7
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm7
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm27
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm7, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm13
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u],zero,xmm0[7],zero,xmm0[5,u,u,u],zero,xmm0[8],zero,xmm0[6,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm29
+; AVX512DQ-FCP-NEXT: vpor %xmm5, %xmm6, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm9
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm9, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm14, %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
+; AVX512DQ-FCP-NEXT: vpor %xmm8, %xmm9, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,5,5,6]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm18
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm17
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm6, %zmm8, %zmm0
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm8, %zmm7, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm9
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm23
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm25
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm16
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm8, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
+; AVX512DQ-FCP-NEXT: vporq %xmm7, %xmm8, %xmm31
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,ymm4[14],zero,zero,zero,zero,zero,zero,ymm4[15],zero,zero,zero,zero,zero,zero,ymm4[16],zero,zero,zero,zero,zero,zero,ymm4[17],zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[0,1,0,1,14],zero,ymm10[14,15,0,1,14,15],zero,ymm10[13,14,15,16,17,16],zero,ymm10[30,31,30,31,16,17],zero,ymm10[31,28,29,30,31]
+; AVX512DQ-FCP-NEXT: vpor %ymm7, %ymm8, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm8
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[18],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm10[18,19,20,21],zero,ymm10[19],zero,ymm10[25,26,27,22],zero,ymm10[20],zero
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero,zero,ymm15[18]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[0,1,14],zero,ymm11[12,13,0,1,14,15],zero,ymm11[3,12,13,2,3,16],zero,ymm11[30,31,28,29,16,17],zero,ymm11[31,18,19,28,29,18],zero
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
+; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm1
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm11[14],zero,zero,zero,zero,zero,zero,ymm11[15],zero,zero,zero,zero,zero,zero,ymm11[16],zero,zero,zero,zero,zero,zero,ymm11[17],zero,zero,zero,zero,zero,zero,ymm11[18]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm12
-; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm2
-; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512DQ-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm11
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm2
-; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm10
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm18, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm7
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm11[u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm29, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm17, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm17
+; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm5
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-FCP-NEXT: vporq %xmm3, %xmm4, %xmm26
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512DQ-FCP-NEXT: vporq %xmm3, %xmm4, %xmm29
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero,ymm2[27],zero,ymm2[25]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm1[23,u,u,u],zero,ymm1[26],zero,ymm1[24,u,u,u],zero,ymm1[27],zero
; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm25
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm4
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm23
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm24
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm23
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm14[23],zero,ymm14[23,24,25,26],zero,ymm14[24],zero,ymm14[30,31]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm14[18],zero,ymm14[20,21,20,21],zero,ymm14[19],zero,ymm14[19,20,21,22],zero
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm22
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm2[18,19,20,21],zero,ymm2[19],zero,ymm2[25,26,27,22],zero,ymm2[20],zero
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm21
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm12[23],zero,ymm12[23,24,25,26],zero,ymm12[24],zero,ymm12[30,31]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm12[18],zero,ymm12[20,21,20,21],zero,ymm12[19],zero,ymm12[19,20,21,22],zero
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,6]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm0
; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm29
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm14
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm15 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
+; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm3
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm5
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm10
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3],xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3],xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm6[8],xmm11[9],xmm6[9],xmm11[10],xmm6[10],xmm11[11],xmm6[11],xmm11[12],xmm6[12],xmm11[13],xmm6[13],xmm11[14],xmm6[14],xmm11[15],xmm6[15]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm13[8],xmm4[9],xmm13[9],xmm4[10],xmm13[10],xmm4[11],xmm13[11],xmm4[12],xmm13[12],xmm4[13],xmm13[13],xmm4[14],xmm13[14],xmm4[15],xmm13[15]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm13
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm3[0,1,0,1],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm15[8],xmm2[8],xmm15[9],xmm2[9],xmm15[10],xmm2[10],xmm15[11],xmm2[11],xmm15[12],xmm2[12],xmm15[13],xmm2[13],xmm15[14],xmm2[14],xmm15[15],xmm2[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm2[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm14
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm26 = zmm0[0,1,0,1],zmm3[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm9[8],xmm1[8],xmm9[9],xmm1[9],xmm9[10],xmm1[10],xmm9[11],xmm1[11],xmm9[12],xmm1[12],xmm9[13],xmm1[13],xmm9[14],xmm1[14],xmm9[15],xmm1[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm28 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm28, %zmm15
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vpermd %ymm13, %ymm28, %ymm13
-; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3],xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[0,1,0,1],mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm26[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm24[2,3,2,3],zmm25[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm22[2,3,2,3],zmm23[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm12 & (zmm7 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vporq %zmm20, %zmm19, %zmm5
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm31[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm1, %zmm9
+; AVX512DQ-FCP-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm1 = zmm1[0,1,0,1],mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm7 = zmm7[0,1,0,1],mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm2
+; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm29[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm23[2,3,2,3],zmm25[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm21[2,3,2,3],zmm22[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm10 & (zmm8 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vporq %zmm19, %zmm18, %zmm6
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm8))
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, 128(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm7 = zmm7[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm8 = zmm8[2,3,2,3],mem[2,3,2,3]
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm13[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm12 & (zmm5 ^ zmm7))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm17 ^ (zmm12 & (zmm31 ^ zmm17))
-; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm21 ^ (zmm7 & (zmm16 ^ zmm21))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm8))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm26 = zmm16 ^ (zmm10 & (zmm26 ^ zmm16))
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm20 ^ (zmm8 & (zmm12 ^ zmm20))
+; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # ymm10 = mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm10
; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm11 = mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # ymm12 = mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm11 ^ (zmm7 & (zmm12 ^ zmm11))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm2, %zmm11
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm11 = zmm10 ^ (zmm8 & (zmm11 ^ zmm10))
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm4))
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, (%rsp), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm3 & mem)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm7))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm5))
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm8 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm4))
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & mem) | zmm24
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm8))
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm4 ^ (mem & (zmm30 ^ zmm4))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm16))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm1 ^ (mem & (zmm9 ^ zmm1))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm31))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm12))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm18 ^ (mem & (zmm2 ^ zmm18))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm6))
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rax)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm2 ^ (mem & (zmm28 ^ zmm2))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (mem & (zmm28 ^ zmm12))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm2 ^ (mem & (zmm31 ^ zmm2))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm26))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm2 = zmm2[0,1,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm11))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm17 ^ (mem & (zmm5 ^ zmm17))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm7))
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 320(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 256(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm30, 192(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 384(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, 64(%rax)
-; AVX512DQ-FCP-NEXT: addq $1448, %rsp # imm = 0x5A8
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm31, 256(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm28, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 384(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 64(%rax)
+; AVX512DQ-FCP-NEXT: addq $1416, %rsp # imm = 0x588
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
index ef49ae8402c66..92aa5f02a4b28 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
@@ -206,11 +206,11 @@ define float @test_v4f32(<4 x float> %a0) {
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm0, %xmm3
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm0 | xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm4 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vmaxss %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm3 | xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm4 | xmm3)
; AVX512VL-NEXT: vcmpunordss %xmm3, %xmm3, %k1
; AVX512VL-NEXT: vmovss %xmm3, %xmm2, %xmm2 {%k1}
; AVX512VL-NEXT: vmaxss %xmm1, %xmm2, %xmm0
@@ -376,17 +376,17 @@ define float @test_v8f32(<8 x float> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vmaxps %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordps %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovaps %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm0, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm1[3,3,3,3]
@@ -602,22 +602,22 @@ define float @test_v16f32(<16 x float> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vmaxps %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 & (ymm0 | ymm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 & (ymm2 | ymm0)
; AVX512VL-NEXT: vcmpunordps %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovaps %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vmaxps %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordps %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovaps %xmm1, %xmm0 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm1, %xmm0, %xmm1
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -795,7 +795,7 @@ define double @test_v4f64(<4 x double> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vmaxpd %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
@@ -948,12 +948,12 @@ define double @test_v8f64(<8 x double> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vmaxpd %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm2 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 & (ymm0 | ymm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 & (ymm2 | ymm0)
; AVX512VL-NEXT: vcmpunordpd %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovapd %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vmaxpd %xmm0, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordpd %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovapd %xmm1, %xmm3 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm3[1,0]
@@ -1163,7 +1163,7 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmaxpd %zmm1, %zmm0, %zmm1
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm0 | zmm2)
+; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm2 | zmm0)
; AVX512BW-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512BW-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512BW-NEXT: vextractf64x4 $1, %zmm1, %ymm0
@@ -1191,20 +1191,19 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmaxpd %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm0 | zmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm2 | zmm0)
; AVX512VL-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512VL-NEXT: vextractf64x4 $1, %zmm1, %ymm0
; AVX512VL-NEXT: vmaxpd %ymm0, %ymm1, %ymm0
-; AVX512VL-NEXT: vmovdqa %ymm2, %ymm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm3 = ymm0 & (ymm3 | ymm1)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 & (ymm1 | ymm2)
; AVX512VL-NEXT: vcmpunordpd %ymm1, %ymm1, %k1
-; AVX512VL-NEXT: vmovapd %ymm1, %ymm3 {%k1}
-; AVX512VL-NEXT: vextractf128 $1, %ymm3, %xmm0
-; AVX512VL-NEXT: vmaxpd %xmm0, %xmm3, %xmm1
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm3 | xmm2)
-; AVX512VL-NEXT: vcmpunordpd %xmm3, %xmm3, %k1
-; AVX512VL-NEXT: vmovapd %xmm3, %xmm1 {%k1}
+; AVX512VL-NEXT: vmovapd %ymm1, %ymm0 {%k1}
+; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vmaxpd %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
+; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
; AVX512VL-NEXT: vmaxsd %xmm0, %xmm1, %xmm0
; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index dd9594b11d96b..36fc4ddfdd3e9 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1756,8 +1756,8 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v8i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %xmm0, %xmm1
-; AVX512VL-NEXT: vpsrlw $11, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsrlw $11, %xmm0, %xmm1
+; AVX512VL-NEXT: vpsllw $5, %xmm0, %xmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
; AVX512VL-NEXT: retq
;
@@ -1771,8 +1771,8 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v8i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %xmm0, %xmm1
-; AVX512VLBW-NEXT: vpsrlw $11, %xmm0, %xmm0
+; AVX512VLBW-NEXT: vpsrlw $11, %xmm0, %xmm1
+; AVX512VLBW-NEXT: vpsllw $5, %xmm0, %xmm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 850297489624d..9f5229d183e55 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -1570,8 +1570,8 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v16i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm1
-; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm0
+; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm1
+; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
; AVX512VL-NEXT: retq
;
@@ -1585,8 +1585,8 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v16i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %ymm0, %ymm1
-; AVX512VLBW-NEXT: vpsrlw $11, %ymm0, %ymm0
+; AVX512VLBW-NEXT: vpsrlw $11, %ymm0, %ymm1
+; AVX512VLBW-NEXT: vpsllw $5, %ymm0, %ymm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index f2622d9572f51..aee0e758ab7e8 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -841,39 +841,39 @@ define <16 x i32> @splatconstant_rotate_mask_v16i32(<16 x i32> %a) nounwind {
define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
; AVX512F-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpsllw $5, %ymm0, %ymm1
+; AVX512F-NEXT: vpsrlw $11, %ymm0, %ymm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT: vpsllw $5, %ymm2, %ymm3
+; AVX512F-NEXT: vpsrlw $11, %ymm2, %ymm3
; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512F-NEXT: vpsrlw $11, %ymm0, %ymm0
-; AVX512F-NEXT: vpsrlw $11, %ymm2, %ymm2
+; AVX512F-NEXT: vpsllw $5, %ymm0, %ymm0
+; AVX512F-NEXT: vpsllw $5, %ymm2, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm1
+; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm1
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT: vpsllw $5, %ymm2, %ymm3
+; AVX512VL-NEXT: vpsrlw $11, %ymm2, %ymm3
; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm0
-; AVX512VL-NEXT: vpsrlw $11, %ymm2, %ymm2
+; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm0
+; AVX512VL-NEXT: vpsllw $5, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsllw $5, %zmm0, %zmm1
-; AVX512BW-NEXT: vpsrlw $11, %zmm0, %zmm0
+; AVX512BW-NEXT: vpsrlw $11, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsllw $5, %zmm0, %zmm0
; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512BW-NEXT: retq
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %zmm0, %zmm1
-; AVX512VLBW-NEXT: vpsrlw $11, %zmm0, %zmm0
+; AVX512VLBW-NEXT: vpsrlw $11, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsllw $5, %zmm0, %zmm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 249bcbaa1bf9c..eaaccc123f9c5 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -945,9 +945,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512DQVL-LABEL: splatvar_shift_v32i8:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -1314,9 +1314,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi
; AVX512DQVL-LABEL: splatvar_modulo_shift_v32i8:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index aff2228c258b5..49c881fb02a8e 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -191,9 +191,9 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512BW-LABEL: splatvar_shift_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -280,9 +280,9 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi
; AVX512BW-LABEL: splatvar_modulo_shift_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vpternlog.ll b/llvm/test/CodeGen/X86/vpternlog.ll
index 4f9e384bd85a0..d41d74953b719 100644
--- a/llvm/test/CodeGen/X86/vpternlog.ll
+++ b/llvm/test/CodeGen/X86/vpternlog.ll
@@ -4,7 +4,7 @@
define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
; CHECK-LABEL: foo:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm1 | zmm2)
; CHECK-NEXT: retq
%and.demorgan = or <8 x i64> %b, %a
%and3.demorgan = or <8 x i64> %and.demorgan, %c
@@ -15,7 +15,7 @@ define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
; CHECK-LABEL: xorbitcast:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm1 | zmm2)
; CHECK-NEXT: retq
%or1 = or <64 x i8> %a, %b
%or2 = or <64 x i8> %or1, %c
@@ -24,12 +24,12 @@ define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
ret <8 x i64> %xor
}
-define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d, <8 x i64> %e) {
+; Two chained 3-input VPTERNLOGs (5 distinct inputs).
+define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e) {
; CHECK-LABEL: foobar:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $208
-; CHECK-NEXT: vpternlogq $16
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & (zmm2 | ~zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & ~(zmm4 | zmm3)
; CHECK-NEXT: retq
%nb = xor <8 x i64> %b, splat (i64 -1)
%or = or <8 x i64> %nb, %c
@@ -40,13 +40,13 @@ define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
ret <8 x i64> %bar
}
-define <8 x i64> @or_not_and_guard(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d) {
-; CHECK-LABEL: or_not_and_guard:
+; A & ~(B | C) nested under another AND; the inner OR stays a plain vporq.
+define <8 x i64> @or_not_and(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
+; CHECK-LABEL: or_not_and:
; CHECK: # %bb.0:
-; CHECK: vpternlogq $16
-; CHECK-NOT: vpternlogq $252
-; CHECK: retq
+; CHECK-NEXT: vporq %zmm2, %zmm1, %zmm1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm3 & ~zmm1
+; CHECK-NEXT: retq
%or = or <8 x i64> %b, %c
%not_or = xor <8 x i64> %or, splat (i64 -1)
%lhs = and <8 x i64> %a, %not_or
@@ -54,12 +54,13 @@ define <8 x i64> @or_not_and_guard(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
ret <8 x i64> %res
}
-define <8 x i64> @depth4_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d, <8 x i64> %e) {
+; Depth-4 heterogeneous chain.
+define <8 x i64> @depth4_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e) {
; CHECK-LABEL: depth4_chain:
; CHECK: # %bb.0:
-; CHECK: vpternlogq
-; CHECK: retq
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm2 | (zmm0 ^ zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm0 & zmm3)
+; CHECK-NEXT: retq
%t0 = xor <8 x i64> %a, %b
%t1 = or <8 x i64> %t0, %c
%t2 = and <8 x i64> %t1, %d
@@ -67,12 +68,14 @@ define <8 x i64> @depth4_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
ret <8 x i64> %t3
}
-define <8 x i64> @depth5_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; Depth-5 heterogeneous chain.
+define <8 x i64> @depth5_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
; CHECK-LABEL: depth5_chain:
; CHECK: # %bb.0:
-; CHECK: vpternlogq
-; CHECK: retq
+; CHECK-NEXT: vpandq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm3 | (zmm0 ^ zmm2)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm0 & zmm4)
+; CHECK-NEXT: retq
%t0 = and <8 x i64> %a, %b
%t1 = xor <8 x i64> %t0, %c
%t2 = or <8 x i64> %t1, %d
@@ -81,14 +84,14 @@ define <8 x i64> @depth5_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
ret <8 x i64> %t4
}
-define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; Balanced tree.
+define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
; CHECK-LABEL: balanced_depth:
; CHECK: # %bb.0:
-; CHECK: vpternlogq
-; CHECK: vpternlogq
-; CHECK: vpternlogq
-; CHECK: retq
+; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & (zmm2 ^ zmm3)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm4 ^ zmm5)
+; CHECK-NEXT: retq
%l0 = or <8 x i64> %a, %b
%l1 = xor <8 x i64> %c, %d
%l2 = and <8 x i64> %l0, %l1
@@ -97,80 +100,52 @@ define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
ret <8 x i64> %res
}
+; NOT of a load: folds the memory operand into a single VPTERNLOG.
define <8 x i64> @flip(ptr %x) {
; CHECK-LABEL: flip:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $85, (%rdi), %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~mem
; CHECK-NEXT: retq
%ld = load <8 x i64>, ptr %x, align 64
%not = xor <8 x i64> %ld, splat (i64 -1)
ret <8 x i64> %not
}
-define dso_local <8 x i64> @fubar(<8 x i64> %0, <8 x i64> %1, <8 x i64> %2,
- <8 x i64> %3, <8 x i64> %4, <8 x i64> %5,
- <8 x i64> %6, <8 x i64> %7, <8 x i64> %8)
- local_unnamed_addr {
-; CHECK-LABEL: fubar:
-; CHECK: # %bb.0:
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK-NOT: vpternlogq $252
-; CHECK-NOT: vpternlogq $250
-; CHECK: retq
-Entry:
- %9 = or <8 x i64> %1, %0
- %10 = or <8 x i64> %9, %2
- %11 = or <8 x i64> %10, %3
- %12 = or <8 x i64> %11, %4
- %13 = or <8 x i64> %12, %5
- %14 = or <8 x i64> %13, %6
- %15 = or <8 x i64> %14, %7
- %16 = or <8 x i64> %15, %8
- ret <8 x i64> %16
-}
-
-define dso_local <8 x i64> @baz(<8 x i64> %0, <8 x i64> %1, <8 x i64> %2,
- <8 x i64> %3, <8 x i64> %4, <8 x i64> %5,
- <8 x i64> %6, <8 x i64> %7, <8 x i64> %8)
- local_unnamed_addr {
-; CHECK-LABEL: baz:
+; 9-input OR reduction -> chain of 3-input VPTERNLOGs.
+define <8 x i64> @or_reduce9(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f, <8 x i64> %g, <8 x i64> %h, <8 x i64> %i) {
+; CHECK-LABEL: or_reduce9:
; CHECK: # %bb.0:
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK: vpternlogq $254
-; CHECK: retq
-Entry:
- %9 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %0,
- <8 x i64> %1,
- <8 x i64> %2,
- i32 254)
- %10 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %3,
- <8 x i64> %4,
- <8 x i64> %5,
- i32 254)
- %11 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %6,
- <8 x i64> %7,
- <8 x i64> %8,
- i32 254)
- %12 = tail call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %9,
- <8 x i64> %10,
- <8 x i64> %11,
- i32 254)
- ret <8 x i64> %12
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: movq %rsp, %rbp
+; CHECK-NEXT: .cfi_def_cfa_register %rbp
+; CHECK-NEXT: andq $-64, %rsp
+; CHECK-NEXT: subq $64, %rsp
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm2
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm4
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm5 | zmm6
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm7 | mem
+; CHECK-NEXT: movq %rbp, %rsp
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: .cfi_def_cfa %rsp, 8
+; CHECK-NEXT: retq
+ %r1 = or <8 x i64> %b, %a
+ %r2 = or <8 x i64> %r1, %c
+ %r3 = or <8 x i64> %r2, %d
+ %r4 = or <8 x i64> %r3, %e
+ %r5 = or <8 x i64> %r4, %f
+ %r6 = or <8 x i64> %r5, %g
+ %r7 = or <8 x i64> %r6, %h
+ %r8 = or <8 x i64> %r7, %i
+ ret <8 x i64> %r8
}
-declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>,
- <8 x i64>, i32 immarg)
-
-; 256-bit vector case — exercises VLX path.
+; 256-bit vector case - exercises VLX path.
define <4 x i64> @foo_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
; CHECK-LABEL: foo_256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: vpternlogq {{.*#+}} ymm0 = ~(ymm0 | ymm1 | ymm2)
; CHECK-NEXT: retq
%or1 = or <4 x i64> %b, %a
%or2 = or <4 x i64> %or1, %c
@@ -178,11 +153,11 @@ define <4 x i64> @foo_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
ret <4 x i64> %not
}
-; 128-bit vector case — exercises VLX path.
+; 128-bit vector case - exercises VLX path.
define <2 x i64> @foo_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
; CHECK-LABEL: foo_128:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq $1
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1 | xmm2)
; CHECK-NEXT: retq
%or1 = or <2 x i64> %b, %a
%or2 = or <2 x i64> %or1, %c
@@ -190,26 +165,24 @@ define <2 x i64> @foo_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
ret <2 x i64> %not
}
-; Balanced OR tree — tests fallthrough from homogeneous-associative to generic.
-define <8 x i64> @balanced_or4(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c,
- <8 x i64> %d) {
+; Balanced OR-4 tree; one leaf OR stays a plain vporq.
+define <8 x i64> @balanced_or4(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
; CHECK-LABEL: balanced_or4:
; CHECK: # %bb.0:
-; CHECK: vpternlogq $254
-; CHECK-NOT: vporq
-; CHECK: retq
+; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm2 | zmm3
+; CHECK-NEXT: retq
%t1 = or <8 x i64> %a, %b
%t2 = or <8 x i64> %c, %d
%res = or <8 x i64> %t1, %t2
ret <8 x i64> %res
}
-; Multi-use operand — %a used in both ORs. hasOneUse() on the shared
-; value must not block the combine.
+; Multi-use operand - %a feeds both ORs; reuse collapses to one VPTERNLOG.
define <8 x i64> @shared_operand(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
; CHECK-LABEL: shared_operand:
; CHECK: # %bb.0:
-; CHECK: vpternlogq
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & zmm2)
; CHECK-NEXT: retq
%or1 = or <8 x i64> %a, %b
%or2 = or <8 x i64> %a, %c
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index f905fac15bffb..06cdce089de80 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -2731,14 +2731,12 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
@@ -2750,14 +2748,12 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3023,14 +3019,12 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3042,14 +3036,12 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3314,14 +3306,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = ymm0[0,1,0,1]
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
+; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
@@ -3334,14 +3325,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
+; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
index 21a3df0456de1..4a2873070c0a4 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -2149,13 +2149,11 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2166,13 +2164,11 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2399,13 +2395,11 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2416,13 +2410,11 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2649,14 +2641,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2667,14 +2658,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
More information about the llvm-commits
mailing list