[llvm] [AVX-512] make vpternlogq more aggressive for longer chains of bitmanipulations (PR #189971)
Julian Pokrovsky via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 23:50:04 PDT 2026
https://github.com/raventid updated https://github.com/llvm/llvm-project/pull/189971
>From 6c8c084408464f3589816a29be7e57549fcde629 Mon Sep 17 00:00:00 2001
From: raventid <juliankul at gmail.com>
Date: Tue, 2 Jun 2026 20:09:43 +0800
Subject: [PATCH] feat: applying a simplified design
---
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 273 ++-
llvm/test/CodeGen/X86/avx512-cvt.ll | 21 +-
llvm/test/CodeGen/X86/avx512-logic.ll | 2 +-
llvm/test/CodeGen/X86/avx512-mask-op.ll | 14 +-
llvm/test/CodeGen/X86/avx512fp16-arith.ll | 28 +-
llvm/test/CodeGen/X86/avx512vl-logic.ll | 10 +-
llvm/test/CodeGen/X86/bitcast-vector-bool.ll | 2 +-
llvm/test/CodeGen/X86/clmul-vector-256.ll | 84 +-
llvm/test/CodeGen/X86/clmul-vector-512.ll | 478 ++---
llvm/test/CodeGen/X86/combine-fround.ll | 24 +-
llvm/test/CodeGen/X86/combine-or-shuffle.ll | 8 +-
.../X86/elementwise-store-of-scalar-splat.ll | 12 +-
.../CodeGen/X86/fminimumnum-fmaximumnum.ll | 16 +-
llvm/test/CodeGen/X86/fp-round.ll | 74 +-
llvm/test/CodeGen/X86/fshl-splat-undef.ll | 6 +-
llvm/test/CodeGen/X86/gfni-shifts.ll | 18 +-
llvm/test/CodeGen/X86/icmp-pow2-diff.ll | 15 +-
.../CodeGen/X86/min-legal-vector-width.ll | 12 +-
llvm/test/CodeGen/X86/pmul.ll | 4 +-
.../CodeGen/X86/prefer-avx256-wide-mul.ll | 2 +-
llvm/test/CodeGen/X86/sat-add.ll | 48 +-
llvm/test/CodeGen/X86/select-big-integer.ll | 12 +-
.../CodeGen/X86/srem-seteq-vec-nonsplat.ll | 5 +-
.../subvectorwise-store-of-vector-splat.ll | 1618 ++++++++++++-----
.../X86/urem-seteq-vec-tautological.ll | 5 +-
llvm/test/CodeGen/X86/vector-fshr-256.ll | 2 +-
llvm/test/CodeGen/X86/vector-fshr-512.ll | 4 +-
.../vector-interleaved-load-i16-stride-6.ll | 8 +-
.../vector-interleaved-load-i8-stride-6.ll | 64 +-
.../vector-interleaved-load-i8-stride-7.ll | 80 +-
.../vector-interleaved-store-i16-stride-3.ll | 32 +-
.../vector-interleaved-store-i16-stride-5.ll | 40 +-
.../vector-interleaved-store-i16-stride-7.ll | 20 +-
.../vector-interleaved-store-i8-stride-5.ll | 16 +-
.../vector-interleaved-store-i8-stride-7.ll | 1102 ++++++-----
.../CodeGen/X86/vector-reduce-fmaximum.ll | 43 +-
.../CodeGen/X86/vector-reduce-fminimum.ll | 43 +-
llvm/test/CodeGen/X86/vector-rotate-128.ll | 8 +-
llvm/test/CodeGen/X86/vector-rotate-256.ll | 8 +-
llvm/test/CodeGen/X86/vector-rotate-512.ll | 24 +-
.../test/CodeGen/X86/vector-shift-ashr-256.ll | 4 +-
.../test/CodeGen/X86/vector-shift-ashr-512.ll | 4 +-
llvm/test/CodeGen/X86/vpternlog.ll | 170 +-
.../zero_extend_vector_inreg_of_broadcast.ll | 66 +-
...d_vector_inreg_of_broadcast_from_memory.ll | 66 +-
45 files changed, 2739 insertions(+), 1856 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index b794567f6a047..457b5f8093766 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -31,6 +31,7 @@
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include <cstdint>
+#include <functional>
using namespace llvm;
@@ -4696,8 +4697,17 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
SDNode *ParentB, SDNode *ParentC,
SDValue A, SDValue B, SDValue C,
uint8_t Imm) {
- assert(A.isOperandOf(ParentA) && B.isOperandOf(ParentB) &&
- C.isOperandOf(ParentC) && "Incorrect parent node");
+ // Unused operand slots may be padded with an IMPLICIT_DEF (e.g. when a tree
+ // folds to a VPTERNLOG with fewer than three distinct inputs); padding has no
+ // parent node.
+ auto IsPad = [](SDValue V) {
+ return V.isMachineOpcode() &&
+ V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
+ };
+ assert((IsPad(A) || A.isOperandOf(ParentA)) &&
+ (IsPad(B) || B.isOperandOf(ParentB)) &&
+ (IsPad(C) || C.isOperandOf(ParentC)) && "Incorrect parent node");
+ (void)IsPad;
auto tryFoldLoadOrBCast =
[this](SDNode *Root, SDNode *P, SDValue &L, SDValue &Base, SDValue &Scale,
@@ -4815,8 +4825,17 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
return true;
}
-// Try to match two logic ops to a VPTERNLOG.
-// FIXME: Handle more complex patterns that use an operand more than once?
+// Try to match a tree of bitwise logic ops to one or more VPTERNLOG ops.
+//
+// VPTERNLOG implements an arbitrary 3-input bitwise function via an 8-bit
+// truth-table immediate. We evaluate the logic sub-DAG rooted at N
+// symbolically: each of the (up to three) distinct leaf inputs is seeded with a
+// canonical truth-table column (0xF0/0xCC/0xAA) and AND/OR/XOR/ANDNP/NOT are
+// folded bitwise over those seeds, yielding the control immediate. Operand
+// reuse, inverted inputs and arbitrary nesting depth all fall out of the
+// recursion. Trees with more than three distinct leaves are split by leaving
+// one subtree "opaque" (it selects into its own VPTERNLOG when isel reaches it)
+// and folding the rest around it.
bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
MVT NVT = N->getSimpleValueType(0);
@@ -4829,118 +4848,170 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
return false;
- auto getFoldableLogicOp = [](SDValue Op) {
- // Peek through single use bitcast.
- if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
- Op = Op.getOperand(0);
-
- if (!Op.hasOneUse())
- return SDValue();
-
- unsigned Opc = Op.getOpcode();
- if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
- Opc == X86ISD::ANDNP)
- return Op;
-
- return SDValue();
+ auto IsLogic = [](unsigned Opc) {
+ return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
+ Opc == X86ISD::ANDNP;
+ };
+ auto IsNot = [](SDValue V) {
+ return V.getOpcode() == ISD::XOR &&
+ ISD::isBuildVectorAllOnes(V.getOperand(1).getNode());
+ };
+ auto PeelBitcast = [](SDValue V) {
+ if (V.getOpcode() == ISD::BITCAST && V.hasOneUse())
+ return V.getOperand(0);
+ return V;
+ };
+ auto IsLoadLike = [](SDValue V) {
+ return isa<LoadSDNode>(V.getNode()) ||
+ V.getOpcode() == X86ISD::VBROADCAST_LOAD;
};
- SDValue N0, N1, A, FoldableOp;
-
- // Identify and (optionally) peel an outer NOT that wraps a pure logic tree
- auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
- if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
- ISD::isBuildVectorAllOnes(Op->getOperand(1).getNode())) {
- SDValue InnerOp = getFoldableLogicOp(Op->getOperand(0));
-
- if (!InnerOp)
- return SDValue();
+ struct Leaf {
+ SDValue V;
+ SDNode *Parent;
+ };
- N0 = InnerOp.getOperand(0);
- N1 = InnerOp.getOperand(1);
- if ((FoldableOp = getFoldableLogicOp(N1))) {
- A = N0;
- return InnerOp;
- }
- if ((FoldableOp = getFoldableLogicOp(N0))) {
- A = N1;
- return InnerOp;
+ // Symbolically evaluate the tree rooted at Root. Opaque, if non-null, is
+ // treated as a leaf even when it is a logic op (used for cascading). On
+ // success fills Leaves (the distinct inputs in seed order, at most three),
+ // Imm and NumOps (the number of logic/NOT nodes folded - a profitability
+ // signal), and returns true. Returns false when more than three distinct
+ // leaves are required. Single-use is required to fold a node; bitcasts are
+ // peeled.
+ auto Evaluate = [&](SDValue Root, SDNode *Opaque,
+ SmallVectorImpl<Leaf> &Leaves, uint8_t &Imm,
+ unsigned &NumOps) -> bool {
+ static constexpr uint8_t Seeds[] = {0xF0, 0xCC, 0xAA};
+ NumOps = 0;
+ std::function<int(SDValue, SDNode *, bool)> Eval =
+ [&](SDValue Op, SDNode *Parent, bool IsRoot) -> int {
+ if (Op.getNode() != Opaque) {
+ if (Op.getOpcode() == ISD::BITCAST && (IsRoot || Op.hasOneUse())) {
+ Parent = Op.getNode();
+ Op = Op.getOperand(0);
+ }
+ if ((IsRoot || Op.hasOneUse()) && IsNot(Op)) {
+ ++NumOps;
+ int Inner = Eval(Op.getOperand(0), Op.getNode(), false);
+ return Inner < 0 ? -1 : (~Inner & 0xFF);
+ }
+ if ((IsRoot || Op.hasOneUse()) && IsLogic(Op.getOpcode())) {
+ ++NumOps;
+ int L = Eval(Op.getOperand(0), Op.getNode(), false);
+ int R = Eval(Op.getOperand(1), Op.getNode(), false);
+ if (L < 0 || R < 0)
+ return -1;
+ switch (Op.getOpcode()) {
+ case ISD::AND:
+ return L & R;
+ case ISD::OR:
+ return L | R;
+ case ISD::XOR:
+ return (L ^ R) & 0xFF;
+ case X86ISD::ANDNP:
+ return ~L & R;
+ default:
+ llvm_unreachable("Checked by IsLogic");
+ }
+ }
}
- }
- return SDValue();
+ // Leaf: reuse the seed of an identical input, else allocate a new one.
+ for (unsigned I = 0, E = Leaves.size(); I != E; ++I)
+ if (Leaves[I].V == Op)
+ return Seeds[I];
+ if (Leaves.size() >= 3)
+ return -1;
+ uint8_t Seed = Seeds[Leaves.size()];
+ Leaves.push_back({Op, Parent});
+ return Seed;
+ };
+ int Result = Eval(Root, Root.getNode(), /*IsRoot=*/true);
+ if (Result < 0)
+ return false;
+ Imm = Result & 0xFF;
+ return true;
};
- bool PeeledOuterNot = false;
- SDNode *OriN = N;
- if (SDValue InnerOp = tryPeelOuterNotWrappingLogic(N)) {
- PeeledOuterNot = true;
- N = InnerOp.getNode();
- } else {
- N0 = N->getOperand(0);
- N1 = N->getOperand(1);
+ // Emit one VPTERNLOG from up to three leaves. Unused operand slots are
+ // padded with undef so that, e.g., a NOT of a single memory operand folds the
+ // load with no false dependency on the other inputs.
+ auto Emit = [&](ArrayRef<Leaf> Leaves, uint8_t Imm) {
+ SDValue Undef(
+ CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SDLoc(N), NVT), 0);
+ SDValue Ops[3] = {Undef, Undef, Undef};
+ SDNode *Parents[3] = {N, N, N};
+ for (unsigned I = 0, E = Leaves.size(); I != E; ++I) {
+ Ops[I] = Leaves[I].V;
+ Parents[I] = Leaves[I].Parent;
+ }
+ return matchVPTERNLOG(N, Parents[0], Parents[1], Parents[2], Ops[0], Ops[1],
+ Ops[2], Imm);
+ };
- if ((FoldableOp = getFoldableLogicOp(N1)))
- A = N0;
- else if ((FoldableOp = getFoldableLogicOp(N0)))
- A = N1;
- else
+ // A cover is profitable when it folds at least two logic ops into one
+ // VPTERNLOG, or when it is a NOT of a single memory operand (which saves
+ // materializing an all-ones vector). A lone binary op (and/or/xor/andn) is
+ // left to its dedicated, cheaper instruction.
+ auto Profitable = [&](ArrayRef<Leaf> Leaves, unsigned NumOps) {
+ if (NumOps >= 2)
+ return true;
+ return Leaves.size() == 1 && IsLoadLike(PeelBitcast(Leaves[0].V));
+ };
+
+ // Whole tree fits in a single VPTERNLOG (<= 3 distinct leaves).
+ SmallVector<Leaf, 3> Leaves;
+ uint8_t Imm = 0;
+ unsigned NumOps = 0;
+ if (Evaluate(SDValue(N, 0), /*Opaque=*/nullptr, Leaves, Imm, NumOps)) {
+ if (!Profitable(Leaves, NumOps))
return false;
+ return Emit(Leaves, Imm);
}
- SDValue B = FoldableOp.getOperand(0);
- SDValue C = FoldableOp.getOperand(1);
- SDNode *ParentA = N;
- SDNode *ParentB = FoldableOp.getNode();
- SDNode *ParentC = FoldableOp.getNode();
-
- // We can build the appropriate control immediate by performing the logic
- // operation we're matching using these constants for A, B, and C.
- uint8_t TernlogMagicA = 0xf0;
- uint8_t TernlogMagicB = 0xcc;
- uint8_t TernlogMagicC = 0xaa;
-
- // Some of the inputs may be inverted, peek through them and invert the
- // magic values accordingly.
- // TODO: There may be a bitcast before the xor that we should peek through.
- auto PeekThroughNot = [](SDValue &Op, SDNode *&Parent, uint8_t &Magic) {
- if (Op.getOpcode() == ISD::XOR && Op.hasOneUse() &&
- ISD::isBuildVectorAllOnes(Op.getOperand(1).getNode())) {
- Magic = ~Magic;
- Parent = Op.getNode();
- Op = Op.getOperand(0);
+ // More than three leaves: cascade. Collect the single-use logic subtrees
+ // that could be made opaque (NOTs are transparent - cutting at one would just
+ // spill it to a separate instruction).
+ SmallVector<SDNode *, 8> Candidates;
+ std::function<void(SDValue)> Collect = [&](SDValue V) {
+ V = PeelBitcast(V);
+ if (!V.hasOneUse())
+ return;
+ if (IsNot(V)) {
+ Collect(V.getOperand(0));
+ return;
+ }
+ if (IsLogic(V.getOpcode())) {
+ Candidates.push_back(V.getNode());
+ Collect(V.getOperand(0));
+ Collect(V.getOperand(1));
}
};
-
- PeekThroughNot(A, ParentA, TernlogMagicA);
- PeekThroughNot(B, ParentB, TernlogMagicB);
- PeekThroughNot(C, ParentC, TernlogMagicC);
-
- uint8_t Imm;
- switch (FoldableOp.getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break;
- case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break;
- case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break;
- case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
- }
-
- switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case X86ISD::ANDNP:
- if (A == N0)
- Imm &= ~TernlogMagicA;
- else
- Imm = ~(Imm) & TernlogMagicA;
- break;
- case ISD::AND: Imm &= TernlogMagicA; break;
- case ISD::OR: Imm |= TernlogMagicA; break;
- case ISD::XOR: Imm ^= TernlogMagicA; break;
+ Collect(N->getOperand(0));
+ Collect(N->getOperand(1));
+
+ // Pick the cut that lets the root fold the most leaves (a tighter cascade),
+ // breaking ties towards folding more ops.
+ SmallVector<Leaf, 3> BestLeaves;
+ uint8_t BestImm = 0;
+ unsigned BestNumOps = 0;
+ for (SDNode *S : Candidates) {
+ SmallVector<Leaf, 3> CutLeaves;
+ uint8_t CutImm = 0;
+ unsigned CutNumOps = 0;
+ if (!Evaluate(SDValue(N, 0), S, CutLeaves, CutImm, CutNumOps))
+ continue;
+ if (CutLeaves.size() > BestLeaves.size() ||
+ (CutLeaves.size() == BestLeaves.size() && CutNumOps > BestNumOps)) {
+ BestLeaves = std::move(CutLeaves);
+ BestImm = CutImm;
+ BestNumOps = CutNumOps;
+ }
}
- if (PeeledOuterNot)
- Imm = ~Imm;
+ if (!BestLeaves.empty() && Profitable(BestLeaves, BestNumOps))
+ return Emit(BestLeaves, BestImm);
- return matchVPTERNLOG(OriN, ParentA, ParentB, ParentC, A, B, C, Imm);
+ return false;
}
/// If the high bits of an 'and' operand are known zero, try setting the
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index 76c87900b04d2..a8a906b69e9cb 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -350,8 +350,8 @@ define <4 x float> @ulto4f32_nneg(<4 x i64> %a) {
define <8 x double> @ulto8f64(<8 x i64> %a) {
; NODQ-LABEL: ulto8f64:
; NODQ: # %bb.0:
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm0
; NODQ-NEXT: vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; NODQ-NEXT: vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
@@ -374,17 +374,16 @@ define <8 x double> @ulto8f64(<8 x i64> %a) {
define <16 x double> @ulto16f64(<16 x i64> %a) {
; NODQ-LABEL: ulto16f64:
; NODQ: # %bb.0:
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT: vmovdqa64 %zmm3, %zmm4
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm0 & zmm2)
-; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm0
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; NODQ-NEXT: vpsrlq $32, %zmm0, %zmm4
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm0 = (zmm0 & zmm3) | zmm2
; NODQ-NEXT: vpbroadcastq {{.*#+}} zmm5 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
-; NODQ-NEXT: vporq %zmm5, %zmm0, %zmm0
+; NODQ-NEXT: vporq %zmm5, %zmm4, %zmm4
; NODQ-NEXT: vbroadcastsd {{.*#+}} zmm6 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
-; NODQ-NEXT: vsubpd %zmm6, %zmm0, %zmm0
-; NODQ-NEXT: vaddpd %zmm0, %zmm4, %zmm0
-; NODQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm1 & zmm2)
+; NODQ-NEXT: vsubpd %zmm6, %zmm4, %zmm4
+; NODQ-NEXT: vaddpd %zmm4, %zmm0, %zmm0
+; NODQ-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & zmm1) | zmm2
; NODQ-NEXT: vpsrlq $32, %zmm1, %zmm1
; NODQ-NEXT: vporq %zmm5, %zmm1, %zmm1
; NODQ-NEXT: vsubpd %zmm6, %zmm1, %zmm1
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index bdcc524545fb1..03cf284b2f26c 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -856,7 +856,7 @@ entry:
define <16 x i32> @ternlog_and_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
; ALL-LABEL: ternlog_and_andn:
; ALL: ## %bb.0:
-; ALL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm2 & zmm1 & ~zmm0
+; ALL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 & zmm2 & ~zmm0
; ALL-NEXT: retq
%a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <16 x i32> %y, %a
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index db53dda401bc2..e21ae6e1a3347 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -3295,12 +3295,7 @@ define <16 x i1> @test_v16i1_select_chain(<16 x i1> %a0, <16 x i1> %a1, <16 x i1
;
; SKX-LABEL: test_v16i1_select_chain:
; SKX: ## %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm3
-; SKX-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & ~(xmm2 & xmm1)
-; SKX-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & xmm0 & xmm1
-; SKX-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & xmm0 & xmm1
-; SKX-NEXT: vpternlogq {{.*#+}} xmm3 = xmm1 & (xmm3 ^ xmm1)
-; SKX-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & xmm2 & xmm3
+; SKX-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & xmm2 & xmm1
; SKX-NEXT: retq
;
; AVX512BW-LABEL: test_v16i1_select_chain:
@@ -3331,12 +3326,7 @@ define <16 x i1> @test_v16i1_select_chain(<16 x i1> %a0, <16 x i1> %a1, <16 x i1
;
; X86-LABEL: test_v16i1_select_chain:
; X86: ## %bb.0:
-; X86-NEXT: vmovdqa %xmm0, %xmm3
-; X86-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & ~(xmm2 & xmm1)
-; X86-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & xmm0 & xmm1
-; X86-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & xmm0 & xmm1
-; X86-NEXT: vpternlogq {{.*#+}} xmm3 = xmm1 & (xmm3 ^ xmm1)
-; X86-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & xmm2 & xmm3
+; X86-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & xmm2 & xmm1
; X86-NEXT: retl
%i3 = select <16 x i1> %a2, <16 x i1> %a1, <16 x i1> zeroinitializer
%i4 = xor <16 x i1> %i3, splat (i1 true)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-arith.ll b/llvm/test/CodeGen/X86/avx512fp16-arith.ll
index 9f74316f3c918..5114bcd98b236 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-arith.ll
@@ -339,9 +339,9 @@ declare half @llvm.copysign.f16(half, half)
define half @fround(half %x) {
; CHECK-LABEL: fround:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; CHECK-NEXT: vaddsh %xmm2, %xmm0, %xmm0
; CHECK-NEXT: vrndscalesh $11, %xmm0, %xmm0, %xmm0
; CHECK-NEXT: retq
@@ -394,9 +394,9 @@ declare <8 x half> @llvm.copysign.v8f16(<8 x half>, <8 x half>)
define <8 x half> @roundv8f16(<8 x half> %x) {
; CHECK-LABEL: roundv8f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; CHECK-NEXT: vaddph %xmm2, %xmm0, %xmm0
; CHECK-NEXT: vrndscaleph $11, %xmm0, %xmm0
; CHECK-NEXT: retq
@@ -449,9 +449,9 @@ declare <16 x half> @llvm.copysign.v16f16(<16 x half>, <16 x half>)
define <16 x half> @roundv16f16(<16 x half> %x) {
; CHECK-LABEL: roundv16f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ymm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm0) | ymm1
; CHECK-NEXT: vaddph %ymm2, %ymm0, %ymm0
; CHECK-NEXT: vrndscaleph $11, %ymm0, %ymm0
; CHECK-NEXT: retq
@@ -504,9 +504,9 @@ declare <32 x half> @llvm.copysign.v32f16(<32 x half>, <32 x half>)
define <32 x half> @roundv32f16(<32 x half> %x) {
; CHECK-LABEL: roundv32f16:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm0 & zmm1)
+; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; CHECK-NEXT: vpbroadcastw {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm2 = (zmm2 & zmm0) | zmm1
; CHECK-NEXT: vaddph %zmm2, %zmm0, %zmm0
; CHECK-NEXT: vrndscaleph $11, %zmm0, %zmm0
; CHECK-NEXT: retq
@@ -659,7 +659,7 @@ entry:
define <8 x half> @ceil_sh(<8 x half> %x, <8 x half> %y) nounwind {
; CHECK-LABEL: ceil_sh:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vrndscalesh $10, %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vrndscalesh $10, %xmm0, %xmm1, %xmm0
; CHECK-NEXT: retq
%s = extractelement <8 x half> %x, i32 0
%call = call half @llvm.ceil.f16(half %s)
@@ -695,7 +695,7 @@ declare half @llvm.rint.f16(half %s)
define <8 x half> @nearbyint_sh(<8 x half> %x, <8 x half> %y) nounwind {
; CHECK-LABEL: nearbyint_sh:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vrndscalesh $12, %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vrndscalesh $12, %xmm0, %xmm1, %xmm0
; CHECK-NEXT: retq
%s = extractelement <8 x half> %x, i32 0
%call = call half @llvm.nearbyint.f16(half %s)
diff --git a/llvm/test/CodeGen/X86/avx512vl-logic.ll b/llvm/test/CodeGen/X86/avx512vl-logic.ll
index c1ae0e36c2c0d..8c95f219d6c48 100644
--- a/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -958,7 +958,7 @@ entry:
define <4 x i32> @ternlog_and_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_and_andn:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm2 & xmm1 & ~xmm0
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 & xmm2 & ~xmm0
; CHECK-NEXT: retq
%a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <4 x i32> %y, %a
@@ -999,14 +999,10 @@ define <4 x i32> @ternlog_and_orn_2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
ret <4 x i32> %c
}
-; FIXME: This should be a single vpternlog, but we accidentally match the xor -1
-; as the second binary op instead of the and.
define <4 x i32> @ternlog_orn_and(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_orn_and:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpand %xmm2, %xmm1, %xmm1
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 | (xmm0 ^ xmm3)
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = (xmm2 & xmm1) | ~xmm0
; CHECK-NEXT: retq
%a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%b = and <4 x i32> %y, %z
@@ -1328,7 +1324,7 @@ define <4 x i32> @ternlog_andn_or(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
define <4 x i32> @ternlog_andn_or_2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
; CHECK-LABEL: ternlog_andn_or_2:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & ~(xmm1 | xmm2)
+; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & ~(xmm2 | xmm1)
; CHECK-NEXT: retq
%a = or <4 x i32> %y, %z
%b = xor <4 x i32> %a, <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index f680520c05cb5..6cc4ad69edd0b 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -1252,7 +1252,7 @@ define i1 @trunc_v128i8_cmp(<128 x i8> %a0) nounwind {
; AVX512-LABEL: trunc_v128i8_cmp:
; AVX512: # %bb.0:
; AVX512-NEXT: vpbroadcastb {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm2 & zmm1
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm1 & zmm2
; AVX512-NEXT: vpcmpneqd %zmm2, %zmm0, %k0
; AVX512-NEXT: kortestw %k0, %k0
; AVX512-NEXT: setne %al
diff --git a/llvm/test/CodeGen/X86/clmul-vector-256.ll b/llvm/test/CodeGen/X86/clmul-vector-256.ll
index 0820dafd55bdd..8229314dff4ba 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-256.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-256.ll
@@ -206,56 +206,56 @@ define <32 x i8> @clmul_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm3 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm4 & ymm3)
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpxor %ymm2, %ymm3, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm3
; AVX512-NEXT: vpmullw %ymm3, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX512-NEXT: vpandn %ymm3, %ymm4, %ymm3
; AVX512-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpsllw $8, %ymm3, %ymm3
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 | ymm5)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm5
-; AVX512-NEXT: vpand %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vpandn %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
-; AVX512-NEXT: vpsllw $8, %ymm2, %ymm2
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 | ymm5)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm4 & ymm5)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm5
+; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm6
+; AVX512-NEXT: vpandn %ymm5, %ymm4, %ymm5
+; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
+; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm4 & ymm6)
+; AVX512-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm4, %ymm3, %ymm3
; AVX512-NEXT: vpandn %ymm1, %ymm4, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 | ymm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm3)
+; AVX512-NEXT: vpxor %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <32 x i8> @llvm.clmul.v32i8(<32 x i8> %a, <32 x i8> %b)
ret <32 x i8> %res
@@ -888,24 +888,24 @@ define <32 x i8> @clmulr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpand %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm7, %ymm5, %ymm7
; AVX512-NEXT: vpxor %ymm4, %ymm7, %ymm4
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512-NEXT: vpand %ymm7, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512-NEXT: vpand %ymm2, %ymm9, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm4 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512-NEXT: vpand %ymm2, %ymm11, %ymm12
; AVX512-NEXT: vpmullw %ymm5, %ymm12, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpand %ymm2, %ymm8, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512-NEXT: vpand %ymm2, %ymm13, %ymm2
; AVX512-NEXT: vpmullw %ymm2, %ymm5, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ ymm12 ^ ymm10
@@ -1881,24 +1881,24 @@ define <32 x i8> @clmulh_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512-NEXT: vpand %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm7, %ymm5, %ymm7
; AVX512-NEXT: vpxor %ymm4, %ymm7, %ymm4
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512-NEXT: vpand %ymm7, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm9 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512-NEXT: vpand %ymm2, %ymm9, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm4 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm8
; AVX512-NEXT: vpmullw %ymm5, %ymm8, %ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512-NEXT: vpand %ymm2, %ymm11, %ymm12
; AVX512-NEXT: vpmullw %ymm5, %ymm12, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm8
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpand %ymm2, %ymm8, %ymm10
; AVX512-NEXT: vpmullw %ymm5, %ymm10, %ymm10
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512-NEXT: vpand %ymm2, %ymm13, %ymm2
; AVX512-NEXT: vpmullw %ymm2, %ymm5, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ ymm12 ^ ymm10
diff --git a/llvm/test/CodeGen/X86/clmul-vector-512.ll b/llvm/test/CodeGen/X86/clmul-vector-512.ll
index f367e7d6ca19d..2ad51492bd84c 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-512.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-512.ll
@@ -19,13 +19,12 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm2, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm6 & zmm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & zmm6)
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -33,27 +32,13 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
-; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
-; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpxorq %zmm4, %zmm5, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -61,27 +46,27 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpxorq %zmm5, %zmm4, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm5
; AVX512-NEXT: vextracti64x4 $1, %zmm5, %ymm6
; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpmullw %ymm5, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
; AVX512-NEXT: vpandn %ymm5, %ymm3, %ymm5
; AVX512-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm5
; AVX512-NEXT: vpsllw $8, %ymm5, %ymm5
@@ -89,27 +74,41 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm5 | zmm7)
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm4
-; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm6
-; AVX512-NEXT: vpmullw %ymm6, %ymm2, %ymm7
-; AVX512-NEXT: vpmullw %ymm4, %ymm0, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512-NEXT: vpandq %zmm3, %zmm7, %zmm7
-; AVX512-NEXT: vpandn %ymm4, %ymm3, %ymm4
-; AVX512-NEXT: vpmaddubsw %ymm4, %ymm0, %ymm4
-; AVX512-NEXT: vpsllw $8, %ymm4, %ymm4
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm7)
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
-; AVX512-NEXT: vpmaddubsw %ymm6, %ymm2, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm4 | zmm7)
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm6
+; AVX512-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512-NEXT: vpmullw %ymm7, %ymm2, %ymm8
+; AVX512-NEXT: vpmullw %ymm6, %ymm0, %ymm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512-NEXT: vpandn %ymm6, %ymm3, %ymm6
+; AVX512-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
+; AVX512-NEXT: vpsllw $8, %ymm6, %ymm6
+; AVX512-NEXT: vpandn %ymm7, %ymm3, %ymm7
+; AVX512-NEXT: vpmaddubsw %ymm7, %ymm2, %ymm7
+; AVX512-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm3 & zmm8)
+; AVX512-NEXT: vpxorq %zmm6, %zmm5, %zmm5
+; AVX512-NEXT: vpxorq %zmm5, %zmm4, %zmm4
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm5
; AVX512-NEXT: vpmullw %ymm5, %ymm2, %ymm6
; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-NEXT: vpandq %zmm3, %zmm6, %zmm6
; AVX512-NEXT: vpandn %ymm1, %ymm3, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
@@ -117,7 +116,8 @@ define <64 x i8> @clmul_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm2, %ymm1
; AVX512-NEXT: vpsllw $8, %ymm1, %ymm1
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm0 | zmm6)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm6)
+; AVX512-NEXT: vpxorq %zmm0, %zmm4, %zmm0
; AVX512-NEXT: retq
%res = call <64 x i8> @llvm.clmul.v64i8(<64 x i8> %a, <64 x i8> %b)
ret <64 x i8> %res
@@ -495,78 +495,78 @@ define <64 x i8> @clmulr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
-; AVX512VL-NEXT: vpor %ymm6, %ymm5, %ymm8
+; AVX512VL-NEXT: vpor %ymm6, %ymm5, %ymm10
; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm11, %ymm5
; AVX512VL-NEXT: vpand %ymm2, %ymm5, %ymm5
; AVX512VL-NEXT: vpshufb %ymm5, %ymm3, %ymm12
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
; AVX512VL-NEXT: vpandq %ymm16, %ymm12, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm8, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm10, %ymm8
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm10
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm9 & ymm5)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; AVX512VL-NEXT: vpandq %ymm17, %ymm12, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm9, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
-; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm8, %ymm9
-; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm14
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm10 ^ (ymm14 | ymm13)
+; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm10, %ymm7
+; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm9
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpandq %ymm17, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm8, %ymm10, %ymm13
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm10, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm13)
+; AVX512VL-NEXT: vpxor %ymm9, %ymm8, %ymm13
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; AVX512VL-NEXT: vpandq %ymm18, %ymm12, %ymm10
-; AVX512VL-NEXT: vpmullw %ymm10, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm10, %ymm5, %ymm10
-; AVX512VL-NEXT: vpmaddubsw %ymm10, %ymm8, %ymm10
-; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm14 ^ (ymm15 | ymm13)
+; AVX512VL-NEXT: vpandq %ymm18, %ymm12, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm9, %ymm10, %ymm14
+; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
+; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm10, %ymm9
+; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm15
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm14)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm19, %ymm12, %ymm12
-; AVX512VL-NEXT: vpmullw %ymm12, %ymm8, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm10, %ymm14
; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
-; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm8, %ymm12
+; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm10, %ymm12
; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm15 ^ (ymm12 | ymm13)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm5 & ymm14)
+; AVX512VL-NEXT: vpxor %ymm12, %ymm15, %ymm12
+; AVX512VL-NEXT: vpxor %ymm12, %ymm13, %ymm14
; AVX512VL-NEXT: vpand %ymm2, %ymm11, %ymm11
-; AVX512VL-NEXT: vpshufb %ymm11, %ymm4, %ymm14
+; AVX512VL-NEXT: vpshufb %ymm11, %ymm4, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm11 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT: vpand %ymm11, %ymm14, %ymm13
-; AVX512VL-NEXT: vpmullw %ymm13, %ymm8, %ymm15
-; AVX512VL-NEXT: vpand %ymm5, %ymm15, %ymm15
+; AVX512VL-NEXT: vpand %ymm6, %ymm11, %ymm12
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm10, %ymm13
+; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
+; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm10, %ymm12
+; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm15
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm13)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpand %ymm6, %ymm12, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm13, %ymm10, %ymm7
; AVX512VL-NEXT: vpandn %ymm13, %ymm5, %ymm13
-; AVX512VL-NEXT: vpmaddubsw %ymm13, %ymm8, %ymm13
+; AVX512VL-NEXT: vpmaddubsw %ymm13, %ymm10, %ymm13
; AVX512VL-NEXT: vpsllw $8, %ymm13, %ymm13
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm12 ^ (ymm13 | ymm15)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512VL-NEXT: vpand %ymm12, %ymm14, %ymm15
-; AVX512VL-NEXT: vpmullw %ymm15, %ymm8, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm13, %ymm15, %ymm7
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpand %ymm6, %ymm13, %ymm15
+; AVX512VL-NEXT: vpmullw %ymm15, %ymm10, %ymm8
; AVX512VL-NEXT: vpandn %ymm15, %ymm5, %ymm15
-; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm8, %ymm15
+; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm10, %ymm15
; AVX512VL-NEXT: vpsllw $8, %ymm15, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm13 ^ (ymm15 | ymm6)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512VL-NEXT: vpand %ymm13, %ymm14, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm7
-; AVX512VL-NEXT: vpand %ymm5, %ymm7, %ymm7
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm15, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm14, %ymm7
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpand %ymm6, %ymm15, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm8
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm10, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm15 ^ (ymm6 | ymm7)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512VL-NEXT: vpand %ymm15, %ymm14, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm8, %ymm14
-; AVX512VL-NEXT: vpand %ymm5, %ymm14, %ymm14
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm8
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 | ymm14)
-; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm10
+; AVX512VL-NEXT: vpand %ymm2, %ymm10, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm14
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
@@ -578,66 +578,66 @@ define <64 x i8> @clmulr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
; AVX512VL-NEXT: vpandq %ymm16, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm9 & ymm5)
-; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm0, %ymm9, %ymm10
-; AVX512VL-NEXT: vpand %ymm5, %ymm10, %ymm10
-; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
-; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm0, %ymm9
-; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm9
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm7 ^ (ymm9 | ymm10)
-; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm10
-; AVX512VL-NEXT: vpand %ymm5, %ymm10, %ymm10
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm9 ^ (ymm7 | ymm10)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
; AVX512VL-NEXT: vpandq %ymm19, %ymm6, %ymm6
; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm9)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm9)
-; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm9)
-; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm9
-; AVX512VL-NEXT: vpand %ymm5, %ymm9, %ymm9
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm9)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm1, %ymm15, %ymm1
-; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512VL-NEXT: vpandn %ymm1, %ymm5, %ymm1
; AVX512VL-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm7 ^ (ymm0 | ymm6)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm0, %ymm6, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm14, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsrlw $4, %ymm8, %ymm4
+; AVX512VL-NEXT: vpsrlw $4, %ymm10, %ymm4
; AVX512VL-NEXT: vpand %ymm2, %ymm4, %ymm4
; AVX512VL-NEXT: vpshufb %ymm4, %ymm3, %ymm4
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
@@ -840,54 +840,54 @@ define <32 x i16> @clmulr_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
; AVX512VL-NEXT: vpandq %ymm22, %ymm7, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpxor %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm24, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm11
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm25, %ymm7, %ymm12
; AVX512VL-NEXT: vpmullw %ymm6, %ymm12, %ymm12
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm11
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm8
; AVX512VL-NEXT: vpshufb %ymm8, %ymm5, %ymm8
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpandq %ymm26, %ymm8, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpandq %ymm27, %ymm8, %ymm14
; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm14
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 ^ ymm12 ^ ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm12, %ymm8, %ymm13
; AVX512VL-NEXT: vpmullw %ymm6, %ymm13, %ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512VL-NEXT: vpand %ymm13, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 ^ ymm14 ^ ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpand %ymm7, %ymm14, %ymm15
; AVX512VL-NEXT: vpmullw %ymm6, %ymm15, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
; AVX512VL-NEXT: vpand %ymm7, %ymm15, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm11
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 ^ ymm9 ^ ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
; AVX512VL-NEXT: vpandq %ymm16, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
; AVX512VL-NEXT: vpandq %ymm17, %ymm7, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 ^ ymm11 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
; AVX512VL-NEXT: vpandq %ymm19, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
; AVX512VL-NEXT: vpandq %ymm20, %ymm8, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm7 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
; AVX512VL-NEXT: vpandq %ymm21, %ymm8, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm23 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm23 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
; AVX512VL-NEXT: vpandq %ymm23, %ymm8, %ymm8
; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm6
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 ^ ymm10 ^ ymm7
@@ -1364,78 +1364,78 @@ define <64 x i8> @clmulh_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm5
; AVX512VL-NEXT: vpand %ymm2, %ymm5, %ymm5
; AVX512VL-NEXT: vpshufb %ymm5, %ymm4, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
; AVX512VL-NEXT: vpandq %ymm16, %ymm10, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm9
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm8 & ymm5)
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512VL-NEXT: vpandq %ymm17, %ymm10, %ymm8
; AVX512VL-NEXT: vpmullw %ymm8, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm14, %ymm8
-; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm9 ^ (ymm12 | ymm11)
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm11)
+; AVX512VL-NEXT: vpxor %ymm9, %ymm8, %ymm11
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm18, %ymm10, %ymm9
-; AVX512VL-NEXT: vpmullw %ymm9, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
+; AVX512VL-NEXT: vpmullw %ymm9, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm9, %ymm5, %ymm9
; AVX512VL-NEXT: vpmaddubsw %ymm9, %ymm14, %ymm9
; AVX512VL-NEXT: vpsllw $8, %ymm9, %ymm13
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm12 ^ (ymm13 | ymm11)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm5 & ymm12)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm9 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpand %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpmullw %ymm10, %ymm14, %ymm11
-; AVX512VL-NEXT: vpand %ymm5, %ymm11, %ymm11
+; AVX512VL-NEXT: vpmullw %ymm10, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm10, %ymm5, %ymm10
; AVX512VL-NEXT: vpmaddubsw %ymm10, %ymm14, %ymm10
-; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm12
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 | ymm11)
+; AVX512VL-NEXT: vpsllw $8, %ymm10, %ymm10
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm5 & ymm12)
+; AVX512VL-NEXT: vpxor %ymm10, %ymm13, %ymm10
+; AVX512VL-NEXT: vpxor %ymm10, %ymm11, %ymm13
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm10
; AVX512VL-NEXT: vpshufb %ymm10, %ymm3, %ymm15
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm10 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpand %ymm10, %ymm15, %ymm11
-; AVX512VL-NEXT: vpmullw %ymm11, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm11, %ymm14, %ymm12
; AVX512VL-NEXT: vpandn %ymm11, %ymm5, %ymm11
; AVX512VL-NEXT: vpmaddubsw %ymm11, %ymm14, %ymm11
; AVX512VL-NEXT: vpsllw $8, %ymm11, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm12 ^ (ymm6 | ymm13)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm12)
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm11 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpand %ymm11, %ymm15, %ymm12
-; AVX512VL-NEXT: vpmullw %ymm12, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
+; AVX512VL-NEXT: vpmullw %ymm12, %ymm14, %ymm7
; AVX512VL-NEXT: vpandn %ymm12, %ymm5, %ymm12
; AVX512VL-NEXT: vpmaddubsw %ymm12, %ymm14, %ymm12
-; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm13)
+; AVX512VL-NEXT: vpsllw $8, %ymm12, %ymm12
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm12, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512VL-NEXT: vpand %ymm12, %ymm15, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm13
-; AVX512VL-NEXT: vpand %ymm5, %ymm13, %ymm13
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm14, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm13)
+; AVX512VL-NEXT: vpand %ymm12, %ymm15, %ymm7
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
+; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
+; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
+; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm13, %ymm6
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm13, %ymm15, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm15
-; AVX512VL-NEXT: vpand %ymm5, %ymm15, %ymm15
+; AVX512VL-NEXT: vpmullw %ymm7, %ymm14, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm14, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm15)
-; AVX512VL-NEXT: vpand %ymm2, %ymm7, %ymm6
-; AVX512VL-NEXT: vpshufb %ymm6, %ymm3, %ymm6
-; AVX512VL-NEXT: vpsrlw $4, %ymm7, %ymm7
-; AVX512VL-NEXT: vpand %ymm2, %ymm7, %ymm7
-; AVX512VL-NEXT: vpshufb %ymm7, %ymm4, %ymm7
-; AVX512VL-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm7
+; AVX512VL-NEXT: vpshufb %ymm7, %ymm3, %ymm7
+; AVX512VL-NEXT: vpsrlw $4, %ymm6, %ymm6
+; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
+; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
+; AVX512VL-NEXT: vpor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpsrlw $1, %ymm6, %ymm14
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm6
@@ -1449,62 +1449,62 @@ define <64 x i8> @clmulh_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512VL-NEXT: vpand %ymm2, %ymm6, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm4, %ymm6
; AVX512VL-NEXT: vpandq %ymm16, %ymm6, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm15
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm15 & ymm5)
-; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm15
-; AVX512VL-NEXT: vpmullw %ymm0, %ymm15, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm15, %ymm5, %ymm15
-; AVX512VL-NEXT: vpmaddubsw %ymm15, %ymm0, %ymm15
-; AVX512VL-NEXT: vpsllw $8, %ymm15, %ymm15
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 | ymm8)
-; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm15 ^ (ymm7 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpandq %ymm17, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm15
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm15)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpandq %ymm18, %ymm6, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm15
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm15)
; AVX512VL-NEXT: vpand %ymm6, %ymm9, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm9
; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX512VL-NEXT: vpxor %ymm6, %ymm7, %ymm6
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm3, %ymm1
; AVX512VL-NEXT: vpand %ymm1, %ymm10, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm8)
-; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm6
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm6, %ymm5, %ymm6
-; AVX512VL-NEXT: vpmaddubsw %ymm6, %ymm0, %ymm6
-; AVX512VL-NEXT: vpsllw $8, %ymm6, %ymm6
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm6 | ymm8)
-; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm7
-; AVX512VL-NEXT: vpmullw %ymm7, %ymm0, %ymm8
-; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT: vpandn %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT: vpmaddubsw %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT: vpsllw $8, %ymm7, %ymm7
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm6 ^ (ymm7 | ymm8)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm5 & ymm8)
+; AVX512VL-NEXT: vpand %ymm1, %ymm11, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpand %ymm1, %ymm12, %ymm8
+; AVX512VL-NEXT: vpmullw %ymm0, %ymm8, %ymm9
+; AVX512VL-NEXT: vpandn %ymm8, %ymm5, %ymm8
+; AVX512VL-NEXT: vpmaddubsw %ymm8, %ymm0, %ymm8
+; AVX512VL-NEXT: vpsllw $8, %ymm8, %ymm8
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm5 & ymm9)
+; AVX512VL-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX512VL-NEXT: vpxor %ymm7, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm1, %ymm13, %ymm1
-; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm6
-; AVX512VL-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm7
; AVX512VL-NEXT: vpandn %ymm1, %ymm5, %ymm1
; AVX512VL-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm7 ^ (ymm0 | ymm6)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm7)
+; AVX512VL-NEXT: vpxor %ymm0, %ymm6, %ymm0
; AVX512VL-NEXT: vpand %ymm2, %ymm0, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm3, %ymm1
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
@@ -1709,54 +1709,54 @@ define <32 x i16> @clmulh_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
; AVX512VL-NEXT: vpandq %ymm23, %ymm7, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpxor %ymm9, %ymm10, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm24 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; AVX512VL-NEXT: vpandq %ymm24, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm11
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm25 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; AVX512VL-NEXT: vpandq %ymm25, %ymm7, %ymm12
; AVX512VL-NEXT: vpmullw %ymm6, %ymm12, %ymm12
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm12 = ymm12 ^ ymm10 ^ ymm11
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm8
; AVX512VL-NEXT: vpshufb %ymm8, %ymm4, %ymm8
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm26 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
; AVX512VL-NEXT: vpandq %ymm26, %ymm8, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm27 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; AVX512VL-NEXT: vpandq %ymm27, %ymm8, %ymm14
; AVX512VL-NEXT: vpmullw %ymm6, %ymm14, %ymm14
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 ^ ymm12 ^ ymm13
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512VL-NEXT: vpand %ymm12, %ymm8, %ymm13
; AVX512VL-NEXT: vpmullw %ymm6, %ymm13, %ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm13 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
; AVX512VL-NEXT: vpand %ymm13, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 ^ ymm14 ^ ymm15
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm14 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpand %ymm7, %ymm14, %ymm15
; AVX512VL-NEXT: vpmullw %ymm6, %ymm15, %ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [512,512,512,512,512,512,512,512,512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm15 = [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
; AVX512VL-NEXT: vpand %ymm7, %ymm15, %ymm11
; AVX512VL-NEXT: vpmullw %ymm6, %ymm11, %ymm11
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 ^ ymm9 ^ ymm10
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm16 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
; AVX512VL-NEXT: vpandq %ymm16, %ymm7, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048,2048]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024]
; AVX512VL-NEXT: vpandq %ymm17, %ymm7, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 ^ ymm11 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm18 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
; AVX512VL-NEXT: vpandq %ymm18, %ymm8, %ymm9
; AVX512VL-NEXT: vpmullw %ymm6, %ymm9, %ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192,8192]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm19 = [4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096,4096]
; AVX512VL-NEXT: vpandq %ymm19, %ymm8, %ymm10
; AVX512VL-NEXT: vpmullw %ymm6, %ymm10, %ymm10
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 ^ ymm7 ^ ymm9
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm20 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
; AVX512VL-NEXT: vpandq %ymm20, %ymm8, %ymm7
; AVX512VL-NEXT: vpmullw %ymm7, %ymm6, %ymm7
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768,32768]
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm21 = [16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384,16384]
; AVX512VL-NEXT: vpandq %ymm21, %ymm8, %ymm8
; AVX512VL-NEXT: vpmullw %ymm6, %ymm8, %ymm6
; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 ^ ymm10 ^ ymm7
diff --git a/llvm/test/CodeGen/X86/combine-fround.ll b/llvm/test/CodeGen/X86/combine-fround.ll
index 42dbaf234dbc7..f88055fc591e1 100644
--- a/llvm/test/CodeGen/X86/combine-fround.ll
+++ b/llvm/test/CodeGen/X86/combine-fround.ll
@@ -45,9 +45,9 @@ define <4 x double> @concat_round_v4f64_v2f64(<2 x double> %a0, <2 x double> %a1
; AVX512-LABEL: concat_round_v4f64_v2f64:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm0) | m64bcst
; AVX512-NEXT: vaddpd %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -98,9 +98,9 @@ define <8 x float> @concat_round_v8f32_v4f32(<4 x float> %a0, <4 x float> %a1) {
; AVX512-LABEL: concat_round_v8f32_v4f32:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 | (ymm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm2 = (ymm2 & ymm0) | m32bcst
; AVX512-NEXT: vaddps %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -178,9 +178,9 @@ define <8 x double> @concat_round_v8f64_v2f64(<2 x double> %a0, <2 x double> %a1
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -262,9 +262,9 @@ define <16 x float> @concat_round_v16f32_v4f32(<4 x float> %a0, <4 x float> %a1,
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -335,9 +335,9 @@ define <8 x double> @concat_round_v8f64_v4f64(<4 x double> %a0, <4 x double> %a1
; AVX512-LABEL: concat_round_v8f64_v4f64:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm0 & m64bcst)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = (zmm2 & zmm0) | m64bcst
; AVX512-NEXT: vaddpd %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -404,9 +404,9 @@ define <16 x float> @concat_round_v16f32_v8f32(<8 x float> %a0, <8 x float> %a1)
; AVX512-LABEL: concat_round_v16f32_v8f32:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm2 = zmm2 | (zmm0 & m32bcst)
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm2 = (zmm2 & zmm0) | m32bcst
; AVX512-NEXT: vaddps %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/combine-or-shuffle.ll b/llvm/test/CodeGen/X86/combine-or-shuffle.ll
index 02f1c96f8b796..2c00a4fbad2c7 100644
--- a/llvm/test/CodeGen/X86/combine-or-shuffle.ll
+++ b/llvm/test/CodeGen/X86/combine-or-shuffle.ll
@@ -801,8 +801,8 @@ define <2 x i64> @or_and_v2i64(<2 x i64> %a0) {
;
; AVX512-LABEL: or_and_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm1 = [7,7]
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 & (xmm0 | m64bcst)
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst & (xmm0 | xmm1)
; AVX512-NEXT: retq
%1 = and <2 x i64> %a0, <i64 7, i64 7>
%2 = or <2 x i64> %1, <i64 3, i64 3>
@@ -830,8 +830,8 @@ define <4 x i32> @or_and_v4i32(<4 x i32> %a0) {
;
; AVX512-LABEL: or_and_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [3,3,15,7]
-; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 | mem)
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [3,2,4294967295,2]
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = mem & (xmm0 | xmm1)
; AVX512-NEXT: retq
%1 = and <4 x i32> %a0, <i32 1, i32 3, i32 5, i32 7>
%2 = or <4 x i32> %1, <i32 3, i32 2, i32 15, i32 2>
diff --git a/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll b/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
index c3822b3f3021f..5ac83ca7a4f13 100644
--- a/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
+++ b/llvm/test/CodeGen/X86/elementwise-store-of-scalar-splat.ll
@@ -651,8 +651,7 @@ define void @vec256_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec256_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovdqa %xmm0, 16(%rsi)
; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
; AVX512-NEXT: retq
@@ -1092,8 +1091,7 @@ define void @vec384_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec384_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
; AVX512-NEXT: vmovdqa %xmm0, 16(%rsi)
; AVX512-NEXT: vmovdqa %xmm0, 32(%rsi)
@@ -1658,8 +1656,7 @@ define void @vec512_i128(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec512_i128:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
; AVX512-NEXT: vzeroupper
@@ -1731,8 +1728,7 @@ define void @vec512_i256(ptr %in.elt.ptr, ptr %out.vec.ptr) nounwind {
;
; AVX512-LABEL: vec512_i256:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX512-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512-NEXT: vmovdqa %ymm0, 32(%rsi)
; AVX512-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 8abd68701676c..4f343bdd141ab 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -1970,7 +1970,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vcvtph2ps %xmm2, %xmm4
; AVX512BF16-NEXT: vmaxss %xmm3, %xmm4, %xmm5
; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 & (xmm4 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 & (xmm2 | xmm4)
; AVX512BF16-NEXT: vcmpunordss %xmm3, %xmm3, %k1
; AVX512BF16-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm5, %xmm3
@@ -1979,7 +1979,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vshufps {{.*#+}} xmm5 = xmm0[3,3,3,3]
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -1989,7 +1989,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -1998,7 +1998,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vshufpd {{.*#+}} xmm6 = xmm0[1,0]
; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2009,7 +2009,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrlq $48, %xmm0, %xmm5
; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5
; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm5)
; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1
; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -2018,7 +2018,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2026,7 +2026,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vcvtph2ps %xmm1, %xmm5
; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm6
; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm2 | xmm6)
; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1
; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5
@@ -2035,7 +2035,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
; AVX512BF16-NEXT: vpsrld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm0
; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm6
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm0 | xmm2)
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm2 | xmm0)
; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512BF16-NEXT: vmovss %xmm0, %xmm6, %xmm6 {%k1}
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm0
diff --git a/llvm/test/CodeGen/X86/fp-round.ll b/llvm/test/CodeGen/X86/fp-round.ll
index 8595b63fc8107..ea9179f6cc61e 100644
--- a/llvm/test/CodeGen/X86/fp-round.ll
+++ b/llvm/test/CodeGen/X86/fp-round.ll
@@ -51,8 +51,8 @@ define half @round_f16(half %h) {
; AVX512F-LABEL: round_f16:
; AVX512F: # %bb.0: # %entry
; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vcvtps2ph $4, %xmm0, %xmm0
@@ -60,9 +60,9 @@ define half @round_f16(half %h) {
;
; AVX512FP16-LABEL: round_f16:
; AVX512FP16: ## %bb.0: ## %entry
-; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & xmm1)
+; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; AVX512FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm2 = (xmm2 & xmm0) | xmm1
; AVX512FP16-NEXT: vaddsh %xmm2, %xmm0, %xmm0
; AVX512FP16-NEXT: vrndscalesh $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -97,16 +97,16 @@ define float @round_f32(float %x) {
;
; AVX512F-LABEL: round_f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512FP16-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -141,16 +141,16 @@ define double @round_f64(double %x) {
;
; AVX512F-LABEL: round_f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512F-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundsd $11, %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512FP16-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundsd $11, %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -207,16 +207,16 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
;
; AVX512F-LABEL: round_v4f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512F-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundps $11, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v4f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} xmm1 = (xmm1 & xmm0) | m32bcst
; AVX512FP16-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundps $11, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -261,16 +261,16 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
;
; AVX512F-LABEL: round_v2f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512F-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vroundpd $11, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v2f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} xmm1 = (xmm1 & xmm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512FP16-NEXT: vroundpd $11, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
@@ -355,16 +355,16 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
;
; AVX512F-LABEL: round_v8f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm1 = (ymm1 & ymm0) | m32bcst
; AVX512F-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v8f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} ymm1 = (ymm1 & ymm0) | m32bcst
; AVX512FP16-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512FP16-NEXT: vroundps $11, %ymm0, %ymm0
; AVX512FP16-NEXT: retq
@@ -425,16 +425,16 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
;
; AVX512F-LABEL: round_v4f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ymm0) | m64bcst
; AVX512F-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v4f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ymm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512FP16-NEXT: vroundpd $11, %ymm0, %ymm0
; AVX512FP16-NEXT: retq
@@ -581,16 +581,16 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
;
; AVX512F-LABEL: round_v16f32:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512F-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512F-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v16f32:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT: vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
+; AVX512FP16-NEXT: vpbroadcastd {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogd {{.*#+}} zmm1 = (zmm1 & zmm0) | m32bcst
; AVX512FP16-NEXT: vaddps %zmm1, %zmm0, %zmm0
; AVX512FP16-NEXT: vrndscaleps $11, %zmm0, %zmm0
; AVX512FP16-NEXT: retq
@@ -689,16 +689,16 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
;
; AVX512F-LABEL: round_v8f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512F-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512F-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512F-NEXT: retq
;
; AVX512FP16-LABEL: round_v8f64:
; AVX512FP16: ## %bb.0:
-; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
+; AVX512FP16-NEXT: vpbroadcastq {{.*#+}} zmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm0) | m64bcst
; AVX512FP16-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512FP16-NEXT: vrndscalepd $11, %zmm0, %zmm0
; AVX512FP16-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/fshl-splat-undef.ll b/llvm/test/CodeGen/X86/fshl-splat-undef.ll
index a2d0345077b87..09fc9d48bbd10 100644
--- a/llvm/test/CodeGen/X86/fshl-splat-undef.ll
+++ b/llvm/test/CodeGen/X86/fshl-splat-undef.ll
@@ -21,10 +21,10 @@ define void @test_fshl(<8 x i64> %lo, <8 x i64> %hi, ptr %arr) {
; CHECK-LABEL: test_fshl:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT: vpsllq $12, %zmm1, %zmm1
; CHECK-NEXT: vpsrlq $52, %zmm0, %zmm0
-; CHECK-NEXT: vpternlogq $168, {{\.?LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm0
-; CHECK-NEXT: vmovdqa64 %zmm0, (%eax)
+; CHECK-NEXT: vpsllq $12, %zmm1, %zmm1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm1 = mem & (zmm1 | zmm0)
+; CHECK-NEXT: vmovdqa64 %zmm1, (%eax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retl
entry:
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index ceccfede3d557..1a60fbede5f86 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -1026,9 +1026,9 @@ define <32 x i8> @splatvar_ashr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; GFNIAVX512VL-LABEL: splatvar_ashr_v32i8:
; GFNIAVX512VL: # %bb.0:
; GFNIAVX512VL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; GFNIAVX512VL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; GFNIAVX512VL-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -1987,29 +1987,29 @@ define <64 x i8> @splatvar_ashr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; GFNIAVX512VL-LABEL: splatvar_ashr_v64i8:
; GFNIAVX512VL: # %bb.0:
-; GFNIAVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; GFNIAVX512VL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm3 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
+; GFNIAVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm3, %ymm3
; GFNIAVX512VL-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpsrlw $8, %xmm4, %xmm4
; GFNIAVX512VL-NEXT: vpbroadcastb %xmm4, %ymm4
-; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 & ymm4)
-; GFNIAVX512VL-NEXT: vpsubb %ymm3, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & ymm4)
+; GFNIAVX512VL-NEXT: vpsubb %ymm2, %ymm3, %ymm3
; GFNIAVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & ymm4)
-; GFNIAVX512VL-NEXT: vpsubb %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & ymm4)
+; GFNIAVX512VL-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; GFNIAVX512VL-NEXT: retq
;
; GFNIAVX512BW-LABEL: splatvar_ashr_v64i8:
; GFNIAVX512BW: # %bb.0:
; GFNIAVX512BW-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; GFNIAVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; GFNIAVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
index dada1726be424..3d2538b485ff5 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
@@ -149,9 +149,9 @@ define <8 x i1> @andnot_ne_v8i16_todo_no_splat(<8 x i16> %x) nounwind {
; AVX512-LABEL: andnot_ne_v8i16_todo_no_splat:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16_todo_no_splat:
@@ -184,7 +184,7 @@ define <8 x i1> @andnot_ne_v8i16(<8 x i16> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16:
@@ -215,9 +215,9 @@ define <16 x i1> @andnot_ne_v16i8_fail_max_not_n1(<16 x i8> %x) nounwind {
; AVX512-LABEL: andnot_ne_v16i8_fail_max_not_n1:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8_fail_max_not_n1:
@@ -250,7 +250,7 @@ define <16 x i1> @andnot_ne_v16i8(<16 x i8> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8:
@@ -307,9 +307,8 @@ define <8 x i1> @addand_ne_v8i16_fail(<8 x i16> %x) nounwind {
; AVX512-LABEL: addand_ne_v8i16_fail:
; AVX512: # %bb.0:
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $86, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: addand_ne_v8i16_fail:
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..f921eb0247e7f 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -858,12 +858,12 @@ define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm3, %ymm5, %ymm3
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm1, %ymm1
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm5 & ymm4)
; CHECK-SKX-NOVBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm3
; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm2, %ymm5, %ymm2
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm0, %ymm0
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm3)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-SKX-NOVBMI-NEXT: vzeroupper
@@ -901,12 +901,12 @@ define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-AVX512-NEXT: vpandn %ymm3, %ymm5, %ymm3
; CHECK-AVX512-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpsllw $8, %ymm1, %ymm1
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm5 & ymm4)
; CHECK-AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm3
; CHECK-AVX512-NEXT: vpandn %ymm2, %ymm5, %ymm2
; CHECK-AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vpsllw $8, %ymm0, %ymm0
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm5 & ymm3)
; CHECK-AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-AVX512-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-AVX512-NEXT: vzeroupper
@@ -949,7 +949,7 @@ define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-SKX-NOVBMI-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %zmm0, %zmm0
-; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; CHECK-SKX-NOVBMI-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT: vzeroupper
; CHECK-SKX-NOVBMI-NEXT: retq
@@ -976,7 +976,7 @@ define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="
; CHECK-AVX512-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; CHECK-AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vpsllw $8, %zmm0, %zmm0
-; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; CHECK-AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 625d18bcf4edf..8fbef0d45c41f 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -956,7 +956,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind {
; AVX512F-NEXT: vpmaddubsw %ymm1, %ymm3, %ymm1
; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm4 & zmm5)
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm5 & zmm4)
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: mul_v64i8:
@@ -966,7 +966,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind {
; AVX512BW-NEXT: vpandnq %zmm1, %zmm3, %zmm1
; AVX512BW-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0
-; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
+; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm3 & zmm2)
; AVX512BW-NEXT: retq
entry:
%A = mul <64 x i8> %i, %j
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll b/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
index c9e48f817fb44..a8bf9f0190fed 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-wide-mul.ll
@@ -63,7 +63,7 @@ define <32 x i8> @test_mul_32i8(<32 x i8> %a, <32 x i8> %b) {
; AVX256BW-NEXT: vpandn %ymm1, %ymm3, %ymm1
; AVX256BW-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX256BW-NEXT: vpsllw $8, %ymm0, %ymm0
-; AVX256BW-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm3)
+; AVX256BW-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm2)
; AVX256BW-NEXT: retq
;
; AVX512BWVL-LABEL: test_mul_32i8:
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index 69e6ff7770ebe..625775d64a2b5 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -363,13 +363,13 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_min(<16 x i8> %x) {
; SSE-LABEL: unsigned_sat_constant_v16i8_using_min:
; SSE: # %bb.0:
; SSE-NEXT: pminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
; SSE-NEXT: retq
;
; AVX-LABEL: unsigned_sat_constant_v16i8_using_min:
; AVX: # %bb.0:
; AVX-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
; AVX-NEXT: retq
%c = icmp ult <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
%s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
@@ -415,19 +415,19 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_min(<8 x i16> %x) {
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psubusw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE2-NEXT: psubw %xmm1, %xmm0
-; SSE2-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42,42,42,42,42]
; SSE2-NEXT: retq
;
; SSE4-LABEL: unsigned_sat_constant_v8i16_using_min:
; SSE4: # %bb.0:
; SSE4-NEXT: pminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42,42,42,42,42]
; SSE4-NEXT: retq
;
; AVX-LABEL: unsigned_sat_constant_v8i16_using_min:
; AVX: # %bb.0:
; AVX-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [42,42,42,42,42,42,42,42]
; AVX-NEXT: retq
%c = icmp ult <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
%s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
@@ -477,14 +477,14 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_min(<4 x i32> %x) {
; SSE2-NEXT: pandn %xmm0, %xmm2
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [42,42,42,42]
; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE4-LABEL: unsigned_sat_constant_v4i32_using_min:
; SSE4: # %bb.0:
; SSE4-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42]
; SSE4-NEXT: retq
;
; AVX2-LABEL: unsigned_sat_constant_v4i32_using_min:
@@ -521,7 +521,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
; SSE4: # %bb.0:
; SSE4-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42]
; SSE4-NEXT: retq
;
; AVX2-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
@@ -556,7 +556,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
; SSE4: # %bb.0:
; SSE4-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42,42,42]
; SSE4-NEXT: retq
;
; AVX2-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
@@ -591,13 +591,13 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat(<4 x i32
; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
; SSE4: # %bb.0:
; SSE4-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [43,44,45,46]
; SSE4-NEXT: retq
;
; AVX-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
; AVX: # %bb.0:
; AVX-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [43,44,45,46]
; AVX-NEXT: retq
%a = add <4 x i32> %x, <i32 43, i32 44, i32 45, i32 46>
%c = icmp ugt <4 x i32> %x, <i32 -44, i32 -45, i32 -46, i32 -47>
@@ -621,7 +621,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42]
; SSE2-NEXT: retq
;
; SSE41-LABEL: unsigned_sat_constant_v2i64_using_min:
@@ -639,7 +639,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
; SSE41-NEXT: por %xmm5, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
-; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [42,42]
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
@@ -650,7 +650,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE42-NEXT: blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [42,42]
; SSE42-NEXT: movdqa %xmm1, %xmm0
; SSE42-NEXT: retq
;
@@ -659,7 +659,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
; AVX2-NEXT: vblendvpd %xmm1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [42,42]
; AVX2-NEXT: retq
;
; AVX512-LABEL: unsigned_sat_constant_v2i64_using_min:
@@ -715,7 +715,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pxor %xmm1, %xmm2
-; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42]
; SSE42-NEXT: pxor %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq %xmm1, %xmm2
; SSE42-NEXT: por %xmm2, %xmm0
@@ -725,7 +725,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
; AVX2: # %bb.0:
; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [42,42]
; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm1
; AVX2-NEXT: vpcmpgtq %xmm1, %xmm2, %xmm1
; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
@@ -774,7 +774,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pxor %xmm1, %xmm2
-; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [42,42]
; SSE42-NEXT: pxor %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq %xmm1, %xmm2
; SSE42-NEXT: por %xmm2, %xmm0
@@ -784,7 +784,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
; AVX2: # %bb.0:
; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [42,42]
; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm1
; AVX2-NEXT: vpcmpgtq %xmm1, %xmm2, %xmm1
; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
@@ -874,12 +874,11 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
;
; AVX512-LABEL: unsigned_sat_variable_v16i8_using_cmp_notval:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm3
+; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~xmm1
; AVX512-NEXT: vpminub %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm3 | (xmm0 ^ xmm2)
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 | ~xmm0
; AVX512-NEXT: retq
%noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
%a = add <16 x i8> %x, %y
@@ -981,12 +980,11 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
;
; AVX512-LABEL: unsigned_sat_variable_v8i16_using_cmp_notval:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm3
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~xmm1
; AVX512-NEXT: vpminuw %xmm1, %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm3 | (xmm0 ^ xmm2)
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 | ~xmm0
; AVX512-NEXT: retq
%noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
%a = add <8 x i16> %x, %y
diff --git a/llvm/test/CodeGen/X86/select-big-integer.ll b/llvm/test/CodeGen/X86/select-big-integer.ll
index f253ab8a6cf6f..324008aadef1c 100644
--- a/llvm/test/CodeGen/X86/select-big-integer.ll
+++ b/llvm/test/CodeGen/X86/select-big-integer.ll
@@ -140,8 +140,7 @@ define void @test_not_i128(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i128:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor (%rsi), %xmm0, %xmm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512VL-NEXT: negb %dl
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqa32 (%rdi), %xmm0 {%k1}
@@ -317,8 +316,7 @@ define void @test_not_i256(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX512VL-NEXT: vpxor (%rsi), %ymm0, %ymm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
; AVX512VL-NEXT: negb %dl
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqu32 (%rdi), %ymm0 {%k1}
@@ -550,8 +548,7 @@ define void @test_not_i512(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512F-LABEL: test_not_i512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = -1
-; AVX512F-NEXT: vpxord (%rsi), %zmm0, %zmm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = ~mem
; AVX512F-NEXT: negl %edx
; AVX512F-NEXT: kmovw %edx, %k1
; AVX512F-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1}
@@ -560,8 +557,7 @@ define void @test_not_i512(ptr %p0, ptr %p1, i1 zeroext %a2, ptr %p3) nounwind {
;
; AVX512VL-LABEL: test_not_i512:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = -1
-; AVX512VL-NEXT: vpxord (%rsi), %zmm0, %zmm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = ~mem
; AVX512VL-NEXT: negl %edx
; AVX512VL-NEXT: kmovd %edx, %k1
; AVX512VL-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1}
diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
index d89cd74214ecb..5cffa737e2c8e 100644
--- a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
@@ -2345,9 +2345,8 @@ define <32 x i1> @pr51133(<32 x i8> %x, <32 x i8> %y) {
; CHECK-AVX512VL-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm2
; CHECK-AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm0, %ymm1, %ymm1
-; CHECK-AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (ymm1 | ymm2)
+; CHECK-AVX512VL-NEXT: vpcmpeqb %ymm0, %ymm1, %ymm0
+; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ~(ymm0 | ymm2)
; CHECK-AVX512VL-NEXT: retq
%rem = srem <32 x i8> %x, <i8 13, i8 5, i8 19, i8 34, i8 2, i8 8, i8 2, i8 88, i8 62, i8 62, i8 5, i8 7, i8 97, i8 2, i8 3, i8 60, i8 3, i8 87, i8 7, i8 6, i8 84, i8 -128, i8 127, i8 56, i8 114, i8 1, i8 50, i8 7, i8 2, i8 8, i8 97, i8 117>
%cmp = icmp ne <32 x i8> %rem, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
index 21915a0d50a77..9c58016fbd2b1 100644
--- a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
+++ b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
@@ -252,14 +252,22 @@ define void @vec128_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -319,14 +327,22 @@ define void @vec128_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -519,14 +535,22 @@ define void @vec128_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec128_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec128_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec128_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -822,15 +846,24 @@ define void @vec256_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -916,15 +949,24 @@ define void @vec256_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1124,14 +1166,31 @@ define void @vec256_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1166,14 +1225,31 @@ define void @vec256_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -1254,15 +1330,24 @@ define void @vec256_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec256_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec256_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1394,14 +1479,31 @@ define void @vec256_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1431,14 +1533,31 @@ define void @vec256_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -1626,14 +1745,31 @@ define void @vec256_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1793,14 +1929,31 @@ define void @vec256_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec256_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec256_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec256_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec256_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -1946,16 +2099,26 @@ define void @vec384_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v2i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v2i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i8> %in.subvec.not, <i8 -1, i8 -1>
store <2 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2067,16 +2230,26 @@ define void @vec384_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v2i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v2i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i16> %in.subvec.not, <i16 -1, i16 -1>
store <2 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2311,15 +2484,34 @@ define void @vec384_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -2359,15 +2551,34 @@ define void @vec384_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -3150,21 +3361,52 @@ define void @vec384_v3i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v3i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v3i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v3i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
store <3 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3281,24 +3523,55 @@ define void @vec384_v3f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v3f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
- %in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
- %in.subvec.int = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
- %in.subvec = bitcast <3 x i32> %in.subvec.int to <3 x float>
+; AVX1-LABEL: vec384_v3f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v3f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
+ %in.subvec.not = load <3 x i32>, ptr %in.subvec.ptr, align 64
+ %in.subvec.int = xor <3 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1>
+ %in.subvec = bitcast <3 x i32> %in.subvec.int to <3 x float>
store <3 x float> %in.subvec, ptr %out.subvec.ptr, align 64
%out.subvec0.ptr = getelementptr <3 x float>, ptr %out.vec.ptr, i64 0
store <3 x float> %in.subvec, ptr %out.subvec0.ptr, align 64
@@ -3360,19 +3633,32 @@ define void @vec384_v3i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v3i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v3i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <3 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1>
store <3 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3432,19 +3718,32 @@ define void @vec384_v3f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v3f64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v3f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v3f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <3 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <3 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1>
%in.subvec = bitcast <3 x i64> %in.subvec.int to <3 x double>
@@ -3543,16 +3842,26 @@ define void @vec384_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v4i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, (%rsi)
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v4i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovd %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovd %xmm0, (%rsi)
+; AVX512-NEXT: vpbroadcastd %xmm0, %ymm0
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1>
store <4 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3710,15 +4019,34 @@ define void @vec384_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -3752,15 +4080,34 @@ define void @vec384_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -4127,21 +4474,52 @@ define void @vec384_v6i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v6i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v6i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v6i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <6 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <6 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4205,19 +4583,32 @@ define void @vec384_v6i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v6i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v6i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <6 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
store <6 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4277,19 +4668,32 @@ define void @vec384_v6f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v6f32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v6f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v6f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <6 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <6 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <6 x i32> %in.subvec.int to <6 x float>
@@ -4510,15 +4914,34 @@ define void @vec384_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4633,21 +5056,52 @@ define void @vec384_v12i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE42-NEXT: movq %xmm0, 48(%rdx)
; SSE42-NEXT: retq
;
-; AVX-LABEL: vec384_v12i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rsi)
-; AVX-NEXT: vmovq %xmm0, (%rsi)
-; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdx)
-; AVX-NEXT: vmovq %xmm0, (%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 24(%rdx)
-; AVX-NEXT: vmovq %xmm0, 16(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 40(%rdx)
-; AVX-NEXT: vmovq %xmm0, 32(%rdx)
-; AVX-NEXT: vpextrd $2, %xmm0, 56(%rdx)
-; AVX-NEXT: vmovq %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v12i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX1-NEXT: vmovq %xmm0, (%rsi)
+; AVX1-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX1-NEXT: vmovq %xmm0, (%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX1-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX1-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v12i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v12i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rsi)
+; AVX512-NEXT: vmovq %xmm0, (%rsi)
+; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rdx)
+; AVX512-NEXT: vmovq %xmm0, (%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 24(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 16(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 40(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 32(%rdx)
+; AVX512-NEXT: vpextrd $2, %xmm0, 56(%rdx)
+; AVX512-NEXT: vmovq %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <12 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <12 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <12 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4711,19 +5165,32 @@ define void @vec384_v12i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v12i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v12i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v12i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <12 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <12 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <12 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4901,15 +5368,34 @@ define void @vec384_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec384_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec384_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec384_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -4971,19 +5457,32 @@ define void @vec384_v24i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec384_v24i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovq %xmm1, 16(%rdx)
-; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX2-NEXT: vmovq %xmm1, 48(%rdx)
-; AVX2-NEXT: vmovdqu %xmm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec384_v24i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec384_v24i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovq %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovq %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqu %xmm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <24 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <24 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <24 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5163,8 +5662,7 @@ define void @vec512_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512F-LABEL: vec512_v2i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsi)
; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
@@ -5174,8 +5672,7 @@ define void @vec512_v2i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512BW-LABEL: vec512_v2i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
; AVX512BW-NEXT: vpextrw $0, %xmm0, (%rsi)
; AVX512BW-NEXT: vpbroadcastw %xmm0, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -5331,8 +5828,7 @@ define void @vec512_v2i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
;
; AVX512-LABEL: vec512_v2i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovd %xmm0, (%rsi)
; AVX512-NEXT: vpbroadcastd %xmm0, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -5599,16 +6095,37 @@ define void @vec512_v2i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
store <2 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5653,16 +6170,37 @@ define void @vec512_v2f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2f64:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <2 x i64> %in.subvec.not, <i64 -1, i64 -1>
%in.subvec = bitcast <2 x i64> %in.subvec.int to <2 x double>
@@ -5717,18 +6255,43 @@ define void @vec512_v2i128(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; SSE2-NEXT: movdqa %xmm1, 32(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v2i128:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm1
-; AVX-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, 16(%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: vmovdqa %xmm1, 32(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v2i128:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm1
+; AVX1-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rsi)
+; AVX1-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: vmovdqa %xmm1, 32(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v2i128:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm1
+; AVX2-ONLY-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm1, 32(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v2i128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = ~mem
+; AVX512-NEXT: vmovdqa %xmm1, 16(%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm1, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm1, 48(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <2 x i128>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <2 x i128> %in.subvec.not, <i128 -1, i128 -1>
store <2 x i128> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -5857,8 +6420,7 @@ define void @vec512_v4i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.p
;
; AVX512-LABEL: vec512_v4i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
; AVX512-NEXT: vmovd %xmm0, (%rsi)
; AVX512-NEXT: vpbroadcastd %xmm0, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -6043,16 +6605,37 @@ define void @vec512_v4i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v4i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
store <4 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6090,16 +6673,37 @@ define void @vec512_v4f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v4f32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v4f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v4f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <4 x i32> %in.subvec.int to <4 x float>
@@ -6165,15 +6769,24 @@ define void @vec512_v4i64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v4i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v4i64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <4 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1, i64 -1>
store <4 x i64> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6234,15 +6847,24 @@ define void @vec512_v4f64(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v4f64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v4f64:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v4f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <4 x i64>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <4 x i64> %in.subvec.not, <i64 -1, i64 -1, i64 -1, i64 -1>
%in.subvec = bitcast <4 x i64> %in.subvec.int to <4 x double>
@@ -6492,16 +7114,37 @@ define void @vec512_v8i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v8i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <8 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6588,15 +7231,24 @@ define void @vec512_v8i32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v8i32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <8 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
store <8 x i32> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -6679,15 +7331,24 @@ define void @vec512_v8f32(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v8f32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v8f32:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v8f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <8 x i32>, ptr %in.subvec.ptr, align 64
%in.subvec.int = xor <8 x i32> %in.subvec.not, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%in.subvec = bitcast <8 x i32> %in.subvec.int to <8 x float>
@@ -6898,16 +7559,37 @@ define void @vec512_v16i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: retq
;
-; AVX-LABEL: vec512_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm0, (%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 32(%rdx)
-; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
-; AVX-NEXT: retq
+; AVX1-LABEL: vec512_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX1-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX1-NEXT: retq
+;
+; AVX2-ONLY-LABEL: vec512_v16i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %xmm0, %xmm0
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~mem
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 32(%rdx)
+; AVX512-NEXT: vmovdqa %xmm0, 48(%rdx)
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <16 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7078,15 +7760,24 @@ define void @vec512_v16i16(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v16i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v16i16:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v16i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <16 x i16>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <16 x i16> %in.subvec.not, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
store <16 x i16> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7393,15 +8084,24 @@ define void @vec512_v32i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: vec512_v32i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: vpxor (%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-ONLY-LABEL: vec512_v32i8:
+; AVX2-ONLY: # %bb.0:
+; AVX2-ONLY-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vpxor (%rdi), %ymm0, %ymm0
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX2-ONLY-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX2-ONLY-NEXT: vzeroupper
+; AVX2-ONLY-NEXT: retq
+;
+; AVX512-LABEL: vec512_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ~mem
+; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm0, 32(%rdx)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%in.subvec.not = load <32 x i8>, ptr %in.subvec.ptr, align 64
%in.subvec = xor <32 x i8> %in.subvec.not, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
store <32 x i8> %in.subvec, ptr %out.subvec.ptr, align 64
@@ -7412,4 +8112,6 @@ define void @vec512_v32i8(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec.
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; AVX2: {{.*}}
; SSSE3: {{.*}}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
index e5b19a5efe6f2..9a5c7def87e81 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
@@ -118,9 +118,8 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
; CHECK-AVX512VL: # %bb.0:
; CHECK-AVX512VL-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; CHECK-AVX512VL-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst | (xmm0 ^ xmm1)
+; CHECK-AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = m64bcst | ~xmm0
; CHECK-AVX512VL-NEXT: retq
%urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
%cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 425441b738399..b4301ecdc8337 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1692,7 +1692,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
; AVX512VL-NEXT: vpsllw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [128,64,32,16,8,4,2,1,128,1,2,4,8,16,32,64,128,64,32,16,8,4,2,1,128,1,2,4,8,16,32,64]
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | ymm1 | ymm2
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | ymm2 | ymm1
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: constant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 4c6cd5feaf593..febf5a00033c0 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -921,7 +921,7 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
; AVX512F-NEXT: vpmullw %ymm4, %ymm2, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
; AVX512F-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm3
+; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm1
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: constant_funnnel_v64i8:
@@ -963,7 +963,7 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
; AVX512VL-NEXT: vpmullw %ymm4, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm3
+; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm1
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: constant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
index 441fa54f98264..c7b5b6f98295c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
@@ -2613,7 +2613,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm10[1],xmm3[2,3],xmm10[4],xmm3[5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm3 & ymm11)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm11 & ymm3)
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm6
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
@@ -2836,7 +2836,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm10[1],xmm3[2,3],xmm10[4],xmm3[5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm3 & ymm11)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm11 & ymm3)
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm6
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
@@ -5489,7 +5489,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm7, %xmm7
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3],xmm9[4],xmm7[5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm7 & ymm14)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm14 & ymm7)
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm5, %xmm5
; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm0, %xmm0
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3],xmm5[4],xmm0[5,6,7]
@@ -6025,7 +6025,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm7, %xmm7
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3],xmm9[4],xmm7[5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm7 & ymm11)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm11 & ymm7)
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm5, %xmm5
; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm0, %xmm0
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3],xmm5[4],xmm0[5,6,7]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index 054df4c00e0a3..52468cad5280c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -3782,7 +3782,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -3798,7 +3798,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -3815,7 +3815,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512-NEXT: vmovdqa %ymm0, %ymm12
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -3829,7 +3829,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -3892,7 +3892,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -3908,7 +3908,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512-FCP-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -3925,7 +3925,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm12
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -3939,7 +3939,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512-FCP-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -4002,7 +4002,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -4018,7 +4018,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512DQ-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -4035,7 +4035,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm12
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -4049,7 +4049,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512DQ-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -4112,7 +4112,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm16 & ymm4)
; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
@@ -4128,7 +4128,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm8, %xmm7
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm16 & ymm7)
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm10, %xmm7
@@ -4145,7 +4145,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm16 & ymm12)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm12
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
@@ -4159,7 +4159,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm11, %xmm10, %xmm10
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm16 & ymm10)
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm10, %xmm2
@@ -7465,7 +7465,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7476,7 +7476,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -7533,7 +7533,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -7550,7 +7550,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -7719,7 +7719,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7730,7 +7730,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512-FCP-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -7787,7 +7787,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -7804,7 +7804,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -7973,7 +7973,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512DQ-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -7984,7 +7984,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512DQ-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -8041,7 +8041,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512DQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512DQ-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -8058,7 +8058,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -8227,7 +8227,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm4 & ymm17)
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
@@ -8238,7 +8238,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm21)
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm2
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
@@ -8295,7 +8295,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm18 & ymm4)
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm1
@@ -8312,7 +8312,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm18 & ymm9)
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 319735da0a6c4..e22d063d7cda0 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -5697,7 +5697,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm14[0,1,2],ymm10[3],ymm14[4,5],ymm10[6],ymm14[7,8,9,10],ymm10[11],ymm14[12,13],ymm10[14],ymm14[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpmovsxdq {{.*#+}} ymm18 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm8)
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm1[1,2,3,4,5,6,7],ymm14[8],ymm1[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm20
@@ -5724,7 +5724,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm3 ^ ymm9))
; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2],ymm10[3],ymm14[4,5,6],ymm10[7,8],ymm14[9,10],ymm10[11],ymm14[12,13,14],ymm10[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm1 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm1)
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm8[1,2,3,4,5,6,7],ymm14[8],ymm8[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm21
@@ -5750,7 +5750,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm3 ^ ymm9))
; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2,3],ymm10[4],ymm14[5,6],ymm10[7,8],ymm14[9,10,11],ymm10[12],ymm14[13,14],ymm10[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm1 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm1)
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm8[1,2,3,4,5,6,7],ymm14[8],ymm8[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa %ymm11, %ymm1
@@ -5775,7 +5775,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm3 ^ (ymm15 & (ymm9 ^ ymm3))
; AVX512-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm10[1],ymm15[2,3],ymm10[4],ymm15[5,6,7,8],ymm10[9],ymm15[10,11],ymm10[12],ymm15[13,14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm1 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm18 & ymm1)
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0],ymm8[1,2,3,4,5,6,7],ymm15[8],ymm8[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm13 = ymm17 ^ (ymm13 & (ymm2 ^ ymm17))
@@ -5797,7 +5797,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm9 ^ ymm3))
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm10[1],ymm0[2,3,4],ymm10[5],ymm0[6,7,8],ymm10[9],ymm0[10,11,12],ymm10[13],ymm0[14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm18)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm18 & ymm2)
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm4[1,2,3,4,5,6,7],ymm0[8],ymm4[9,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm19, (%rsi)
@@ -5894,7 +5894,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1,2],ymm8[3],ymm13[4,5],ymm8[6],ymm13[7,8,9,10],ymm8[11],ymm13[12,13],ymm8[14],ymm13[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm12 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm17 & ymm12)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0],ymm6[1,2,3,4,5,6,7],ymm13[8],ymm6[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm19
@@ -5923,7 +5923,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2],ymm8[3],ymm15[4,5,6],ymm8[7,8],ymm15[9,10],ymm8[11],ymm15[12,13,14],ymm8[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm20
@@ -5949,7 +5949,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2,3],ymm8[4],ymm15[5,6],ymm8[7,8],ymm15[9,10,11],ymm8[12],ymm15[13,14],ymm8[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm21
@@ -5975,7 +5975,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm7 ^ ymm1))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm8[1],ymm15[2,3],ymm8[4],ymm15[5,6,7,8],ymm8[9],ymm15[10,11],ymm8[12],ymm15[13,14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm2 ^ (ymm11 & (ymm3 ^ ymm2))
@@ -5997,7 +5997,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm7 ^ ymm1))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm8[1],ymm0[2,3,4],ymm8[5],ymm0[6,7,8],ymm8[9],ymm0[10,11,12],ymm8[13],ymm0[14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -6098,7 +6098,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm14[0,1,2],ymm10[3],ymm14[4,5],ymm10[6],ymm14[7,8,9,10],ymm10[11],ymm14[12,13],ymm10[14],ymm14[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vpmovsxdq {{.*#+}} ymm18 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm8 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm8)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm1[1,2,3,4,5,6,7],ymm14[8],ymm1[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm20
@@ -6125,7 +6125,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm3 ^ ymm9))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2],ymm10[3],ymm14[4,5,6],ymm10[7,8],ymm14[9,10],ymm10[11],ymm14[12,13,14],ymm10[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm1 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm1)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm8[1,2,3,4,5,6,7],ymm14[8],ymm8[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm21
@@ -6151,7 +6151,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm9 ^ (ymm14 & (ymm3 ^ ymm9))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm10[0],ymm14[1,2,3],ymm10[4],ymm14[5,6],ymm10[7,8],ymm14[9,10,11],ymm10[12],ymm14[13,14],ymm10[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm1 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm18 & ymm1)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0],ymm8[1,2,3,4,5,6,7],ymm14[8],ymm8[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa %ymm11, %ymm1
@@ -6176,7 +6176,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm3 ^ (ymm15 & (ymm9 ^ ymm3))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm10[1],ymm15[2,3],ymm10[4],ymm15[5,6,7,8],ymm10[9],ymm15[10,11],ymm10[12],ymm15[13,14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm1 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm18 & ymm1)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0],ymm8[1,2,3,4,5,6,7],ymm15[8],ymm8[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm13 = ymm17 ^ (ymm13 & (ymm2 ^ ymm17))
@@ -6198,7 +6198,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm9 ^ ymm3))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm10[1],ymm0[2,3,4],ymm10[5],ymm0[6,7,8],ymm10[9],ymm0[10,11,12],ymm10[13],ymm0[14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm18)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm18 & ymm2)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm4[1,2,3,4,5,6,7],ymm0[8],ymm4[9,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm19, (%rsi)
@@ -6295,7 +6295,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1,2],ymm8[3],ymm13[4,5],ymm8[6],ymm13[7,8,9,10],ymm8[11],ymm13[12,13],ymm8[14],ymm13[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} ymm17 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm12 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm13 | (ymm17 & ymm12)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0],ymm6[1,2,3,4,5,6,7],ymm13[8],ymm6[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm19
@@ -6324,7 +6324,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2],ymm8[3],ymm15[4,5,6],ymm8[7,8],ymm15[9,10],ymm8[11],ymm15[12,13,14],ymm8[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm20
@@ -6350,7 +6350,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm7 ^ (ymm15 & (ymm1 ^ ymm7))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm15[1,2,3],ymm8[4],ymm15[5,6],ymm8[7,8],ymm15[9,10,11],ymm8[12],ymm15[13,14],ymm8[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm21
@@ -6376,7 +6376,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm1 ^ (ymm15 & (ymm7 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm15[0],ymm8[1],ymm15[2,3],ymm8[4],ymm15[5,6,7,8],ymm8[9],ymm15[10,11],ymm8[12],ymm15[13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm6 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm15 = ymm15 | (ymm17 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm15[0],ymm12[1,2,3,4,5,6,7],ymm15[8],ymm12[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm2 ^ (ymm11 & (ymm3 ^ ymm2))
@@ -6398,7 +6398,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & (ymm7 ^ ymm1))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm8[1],ymm0[2,3,4],ymm8[5],ymm0[6,7,8],ymm8[9],ymm0[10,11,12],ymm8[13],ymm0[14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm2 & ymm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm2)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm3[1,2,3,4,5,6,7],ymm0[8],ymm3[9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, (%rsi)
@@ -11772,7 +11772,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqu64 %ymm26, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpmovsxdq {{.*#+}} ymm26 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm26)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm26 & ymm2)
; AVX512-NEXT: vmovdqa64 %ymm23, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm30 ^ (ymm2 & (ymm27 ^ ymm30))
; AVX512-NEXT: vmovdqu64 %ymm30, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -11805,7 +11805,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm29 ^ (ymm3 & (ymm31 ^ ymm29))
; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1,2],ymm6[3],ymm3[4,5,6],ymm6[7,8],ymm3[9,10],ymm6[11],ymm3[12,13,14],ymm6[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm26)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm26 & ymm2)
; AVX512-NEXT: vmovdqa %ymm10, %ymm2
; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm30 ^ (ymm2 & (ymm27 ^ ymm30))
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm4
@@ -11978,13 +11978,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6],ymm0[7,8],ymm4[9,10,11],ymm0[12],ymm4[13,14],ymm0[15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpmovsxdq {{.*#+}} ymm7 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm2 & ymm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 | (ymm7 & ymm2)
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm0[1],ymm9[2,3],ymm0[4],ymm9[5,6,7,8],ymm0[9],ymm9[10,11],ymm0[12],ymm9[13,14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm3 & ymm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm7 & ymm3)
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm14[0],ymm0[1],ymm14[2,3,4],ymm0[5],ymm14[6,7,8],ymm0[9],ymm14[10,11,12],ymm0[13],ymm14[14,15]
; AVX512-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm5 & ymm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm7 & ymm5)
; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm14 ^ (ymm12 & (ymm19 ^ ymm14))
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm12[u,u,2,9],zero,zero,zero,xmm12[5,12],zero,zero,xmm12[u,u,u,u,u]
@@ -12218,7 +12218,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm12[3],ymm7[4,5],ymm12[6],ymm7[7,8,9,10],ymm12[11],ymm7[12,13],ymm12[14],ymm7[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} ymm25 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm6 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm6)
; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm30 ^ (ymm6 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm8
@@ -12267,7 +12267,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm31 ^ (ymm6 & (ymm27 ^ ymm31))
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm12[0],ymm6[1,2],ymm12[3],ymm6[4,5,6],ymm12[7,8],ymm6[9,10],ymm12[11],ymm6[12,13,14],ymm12[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm3 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm25 & ymm3)
; AVX512-FCP-NEXT: vmovdqa %ymm1, %ymm3
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm30 ^ (ymm3 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm7
@@ -12381,13 +12381,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0],ymm7[1,2,3],ymm0[4],ymm7[5,6],ymm0[7,8],ymm7[9,10,11],ymm0[12],ymm7[13,14],ymm0[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm6 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm25 & ymm6)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0],ymm0[1],ymm13[2,3],ymm0[4],ymm13[5,6,7,8],ymm0[9],ymm13[10,11],ymm0[12],ymm13[13,14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm3 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm3)
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm9[0],ymm0[1],ymm9[2,3,4],ymm0[5],ymm9[6,7,8],ymm0[9],ymm9[10,11,12],ymm0[13],ymm9[14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm4 & ymm25)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm25 & ymm4)
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm14 ^ (ymm1 & (ymm2 ^ ymm14))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm28 ^ ymm30))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm30 ^ (ymm8 & (ymm28 ^ ymm30))
@@ -12591,7 +12591,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vpmovsxdq {{.*#+}} ymm29 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm29)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm29 & ymm2)
; AVX512DQ-NEXT: vmovdqa %ymm7, %ymm2
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm28 ^ (ymm2 & (ymm27 ^ ymm28))
; AVX512DQ-NEXT: vmovdqa64 %ymm27, %ymm19
@@ -12628,7 +12628,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm21 ^ (ymm3 & (ymm31 ^ ymm21))
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1,2],ymm6[3],ymm3[4,5,6],ymm6[7,8],ymm3[9,10],ymm6[11],ymm3[12,13,14],ymm6[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm29)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm29 & ymm2)
; AVX512DQ-NEXT: vmovdqa64 %ymm29, %ymm6
; AVX512DQ-NEXT: vmovdqa %ymm10, %ymm2
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm28 ^ (ymm2 & (ymm27 ^ ymm28))
@@ -12800,13 +12800,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm0[0],ymm3[1,2,3],ymm0[4],ymm3[5,6],ymm0[7,8],ymm3[9,10,11],ymm0[12],ymm3[13,14],ymm0[15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm29 & ymm6)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm6 & ymm29)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm9 = ymm9[0],ymm0[1],ymm9[2,3],ymm0[4],ymm9[5,6,7,8],ymm0[9],ymm9[10,11],ymm0[12],ymm9[13,14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm2 & ymm6)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm6 & ymm2)
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm14[0],ymm0[1],ymm14[2,3,4],ymm0[5],ymm14[6,7,8],ymm0[9],ymm14[10,11,12],ymm0[13],ymm14[14,15]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm4 & ymm6)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm4)
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm28 ^ (ymm5 & (ymm19 ^ ymm28))
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm5[u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u,u,u]
; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm4
@@ -13037,7 +13037,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm12[3],ymm7[4,5],ymm12[6],ymm7[7,8,9,10],ymm12[11],ymm7[12,13],ymm12[14],ymm7[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} ymm25 = [18446744073709551615,255,18446744073709486080,18446744073709551615]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm6 & ymm25)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm6)
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm30 ^ (ymm6 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm6, %xmm8
@@ -13086,7 +13086,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm31 ^ (ymm6 & (ymm27 ^ ymm31))
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm12[0],ymm6[1,2],ymm12[3],ymm6[4,5,6],ymm12[7,8],ymm6[9,10],ymm12[11],ymm6[12,13,14],ymm12[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm3 & ymm25)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm25 & ymm3)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, %ymm3
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm30 ^ (ymm3 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm7
@@ -13200,13 +13200,13 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0],ymm7[1,2,3],ymm0[4],ymm7[5,6],ymm0[7,8],ymm7[9,10,11],ymm0[12],ymm7[13,14],ymm0[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm6 & ymm25)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm25 & ymm6)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0],ymm0[1],ymm13[2,3],ymm0[4],ymm13[5,6,7,8],ymm0[9],ymm13[10,11],ymm0[12],ymm13[13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm3 & ymm25)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm7 | (ymm25 & ymm3)
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm9[0],ymm0[1],ymm9[2,3,4],ymm0[5],ymm9[6,7,8],ymm0[9],ymm9[10,11,12],ymm0[13],ymm9[14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm4 & ymm25)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm25 & ymm4)
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm14 ^ (ymm1 & (ymm2 ^ ymm14))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm28 ^ ymm30))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm30 ^ (ymm8 & (ymm28 ^ ymm30))
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
index 7dbff047e4f87..57d667563f8d4 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
@@ -3100,7 +3100,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512-NEXT: vprold $16, %xmm10, %xmm11
; AVX512-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3145,7 +3145,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3164,7 +3164,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3183,7 +3183,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512-NEXT: vprold $16, %xmm24, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3244,7 +3244,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512-FCP-NEXT: vprold $16, %xmm10, %xmm11
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3289,7 +3289,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3308,7 +3308,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3327,7 +3327,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512-FCP-NEXT: vprold $16, %xmm24, %xmm0
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3388,7 +3388,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512DQ-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512DQ-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512DQ-NEXT: vprold $16, %xmm10, %xmm11
; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3433,7 +3433,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3452,7 +3452,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512DQ-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3471,7 +3471,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512DQ-NEXT: vprold $16, %xmm24, %xmm0
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
@@ -3532,7 +3532,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
; AVX512DQ-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm3
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm15 & zmm10)
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %xmm10
; AVX512DQ-FCP-NEXT: vprold $16, %xmm10, %xmm11
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm12
@@ -3577,7 +3577,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm17
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm19 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm7
@@ -3596,7 +3596,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm6
; AVX512DQ-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm6
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm15 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm6
@@ -3615,7 +3615,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm6
; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm7
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm19 & zmm0)
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
; AVX512DQ-FCP-NEXT: vprold $16, %xmm24, %xmm0
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index 1fd85afea7d2d..9aadbae4c05f5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -3382,7 +3382,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,1,1]
; AVX512-NEXT: vpandnq %ymm13, %ymm20, %ymm13
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm13
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm11 & zmm20)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm20 & zmm11)
; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[0,1,2,1,4,5,6,5]
; AVX512-NEXT: vprolq $16, %ymm7, %ymm11
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0,1],ymm2[2],ymm11[3],ymm2[4],ymm11[5,6],ymm2[7],ymm11[8,9],ymm2[10],ymm11[11],ymm2[12],ymm11[13,14],ymm2[15]
@@ -3506,7 +3506,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,1]
; AVX512-FCP-NEXT: vpandnq %ymm3, %ymm17, %ymm3
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm12 & zmm17)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm17 & zmm12)
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm12
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm12[0,1,2,1,4,5,6,5]
; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm15
@@ -3688,7 +3688,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,1,1]
; AVX512DQ-NEXT: vpandnq %ymm13, %ymm20, %ymm13
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm13
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm11 & zmm20)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm20 & zmm11)
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[0,1,2,1,4,5,6,5]
; AVX512DQ-NEXT: vprolq $16, %ymm7, %ymm11
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0,1],ymm2[2],ymm11[3],ymm2[4],ymm11[5,6],ymm2[7],ymm11[8,9],ymm2[10],ymm11[11],ymm2[12],ymm11[13,14],ymm2[15]
@@ -3812,7 +3812,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,1]
; AVX512DQ-FCP-NEXT: vpandnq %ymm3, %ymm17, %ymm3
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm12 & zmm17)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm17 & zmm12)
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm12[0,1,2,1,4,5,6,5]
; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm15
@@ -6962,8 +6962,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: # zmm28 = mem ^ (zmm12 & (zmm28 ^ mem))
; AVX512-NEXT: vpternlogq {{.*#+}} zmm22 = zmm25 ^ (zmm12 & (zmm22 ^ zmm25))
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm28 & zmm12)
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm22 & zmm12)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm12 & zmm28)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm12 & zmm22)
; AVX512-NEXT: vpbroadcastq 64(%r8), %ymm12
; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm12, %zmm9
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
@@ -6974,8 +6974,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm10 & (zmm3 ^ zmm0))
; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm4 ^ (zmm10 & (zmm2 ^ zmm4))
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm0)
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm2 & zmm0)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm3)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm0 & zmm2)
; AVX512-NEXT: vmovdqa64 %zmm8, 384(%r9)
; AVX512-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512-NEXT: vmovdqa64 %zmm6, (%r9)
@@ -7257,15 +7257,15 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm20 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm20 = mem ^ (zmm8 & (zmm20 ^ mem))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm10 & zmm7)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm20 & zmm7)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm7 & zmm10)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm7 & zmm20)
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm28 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm28 = mem ^ (zmm2 & (zmm28 ^ mem))
; AVX512-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm19 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm19 = mem ^ (zmm2 & (zmm19 ^ mem))
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm28 & zmm2)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm19 & zmm2)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm2 & zmm28)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm2 & zmm19)
; AVX512-FCP-NEXT: vpbroadcastq 64(%r8), %ymm2
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm2, %zmm2
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
@@ -7555,8 +7555,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: # zmm28 = mem ^ (zmm12 & (zmm28 ^ mem))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm22 = zmm25 ^ (zmm12 & (zmm22 ^ zmm25))
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm28 & zmm12)
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm22 & zmm12)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm12 & zmm28)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm12 & zmm22)
; AVX512DQ-NEXT: vpbroadcastq 64(%r8), %ymm12
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm9, %zmm12, %zmm9
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
@@ -7567,8 +7567,8 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm10 & (zmm3 ^ zmm0))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm4 ^ (zmm10 & (zmm2 ^ zmm4))
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm3 & zmm0)
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm2 & zmm0)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm3)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm0 & zmm2)
; AVX512DQ-NEXT: vmovdqa64 %zmm8, 384(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%r9)
@@ -7850,15 +7850,15 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm20 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm20 = mem ^ (zmm8 & (zmm20 ^ mem))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm10 & zmm7)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm20 & zmm7)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 | (zmm7 & zmm10)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm7 & zmm20)
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm28 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm28 = mem ^ (zmm2 & (zmm28 ^ mem))
; AVX512DQ-FCP-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm19 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm19 = mem ^ (zmm2 & (zmm19 ^ mem))
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm28 & zmm2)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm19 & zmm2)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm2 & zmm28)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 | (zmm2 & zmm19)
; AVX512DQ-FCP-NEXT: vpbroadcastq 64(%r8), %ymm2
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm2, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 6e0d9efbb083c..91c2c305ea4e1 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -12989,13 +12989,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11
; AVX512-NEXT: vpternlogq {{.*#+}} zmm11 = zmm1 ^ (zmm23 & (zmm11 ^ zmm1))
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm11 & zmm26)
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm13 & zmm16)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm26 & zmm11)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm16 & zmm13)
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm7[0,1,2,3]
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm7 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm7) | zmm22
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm14 & zmm7)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm7 & zmm14)
; AVX512-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm29 # 64-byte Folded Reload
; AVX512-NEXT: # zmm29 = zmm29 ^ (zmm24 & (zmm29 ^ mem))
; AVX512-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
@@ -13620,12 +13620,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm4, %zmm4
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm12 & (zmm4 ^ zmm0))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm4 & zmm31)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm31 & zmm4)
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm6 & mem)
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm9 = zmm9 | (zmm0 & mem)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm13 & zmm0)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm0 & zmm13)
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm4 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm4 = zmm4 ^ (zmm17 & (zmm4 ^ mem))
@@ -14217,13 +14217,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm11 = zmm1 ^ (zmm23 & (zmm11 ^ zmm1))
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm11 & zmm26)
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm13 & zmm16)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm26 & zmm11)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm16 & zmm13)
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm7[0,1,2,3]
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm7 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = (zmm1 & zmm7) | zmm22
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm14 & zmm7)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm7 & zmm14)
; AVX512DQ-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm29 # 64-byte Folded Reload
; AVX512DQ-NEXT: # zmm29 = zmm29 ^ (zmm24 & (zmm29 ^ mem))
; AVX512DQ-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
@@ -14848,12 +14848,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm4, %zmm4
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm12 & (zmm4 ^ zmm0))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm4 & zmm31)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm31 & zmm4)
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm13 = zmm13 | (zmm6 & mem)
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm9 = zmm9 | (zmm0 & mem)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm13 & zmm0)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm0 & zmm13)
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm4 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm4 = zmm4 ^ (zmm17 & (zmm4 ^ mem))
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 480cfef7fda1e..0966738decccf 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -1481,7 +1481,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512BW-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512BW-NEXT: vzeroupper
@@ -1511,7 +1511,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512BW-FCP-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512BW-FCP-NEXT: vzeroupper
@@ -1549,7 +1549,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512DQ-BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512DQ-BW-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512DQ-BW-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512DQ-BW-NEXT: vzeroupper
@@ -1579,7 +1579,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[5,13],zero,zero,zero,xmm0[6,14],zero,zero,zero,xmm0[7,15],zero,zero,zero
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[12],zero,zero,zero,zero,xmm2[13],zero,zero,zero,zero,xmm2[14],zero,zero,zero,zero,xmm2[15]
-; AVX512DQ-BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
+; AVX512DQ-BW-FCP-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | xmm1 | xmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, 64(%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
@@ -4776,7 +4776,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vinserti64x4 $1, %ymm19, %zmm6, %zmm6
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm7 & (zmm6 ^ zmm4))
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm6 & zmm11)
+; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm11 & zmm6)
; AVX512-NEXT: vpermq {{.*#+}} zmm4 = zmm22[2,2,3,3,6,6,7,7]
; AVX512-NEXT: vpermq {{.*#+}} zmm6 = zmm23[2,2,3,3,6,6,7,7]
; AVX512-NEXT: vporq %zmm4, %zmm6, %zmm4
@@ -4954,7 +4954,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm23, %zmm5, %zmm5
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm6 & (zmm5 ^ zmm3))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm5 & zmm14)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm14 & zmm5)
; AVX512-FCP-NEXT: vpermq {{.*#+}} zmm3 = zmm25[2,2,3,3,6,6,7,7]
; AVX512-FCP-NEXT: vpermq {{.*#+}} zmm5 = zmm26[2,2,3,3,6,6,7,7]
; AVX512-FCP-NEXT: vporq %zmm3, %zmm5, %zmm3
@@ -5116,7 +5116,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm19, %zmm6, %zmm6
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm7 & (zmm6 ^ zmm4))
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm6 & zmm11)
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm11 & zmm6)
; AVX512DQ-NEXT: vpermq {{.*#+}} zmm4 = zmm22[2,2,3,3,6,6,7,7]
; AVX512DQ-NEXT: vpermq {{.*#+}} zmm6 = zmm23[2,2,3,3,6,6,7,7]
; AVX512DQ-NEXT: vporq %zmm4, %zmm6, %zmm4
@@ -5294,7 +5294,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm23, %zmm5, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm6 & (zmm5 ^ zmm3))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm5 & zmm14)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm14 & zmm5)
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} zmm3 = zmm25[2,2,3,3,6,6,7,7]
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} zmm5 = zmm26[2,2,3,3,6,6,7,7]
; AVX512DQ-FCP-NEXT: vporq %zmm3, %zmm5, %zmm3
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index ec4c2b0ca1cf5..eec736826bb87 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -8730,71 +8730,73 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
;
; AVX512-FCP-LABEL: store_i8_stride7_vf64:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: subq $1448, %rsp # imm = 0x5A8
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm8
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero,zero,zero,zero,ymm8[18]
+; AVX512-FCP-NEXT: subq $1416, %rsp # imm = 0x588
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm7
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
+; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm21
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm7
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm0
-; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm14
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm14, %ymm1
-; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
+; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm9
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm9[14],zero,zero,zero,zero,zero,zero,ymm9[15],zero,zero,zero,zero,zero,zero,ymm9[16],zero,zero,zero,zero,zero,zero,ymm9[17],zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm6
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[0,1,0,1,14],zero,ymm6[14,15,0,1,14,15],zero,ymm6[13,14,15,16,17,16],zero,ymm6[30,31,30,31,16,17],zero,ymm6[31,28,29,30,31]
+; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm20
-; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm9
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[13,u,u,u,u,u],zero,ymm9[14,u,u,u,u,u],zero,ymm9[15,u,u,u,u,u],zero,ymm9[16,u,u,u,u,u],zero,ymm9[17,u,u,u]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm16
+; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm8
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
+; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm11
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
-; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm15
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
+; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm15, %ymm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm17
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero,zero,ymm5[27],zero,ymm5[25]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm11[23,u,u,u],zero,ymm11[26],zero,ymm11[24,u,u,u],zero,ymm11[27],zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[23],zero,zero,zero,zero,ymm11[26],zero,ymm11[24],zero,zero,zero,zero,ymm11[27],zero,ymm11[25]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm15[23,u,u,u],zero,ymm15[26],zero,ymm15[24,u,u,u],zero,ymm15[27],zero
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm4
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm10
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128]
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm1
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
-; AVX512-FCP-NEXT: # ymm10 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm10, %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
+; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
+; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[25],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm10[23],zero,ymm10[21,22,23,26],zero,ymm10[24],zero,ymm10[28,29,26,27]
; AVX512-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm3
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
-; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm12
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
+; AVX512-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm12
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm2
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128]
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm13
-; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
@@ -8802,300 +8804,295 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm15
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm0
+; AVX512-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
; AVX512-FCP-NEXT: # ymm12 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm15, %ymm13
-; AVX512-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm15[2,3,2,3],zmm13[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm13
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[2,3,2,3],zmm13[2,3,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm21
-; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm13
-; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm8
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm7, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
+; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm7
+; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm7
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm14, %ymm19
-; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm8
-; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm17
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 %ymm9, %ymm24
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rax), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm6
+; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm5
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [4,0,6,0,4,0,6,7,0,17,0,17,0,16,16,0]
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm8
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT: vpermi2d %zmm10, %zmm6, %zmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm7
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT: vpermi2d %zmm9, %zmm5, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm14
-; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
-; AVX512-FCP-NEXT: vmovdqa64 %ymm9, %ymm20
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm7, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm12, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm19
+; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm7
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm7
-; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm27
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm7, %xmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm13
+; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u],zero,xmm0[7],zero,xmm0[5,u,u,u],zero,xmm0[8],zero,xmm0[6,u,u]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm29
+; AVX512-FCP-NEXT: vpor %xmm5, %xmm6, %xmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm6
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm9
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm9, %xmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm14, %xmm8
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm9
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
+; AVX512-FCP-NEXT: vpor %xmm8, %xmm9, %xmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,5,5,6]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm8
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm18
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm17
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
-; AVX512-FCP-NEXT: vpermi2d %zmm6, %zmm8, %zmm0
+; AVX512-FCP-NEXT: vpermi2d %zmm8, %zmm7, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm0
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm9
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm23
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm25
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm16
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm8
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512-FCP-NEXT: vpor %xmm6, %xmm8, %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
+; AVX512-FCP-NEXT: vporq %xmm7, %xmm8, %xmm31
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,ymm4[14],zero,zero,zero,zero,zero,zero,ymm4[15],zero,zero,zero,zero,zero,zero,ymm4[16],zero,zero,zero,zero,zero,zero,ymm4[17],zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[0,1,0,1,14],zero,ymm10[14,15,0,1,14,15],zero,ymm10[13,14,15,16,17,16],zero,ymm10[30,31,30,31,16,17],zero,ymm10[31,28,29,30,31]
+; AVX512-FCP-NEXT: vpor %ymm7, %ymm8, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm6
-; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm8
-; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[18],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm10[18,19,20,21],zero,ymm10[19],zero,ymm10[25,26,27,22],zero,ymm10[20],zero
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero,zero,ymm15[18]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[0,1,14],zero,ymm11[12,13,0,1,14,15],zero,ymm11[3,12,13,2,3,16],zero,ymm11[30,31,28,29,16,17],zero,ymm11[31,18,19,28,29,18],zero
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
+; AVX512-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm15, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
+; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm1
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm11[14],zero,zero,zero,zero,zero,zero,ymm11[15],zero,zero,zero,zero,zero,zero,ymm11[16],zero,zero,zero,zero,zero,zero,ymm11[17],zero,zero,zero,zero,zero,zero,ymm11[18]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm1
-; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm4
-; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
-; AVX512-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm12
-; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm8
-; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm2
-; AVX512-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
+; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
+; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm2
+; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm11
-; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm6
-; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm2
-; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm10
-; AVX512-FCP-NEXT: vmovdqa64 %xmm18, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm18
-; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm7
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm11[u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u]
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 %xmm29, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm17, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm1
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm17
+; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm10
+; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
+; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm8
; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm5
-; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-FCP-NEXT: vporq %xmm3, %xmm4, %xmm26
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm1
+; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512-FCP-NEXT: vporq %xmm3, %xmm4, %xmm29
+; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero,ymm2[27],zero,ymm2[25]
+; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm1[23,u,u,u],zero,ymm1[26],zero,ymm1[24,u,u,u],zero,ymm1[27],zero
; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm25
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm4
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm23
; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm1
-; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm24
-; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm23
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
-; AVX512-FCP-NEXT: vporq %ymm0, %ymm3, %ymm22
-; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm1
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm14[23],zero,ymm14[23,24,25,26],zero,ymm14[24],zero,ymm14[30,31]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm14[18],zero,ymm14[20,21,20,21],zero,ymm14[19],zero,ymm14[19,20,21,22],zero
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
+; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm22
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm2[18,19,20,21],zero,ymm2[19],zero,ymm2[25,26,27,22],zero,ymm2[20],zero
+; AVX512-FCP-NEXT: vporq %ymm3, %ymm4, %ymm21
+; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm12[23],zero,ymm12[23,24,25,26],zero,ymm12[24],zero,ymm12[30,31]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm12[18],zero,ymm12[20,21,20,21],zero,ymm12[19],zero,ymm12[19,20,21,22],zero
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,6]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm0
; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm29
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm14
-; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm15 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
+; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm3
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm5
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm10
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3],xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm1
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm1
; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3],xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm6[8],xmm11[9],xmm6[9],xmm11[10],xmm6[10],xmm11[11],xmm6[11],xmm11[12],xmm6[12],xmm11[13],xmm6[13],xmm11[14],xmm6[14],xmm11[15],xmm6[15]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm13[8],xmm4[9],xmm13[9],xmm4[10],xmm13[10],xmm4[11],xmm13[11],xmm4[12],xmm13[12],xmm4[13],xmm13[13],xmm4[14],xmm13[14],xmm4[15],xmm13[15]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm13
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm3[0,1,0,1],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm15[8],xmm2[8],xmm15[9],xmm2[9],xmm15[10],xmm2[10],xmm15[11],xmm2[11],xmm15[12],xmm2[12],xmm15[13],xmm2[13],xmm15[14],xmm2[14],xmm15[15],xmm2[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm4
-; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm3, %xmm3
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm2[0,1,0,1],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm14
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm26 = zmm0[0,1,0,1],zmm3[0,1,0,1]
; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm9[8],xmm1[8],xmm9[9],xmm1[9],xmm9[10],xmm1[10],xmm9[11],xmm1[11],xmm9[12],xmm1[12],xmm9[13],xmm1[13],xmm9[14],xmm1[14],xmm9[15],xmm1[15]
-; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; AVX512-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm28 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm28, %zmm15
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm0
-; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512-FCP-NEXT: vpermd %ymm13, %ymm28, %ymm13
-; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3],xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7]
-; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm6 = zmm6[0,1,0,1],mem[0,1,0,1]
-; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm1
-; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm26[0,1,0,1]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm24[2,3,2,3],zmm25[2,3,2,3]
-; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm22[2,3,2,3],zmm23[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm12 & (zmm7 ^ zmm5))
-; AVX512-FCP-NEXT: vporq %zmm20, %zmm19, %zmm5
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm5))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm31[0,1,0,1],zmm0[0,1,0,1]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm1, %zmm9
+; AVX512-FCP-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm0, %ymm2
+; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm1
+; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm1 = zmm1[0,1,0,1],mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm7 = zmm7[0,1,0,1],mem[0,1,0,1]
+; AVX512-FCP-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm2
+; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm29[0,1,0,1]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm23[2,3,2,3],zmm25[2,3,2,3]
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm21[2,3,2,3],zmm22[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm10 & (zmm8 ^ zmm6))
+; AVX512-FCP-NEXT: vporq %zmm19, %zmm18, %zmm6
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm6))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm8))
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa64 %zmm15, 128(%rax)
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm7 = zmm7[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm8 = zmm8[2,3,2,3],mem[2,3,2,3]
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm13[0,1,0,1]
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm5))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm12 & (zmm5 ^ zmm7))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm17 ^ (zmm12 & (zmm31 ^ zmm17))
-; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm21 ^ (zmm7 & (zmm16 ^ zmm21))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm8))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm26 = zmm16 ^ (zmm10 & (zmm26 ^ zmm16))
+; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm20 ^ (zmm8 & (zmm12 ^ zmm20))
+; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: # ymm10 = mem[2,3,2,3]
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm10
; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm11 = mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: # ymm12 = mem[2,3,2,3]
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm11 ^ (zmm7 & (zmm12 ^ zmm11))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm2, %zmm11
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm11 = zmm10 ^ (zmm8 & (zmm11 ^ zmm10))
; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm4))
-; AVX512-FCP-NEXT: vinserti64x4 $1, (%rsp), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm3 & mem)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm7))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm5))
+; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm8 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm4))
+; AVX512-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & mem) | zmm24
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm8))
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm4 ^ (mem & (zmm30 ^ zmm4))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm16))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm1 ^ (mem & (zmm9 ^ zmm1))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm31))
-; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT: # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm12))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm18 ^ (mem & (zmm2 ^ zmm18))
-; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm6))
-; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rax)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm2 ^ (mem & (zmm28 ^ zmm2))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (mem & (zmm28 ^ zmm12))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm2 ^ (mem & (zmm31 ^ zmm2))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm26))
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT: # zmm2 = zmm2[0,1,2,3],mem[2,3,2,3]
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm11))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm17 ^ (mem & (zmm5 ^ zmm17))
+; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm7))
+; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 320(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 256(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm30, 192(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 384(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm14, 64(%rax)
-; AVX512-FCP-NEXT: addq $1448, %rsp # imm = 0x5A8
+; AVX512-FCP-NEXT: vmovdqa64 %zmm31, 256(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm28, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 384(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 64(%rax)
+; AVX512-FCP-NEXT: addq $1416, %rsp # imm = 0x588
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -9462,71 +9459,73 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
;
; AVX512DQ-FCP-LABEL: store_i8_stride7_vf64:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: subq $1448, %rsp # imm = 0x5A8
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm8
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero,zero,zero,zero,ymm8[18]
+; AVX512DQ-FCP-NEXT: subq $1416, %rsp # imm = 0x588
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm7[14],zero,zero,zero,zero,zero,zero,ymm7[15],zero,zero,zero,zero,zero,zero,ymm7[16],zero,zero,zero,zero,zero,zero,ymm7[17],zero,zero,zero,zero,zero,zero,ymm7[18]
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,14],zero,ymm2[12,13,0,1,14,15],zero,ymm2[3,12,13,2,3,16],zero,ymm2[30,31,28,29,16,17],zero,ymm2[31,18,19,28,29,18],zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm21
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm14, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
+; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm9
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,ymm9[14],zero,zero,zero,zero,zero,zero,ymm9[15],zero,zero,zero,zero,zero,zero,ymm9[16],zero,zero,zero,zero,zero,zero,ymm9[17],zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm6
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[0,1,0,1,14],zero,ymm6[14,15,0,1,14,15],zero,ymm6[13,14,15,16,17,16],zero,ymm6[30,31,30,31,16,17],zero,ymm6[31,28,29,30,31]
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm20
-; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm9
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[13,u,u,u,u,u],zero,ymm9[14,u,u,u,u,u],zero,ymm9[15,u,u,u,u,u],zero,ymm9[16,u,u,u,u,u],zero,ymm9[17,u,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm16
+; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm8
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0,13,0,0,0,128,16,128,14,0,0,0,128,17,128,15,0]
+; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm11
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
-; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm15
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128]
+; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29,28,29,30,128,28,128,30,31,30,31,128,29,128,31,28,29]
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm17
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero,zero,ymm5[27],zero,ymm5[25]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm11[23,u,u,u],zero,ymm11[26],zero,ymm11[24,u,u,u],zero,ymm11[27],zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[23],zero,zero,zero,zero,ymm11[26],zero,ymm11[24],zero,zero,zero,zero,ymm11[27],zero,ymm11[25]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm15[23,u,u,u],zero,ymm15[26],zero,ymm15[24,u,u,u],zero,ymm15[27],zero
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm10
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,128]
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm1
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
-; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,0]
+; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512DQ-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[25],zero,ymm4[23],zero,zero,zero,zero,ymm4[26],zero,ymm4[24],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm10[23],zero,ymm10[21,22,23,26],zero,ymm10[24],zero,ymm10[28,29,26,27]
; AVX512DQ-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm3
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
-; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29,128,27,128,128,128,128,30,128,28,128,128,128,128,31,128,29]
+; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm2
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128,29,128,27,0,0,0,128,30,128,28,0,0,0,128,31,128]
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512DQ-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
@@ -9534,300 +9533,295 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vpor %ymm12, %ymm13, %ymm12
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
; AVX512DQ-FCP-NEXT: # ymm12 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm15, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm15[2,3,2,3],zmm13[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm13
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[2,3,2,3],zmm13[2,3,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm21
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm8
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm7, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm7
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm14, %ymm19
-; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm17
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, %ymm24
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm7, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm6
+; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm5
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [4,0,6,0,4,0,6,7,0,17,0,17,0,16,16,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm8
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm10, %zmm6, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm7
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm9, %zmm5, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, %ymm20
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm7, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm12, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm19
+; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm7
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm7
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm27
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm7, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm13
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u],zero,xmm0[7],zero,xmm0[5,u,u,u],zero,xmm0[8],zero,xmm0[6,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm29
+; AVX512DQ-FCP-NEXT: vpor %xmm5, %xmm6, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm9
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm28
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm9, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm14, %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
+; AVX512DQ-FCP-NEXT: vpor %xmm8, %xmm9, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,5,5,6]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm18
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm17
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,2,3,0,1,0,18,0,19,18,0,19,0]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm6, %zmm8, %zmm0
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm8, %zmm7, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm9
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm23
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm25
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm16
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm8, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
+; AVX512DQ-FCP-NEXT: vporq %xmm7, %xmm8, %xmm31
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,ymm4[14],zero,zero,zero,zero,zero,zero,ymm4[15],zero,zero,zero,zero,zero,zero,ymm4[16],zero,zero,zero,zero,zero,zero,ymm4[17],zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[0,1,0,1,14],zero,ymm10[14,15,0,1,14,15],zero,ymm10[13,14,15,16,17,16],zero,ymm10[30,31,30,31,16,17],zero,ymm10[31,28,29,30,31]
+; AVX512DQ-FCP-NEXT: vpor %ymm7, %ymm8, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm8
-; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[18],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm10[18,19,20,21],zero,ymm10[19],zero,ymm10[25,26,27,22],zero,ymm10[20],zero
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm15[14],zero,zero,zero,zero,zero,zero,ymm15[15],zero,zero,zero,zero,zero,zero,ymm15[16],zero,zero,zero,zero,zero,zero,ymm15[17],zero,zero,zero,zero,zero,zero,ymm15[18]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm11[0,1,14],zero,ymm11[12,13,0,1,14,15],zero,ymm11[3,12,13,2,3,16],zero,ymm11[30,31,28,29,16,17],zero,ymm11[31,18,19,28,29,18],zero
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
+; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
+; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
+; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm1
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm11[14],zero,zero,zero,zero,zero,zero,ymm11[15],zero,zero,zero,zero,zero,zero,ymm11[16],zero,zero,zero,zero,zero,zero,ymm11[17],zero,zero,zero,zero,zero,zero,ymm11[18]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm5[0,1,14],zero,ymm5[12,13,0,1,14,15],zero,ymm5[3,12,13,2,3,16],zero,ymm5[30,31,28,29,16,17],zero,ymm5[31,18,19,28,29,18],zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128]
-; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm1
-; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23]
-; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm3[18],zero,ymm3[20,21,20,21],zero,ymm3[19],zero,ymm3[19,20,21,22],zero
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
-; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm12
-; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm2
-; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512DQ-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm11
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm2
-; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm10
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm18, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm7
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm11[u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u]
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm29, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-FCP-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm17, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,0,1,0,0,0,0,16,0,16,0,18,19,0,17]
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm17
+; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm5
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-FCP-NEXT: vporq %xmm3, %xmm4, %xmm26
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27],zero,ymm13[25]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512DQ-FCP-NEXT: vporq %xmm3, %xmm4, %xmm29
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero,ymm2[27],zero,ymm2[25]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm1[23,u,u,u],zero,ymm1[26],zero,ymm1[24,u,u,u],zero,ymm1[27],zero
; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm25
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm4
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm23
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm24
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm4[23],zero,ymm4[21,22,23,26],zero,ymm4[24],zero,ymm4[28,29,26,27]
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm23
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm4[18,19,20,21],zero,ymm4[19],zero,ymm4[25,26,27,22],zero,ymm4[20],zero
-; AVX512DQ-FCP-NEXT: vporq %ymm0, %ymm3, %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm14[23],zero,ymm14[23,24,25,26],zero,ymm14[24],zero,ymm14[30,31]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm14[18],zero,ymm14[20,21,20,21],zero,ymm14[19],zero,ymm14[19,20,21,22],zero
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm3[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm2[23],zero,ymm2[21,22,23,26],zero,ymm2[24],zero,ymm2[28,29,26,27]
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm22
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22],zero,ymm1[20]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm2[18,19,20,21],zero,ymm2[19],zero,ymm2[25,26,27,22],zero,ymm2[20],zero
+; AVX512DQ-FCP-NEXT: vporq %ymm3, %ymm4, %ymm21
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm1[25],zero,ymm1[23],zero,zero,zero,zero,ymm1[26],zero,ymm1[24],zero,zero
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[20],zero,ymm1[18],zero,zero,zero,zero,ymm1[21],zero,ymm1[19],zero,zero,zero,zero,ymm1[22]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm12[23],zero,ymm12[23,24,25,26],zero,ymm12[24],zero,ymm12[30,31]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm12[18],zero,ymm12[20,21,20,21],zero,ymm12[19],zero,ymm12[19,20,21,22],zero
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm4[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,6]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,2,3,3,2,2,3,3]
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm0
; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm29
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm14
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm15 = [128,13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128]
+; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm3
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm5
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm10
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3],xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3],xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm30
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 = zmm0[2,3,2,3],zmm3[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm6[8],xmm11[9],xmm6[9],xmm11[10],xmm6[10],xmm11[11],xmm6[11],xmm11[12],xmm6[12],xmm11[13],xmm6[13],xmm11[14],xmm6[14],xmm11[15],xmm6[15]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm0[2,3,2,3],zmm3[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm13[8],xmm4[9],xmm13[9],xmm4[10],xmm13[10],xmm4[11],xmm13[11],xmm4[12],xmm13[12],xmm4[13],xmm13[13],xmm4[14],xmm13[14],xmm4[15],xmm13[15]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm13
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm3[0,1,0,1],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm15[8],xmm2[8],xmm15[9],xmm2[9],xmm15[10],xmm2[10],xmm15[11],xmm2[11],xmm15[12],xmm2[12],xmm15[13],xmm2[13],xmm15[14],xmm2[14],xmm15[15],xmm2[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm2[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm14
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm26 = zmm0[0,1,0,1],zmm3[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm9[8],xmm1[8],xmm9[9],xmm1[9],xmm9[10],xmm1[10],xmm9[11],xmm1[11],xmm9[12],xmm1[12],xmm9[13],xmm1[13],xmm9[14],xmm1[14],xmm9[15],xmm1[15]
-; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; AVX512DQ-FCP-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm28 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm28, %zmm15
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm13, %ymm0
-; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm13 = ymm13[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vpermd %ymm13, %ymm28, %ymm13
-; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3],xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm8 = zmm8[0,1,0,1],mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7]
-; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[0,1,0,1],mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm1
-; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm26[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm24[2,3,2,3],zmm25[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm22[2,3,2,3],zmm23[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm12 & (zmm7 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vporq %zmm20, %zmm19, %zmm5
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm31[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,5,4,0,5,0,4,0,20,21,0,23,0,21,0,23]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm1, %zmm9
+; AVX512DQ-FCP-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm1 = zmm1[0,1,0,1],mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512DQ-FCP-NEXT: vshufi64x2 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm7 = zmm7[0,1,0,1],mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm2
+; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm29[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm23[2,3,2,3],zmm25[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm21[2,3,2,3],zmm22[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm10 & (zmm8 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vporq %zmm19, %zmm18, %zmm6
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm8))
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, 128(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm7 = zmm7[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm8 = zmm8[2,3,2,3],mem[2,3,2,3]
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm13[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm5 = zmm5[2,3,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm12 & (zmm5 ^ zmm7))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm17 ^ (zmm12 & (zmm31 ^ zmm17))
-; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm7 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm21 ^ (zmm7 & (zmm16 ^ zmm21))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $85, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm6 = zmm6[2,3,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm8))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm26 = zmm16 ^ (zmm10 & (zmm26 ^ zmm16))
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm20 ^ (zmm8 & (zmm12 ^ zmm20))
+; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # ymm10 = mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm10
; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm11 = mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512DQ-FCP-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # ymm12 = mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm12 = zmm11 ^ (zmm7 & (zmm12 ^ zmm11))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm2, %zmm11
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm11 = zmm10 ^ (zmm8 & (zmm11 ^ zmm10))
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm4))
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, (%rsp), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm3 & mem)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm7))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm5))
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm8 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm4))
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = (zmm3 & mem) | zmm24
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm8))
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm4 ^ (mem & (zmm30 ^ zmm4))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm16))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm1 ^ (mem & (zmm9 ^ zmm1))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm31))
-; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT: # zmm1 = zmm1[0,1,2,3],mem[2,3,2,3]
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm12))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm18 ^ (mem & (zmm2 ^ zmm18))
-; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm6))
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rax)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm2 ^ (mem & (zmm28 ^ zmm2))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (mem & (zmm28 ^ zmm12))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm2 ^ (mem & (zmm31 ^ zmm2))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm26))
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT: # zmm2 = zmm2[0,1,2,3],mem[2,3,2,3]
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm11))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm17 ^ (mem & (zmm5 ^ zmm17))
+; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm7))
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 320(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 256(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm30, 192(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 384(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, 64(%rax)
-; AVX512DQ-FCP-NEXT: addq $1448, %rsp # imm = 0x5A8
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm31, 256(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm28, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 384(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 64(%rax)
+; AVX512DQ-FCP-NEXT: addq $1416, %rsp # imm = 0x588
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
index 813539b7d3b88..7b959bf13423b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll
@@ -227,11 +227,11 @@ define float @test_v4f32(<4 x float> %a0) {
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm0, %xmm3
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm0 | xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm4 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vmaxss %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm3 | xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm4 | xmm3)
; AVX512VL-NEXT: vcmpunordss %xmm3, %xmm3, %k1
; AVX512VL-NEXT: vmovss %xmm3, %xmm2, %xmm2 {%k1}
; AVX512VL-NEXT: vmaxss %xmm1, %xmm2, %xmm0
@@ -397,17 +397,17 @@ define float @test_v8f32(<8 x float> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vmaxps %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordps %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovaps %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm0, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm1[3,3,3,3]
@@ -623,22 +623,22 @@ define float @test_v16f32(<16 x float> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vmaxps %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 & (ymm0 | ymm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 & (ymm2 | ymm0)
; AVX512VL-NEXT: vcmpunordps %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovaps %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vmaxps %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordps %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovaps %xmm1, %xmm0 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vmaxss %xmm1, %xmm0, %xmm1
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0]
; AVX512VL-NEXT: vmaxss %xmm3, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -816,7 +816,7 @@ define double @test_v4f64(<4 x double> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vmaxpd %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm0 | xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
@@ -969,12 +969,12 @@ define double @test_v8f64(<8 x double> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vmaxpd %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm2 = [NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 & (ymm0 | ymm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 & (ymm2 | ymm0)
; AVX512VL-NEXT: vcmpunordpd %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovapd %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vmaxpd %xmm0, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & (xmm1 | xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 & (xmm2 | xmm1)
; AVX512VL-NEXT: vcmpunordpd %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovapd %xmm1, %xmm3 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm3[1,0]
@@ -1184,7 +1184,7 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmaxpd %zmm1, %zmm0, %zmm1
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm0 | zmm2)
+; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm2 | zmm0)
; AVX512BW-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512BW-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512BW-NEXT: vextractf64x4 $1, %zmm1, %ymm0
@@ -1212,20 +1212,19 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vmaxpd %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm0 | zmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 & (zmm2 | zmm0)
; AVX512VL-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512VL-NEXT: vextractf64x4 $1, %zmm1, %ymm0
; AVX512VL-NEXT: vmaxpd %ymm0, %ymm1, %ymm0
-; AVX512VL-NEXT: vmovdqa %ymm2, %ymm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm3 = ymm0 & (ymm3 | ymm1)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 & (ymm1 | ymm2)
; AVX512VL-NEXT: vcmpunordpd %ymm1, %ymm1, %k1
-; AVX512VL-NEXT: vmovapd %ymm1, %ymm3 {%k1}
-; AVX512VL-NEXT: vextractf128 $1, %ymm3, %xmm0
-; AVX512VL-NEXT: vmaxpd %xmm0, %xmm3, %xmm1
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm3 | xmm2)
-; AVX512VL-NEXT: vcmpunordpd %xmm3, %xmm3, %k1
-; AVX512VL-NEXT: vmovapd %xmm3, %xmm1 {%k1}
+; AVX512VL-NEXT: vmovapd %ymm1, %ymm0 {%k1}
+; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vmaxpd %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 & (xmm2 | xmm0)
+; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
; AVX512VL-NEXT: vmaxsd %xmm0, %xmm1, %xmm0
; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 & (xmm1 | xmm2)
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fminimum.ll b/llvm/test/CodeGen/X86/vector-reduce-fminimum.ll
index 2c85839f3a266..bb63b430fbe7f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fminimum.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fminimum.ll
@@ -227,11 +227,11 @@ define float @test_v4f32(<4 x float> %a0) {
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vminss %xmm3, %xmm0, %xmm3
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm4 & xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vminss %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm3 & xmm4)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm4 & xmm3)
; AVX512VL-NEXT: vcmpunordss %xmm3, %xmm3, %k1
; AVX512VL-NEXT: vmovss %xmm3, %xmm2, %xmm2 {%k1}
; AVX512VL-NEXT: vminss %xmm1, %xmm2, %xmm0
@@ -397,17 +397,17 @@ define float @test_v8f32(<8 x float> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vminps %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm2 & xmm0)
; AVX512VL-NEXT: vcmpunordps %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovaps %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
; AVX512VL-NEXT: vminss %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 | (xmm1 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 | (xmm2 & xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
; AVX512VL-NEXT: vminss %xmm3, %xmm0, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm2 & xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm1[3,3,3,3]
@@ -623,22 +623,22 @@ define float @test_v16f32(<16 x float> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vminps %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & ymm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm2 & ymm0)
; AVX512VL-NEXT: vcmpunordps %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovaps %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vminps %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 | (xmm1 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 | (xmm2 & xmm1)
; AVX512VL-NEXT: vcmpunordps %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovaps %xmm1, %xmm0 {%k1}
; AVX512VL-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX512VL-NEXT: vminss %xmm1, %xmm0, %xmm1
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm2 & xmm0)
; AVX512VL-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0]
; AVX512VL-NEXT: vminss %xmm3, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm1 & xmm2)
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm2 & xmm1)
; AVX512VL-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovss %xmm1, %xmm3, %xmm3 {%k1}
; AVX512VL-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -816,7 +816,7 @@ define double @test_v4f64(<4 x double> %a0) {
; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vminpd %xmm1, %xmm0, %xmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm2 & xmm0)
; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
@@ -969,12 +969,12 @@ define double @test_v8f64(<8 x double> %a0) {
; AVX512VL-NEXT: vextractf64x4 $1, %zmm0, %ymm1
; AVX512VL-NEXT: vminpd %ymm1, %ymm0, %ymm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & ymm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm2 & ymm0)
; AVX512VL-NEXT: vcmpunordpd %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vmovapd %ymm0, %ymm1 {%k1}
; AVX512VL-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX512VL-NEXT: vminpd %xmm0, %xmm1, %xmm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 | (xmm1 & xmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 | (xmm2 & xmm1)
; AVX512VL-NEXT: vcmpunordpd %xmm1, %xmm1, %k1
; AVX512VL-NEXT: vmovapd %xmm1, %xmm3 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm3[1,0]
@@ -1184,7 +1184,7 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vminpd %zmm1, %zmm0, %zmm1
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & zmm2)
+; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm2 & zmm0)
; AVX512BW-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512BW-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512BW-NEXT: vextractf64x4 $1, %zmm1, %ymm0
@@ -1212,20 +1212,19 @@ define double @test_v16f64(<16 x double> %a0) {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vminpd %zmm1, %zmm0, %zmm1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & zmm2)
+; AVX512VL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm2 & zmm0)
; AVX512VL-NEXT: vcmpunordpd %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vmovapd %zmm0, %zmm1 {%k1}
; AVX512VL-NEXT: vextractf64x4 $1, %zmm1, %ymm0
; AVX512VL-NEXT: vminpd %ymm0, %ymm1, %ymm0
-; AVX512VL-NEXT: vmovdqa %ymm2, %ymm3
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm1) | ymm0
+; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm1 & ymm2)
; AVX512VL-NEXT: vcmpunordpd %ymm1, %ymm1, %k1
-; AVX512VL-NEXT: vmovapd %ymm1, %ymm3 {%k1}
-; AVX512VL-NEXT: vextractf128 $1, %ymm3, %xmm0
-; AVX512VL-NEXT: vminpd %xmm0, %xmm3, %xmm1
-; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm3 & xmm2)
-; AVX512VL-NEXT: vcmpunordpd %xmm3, %xmm3, %k1
-; AVX512VL-NEXT: vmovapd %xmm3, %xmm1 {%k1}
+; AVX512VL-NEXT: vmovapd %ymm1, %ymm0 {%k1}
+; AVX512VL-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vminpd %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm2 & xmm0)
+; AVX512VL-NEXT: vcmpunordpd %xmm0, %xmm0, %k1
+; AVX512VL-NEXT: vmovapd %xmm0, %xmm1 {%k1}
; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
; AVX512VL-NEXT: vminsd %xmm0, %xmm1, %xmm0
; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm0 | (xmm1 & xmm2)
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 9ae98a2aa284f..3ae52d268813e 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1753,8 +1753,8 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v8i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %xmm0, %xmm1
-; AVX512VL-NEXT: vpsrlw $11, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsrlw $11, %xmm0, %xmm1
+; AVX512VL-NEXT: vpsllw $5, %xmm0, %xmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
; AVX512VL-NEXT: retq
;
@@ -1768,8 +1768,8 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v8i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %xmm0, %xmm1
-; AVX512VLBW-NEXT: vpsrlw $11, %xmm0, %xmm0
+; AVX512VLBW-NEXT: vpsrlw $11, %xmm0, %xmm1
+; AVX512VLBW-NEXT: vpsllw $5, %xmm0, %xmm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 48083ac9e810a..137deef7f8b86 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -1571,8 +1571,8 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v16i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm1
-; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm0
+; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm1
+; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
; AVX512VL-NEXT: retq
;
@@ -1586,8 +1586,8 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v16i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %ymm0, %ymm1
-; AVX512VLBW-NEXT: vpsrlw $11, %ymm0, %ymm0
+; AVX512VLBW-NEXT: vpsrlw $11, %ymm0, %ymm1
+; AVX512VLBW-NEXT: vpsllw $5, %ymm0, %ymm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index cea29414fe808..dd8aeb66499b1 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -847,39 +847,39 @@ define <16 x i32> @splatconstant_rotate_mask_v16i32(<16 x i32> %a) nounwind {
define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
; AVX512F-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpsllw $5, %ymm0, %ymm1
+; AVX512F-NEXT: vpsrlw $11, %ymm0, %ymm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT: vpsllw $5, %ymm2, %ymm3
+; AVX512F-NEXT: vpsrlw $11, %ymm2, %ymm3
; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512F-NEXT: vpsrlw $11, %ymm0, %ymm0
-; AVX512F-NEXT: vpsrlw $11, %ymm2, %ymm2
+; AVX512F-NEXT: vpsllw $5, %ymm0, %ymm0
+; AVX512F-NEXT: vpsllw $5, %ymm2, %ymm2
; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm1
+; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm1
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT: vpsllw $5, %ymm2, %ymm3
+; AVX512VL-NEXT: vpsrlw $11, %ymm2, %ymm3
; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsrlw $11, %ymm0, %ymm0
-; AVX512VL-NEXT: vpsrlw $11, %ymm2, %ymm2
+; AVX512VL-NEXT: vpsllw $5, %ymm0, %ymm0
+; AVX512VL-NEXT: vpsllw $5, %ymm2, %ymm2
; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsllw $5, %zmm0, %zmm1
-; AVX512BW-NEXT: vpsrlw $11, %zmm0, %zmm0
+; AVX512BW-NEXT: vpsrlw $11, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsllw $5, %zmm0, %zmm0
; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512BW-NEXT: retq
;
; AVX512VLBW-LABEL: splatconstant_rotate_mask_v32i16:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpsllw $5, %zmm0, %zmm1
-; AVX512VLBW-NEXT: vpsrlw $11, %zmm0, %zmm0
+; AVX512VLBW-NEXT: vpsrlw $11, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsllw $5, %zmm0, %zmm0
; AVX512VLBW-NEXT: vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
; AVX512VLBW-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 63554180959a9..0f22f16c78b66 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -945,9 +945,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512DQVL-LABEL: splatvar_shift_v32i8:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -1314,9 +1314,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi
; AVX512DQVL-LABEL: splatvar_modulo_shift_v32i8:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQVL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index 6cfb2bd08dc0b..7b35c00bd306f 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -191,9 +191,9 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512BW-LABEL: splatvar_shift_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
@@ -280,9 +280,9 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi
; AVX512BW-LABEL: splatvar_modulo_shift_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2
+; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1
; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vpternlog.ll b/llvm/test/CodeGen/X86/vpternlog.ll
index bd7478d3a82d5..d41d74953b719 100644
--- a/llvm/test/CodeGen/X86/vpternlog.ll
+++ b/llvm/test/CodeGen/X86/vpternlog.ll
@@ -4,7 +4,7 @@
define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
; CHECK-LABEL: foo:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm2 | zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm1 | zmm2)
; CHECK-NEXT: retq
%and.demorgan = or <8 x i64> %b, %a
%and3.demorgan = or <8 x i64> %and.demorgan, %c
@@ -15,7 +15,7 @@ define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
; CHECK-LABEL: xorbitcast:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm2 | zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm1 | zmm2)
; CHECK-NEXT: retq
%or1 = or <64 x i8> %a, %b
%or2 = or <64 x i8> %or1, %c
@@ -23,3 +23,169 @@ define <8 x i64> @xorbitcast(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c) {
%xor = xor <8 x i64> %cast, splat (i64 -1)
ret <8 x i64> %xor
}
+
+; Two chained 3-input VPTERNLOGs (5 distinct inputs).
+define <8 x i64> @foobar(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e) {
+; CHECK-LABEL: foobar:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & (zmm2 | ~zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & ~(zmm4 | zmm3)
+; CHECK-NEXT: retq
+ %nb = xor <8 x i64> %b, splat (i64 -1)
+ %or = or <8 x i64> %nb, %c
+ %foo = and <8 x i64> %or, %a
+ %de = or <8 x i64> %d, %e
+ %nde = xor <8 x i64> %de, splat (i64 -1)
+ %bar = and <8 x i64> %foo, %nde
+ ret <8 x i64> %bar
+}
+
+; A & ~(B | C) nested under another AND; the inner OR stays a plain vporq.
+define <8 x i64> @or_not_and(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
+; CHECK-LABEL: or_not_and:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vporq %zmm2, %zmm1, %zmm1
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm3 & ~zmm1
+; CHECK-NEXT: retq
+ %or = or <8 x i64> %b, %c
+ %not_or = xor <8 x i64> %or, splat (i64 -1)
+ %lhs = and <8 x i64> %a, %not_or
+ %res = and <8 x i64> %lhs, %d
+ ret <8 x i64> %res
+}
+
+; Depth-4 heterogeneous chain.
+define <8 x i64> @depth4_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e) {
+; CHECK-LABEL: depth4_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm2 | (zmm0 ^ zmm1)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm0 & zmm3)
+; CHECK-NEXT: retq
+ %t0 = xor <8 x i64> %a, %b
+ %t1 = or <8 x i64> %t0, %c
+ %t2 = and <8 x i64> %t1, %d
+ %t3 = xor <8 x i64> %t2, %e
+ ret <8 x i64> %t3
+}
+
+; Depth-5 heterogeneous chain.
+define <8 x i64> @depth5_chain(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; CHECK-LABEL: depth5_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpandq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm3 | (zmm0 ^ zmm2)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm0 & zmm4)
+; CHECK-NEXT: retq
+ %t0 = and <8 x i64> %a, %b
+ %t1 = xor <8 x i64> %t0, %c
+ %t2 = or <8 x i64> %t1, %d
+ %t3 = and <8 x i64> %t2, %e
+ %t4 = xor <8 x i64> %t3, %f
+ ret <8 x i64> %t4
+}
+
+; Balanced tree.
+define <8 x i64> @balanced_depth(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f) {
+; CHECK-LABEL: balanced_depth:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & (zmm2 ^ zmm3)
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm4 ^ zmm5)
+; CHECK-NEXT: retq
+ %l0 = or <8 x i64> %a, %b
+ %l1 = xor <8 x i64> %c, %d
+ %l2 = and <8 x i64> %l0, %l1
+ %r0 = xor <8 x i64> %e, %f
+ %res = or <8 x i64> %l2, %r0
+ ret <8 x i64> %res
+}
+
+; NOT of a load: folds the memory operand into a single VPTERNLOG.
+define <8 x i64> @flip(ptr %x) {
+; CHECK-LABEL: flip:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = ~mem
+; CHECK-NEXT: retq
+ %ld = load <8 x i64>, ptr %x, align 64
+ %not = xor <8 x i64> %ld, splat (i64 -1)
+ ret <8 x i64> %not
+}
+
+; 9-input OR reduction -> chain of 3-input VPTERNLOGs.
+define <8 x i64> @or_reduce9(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d, <8 x i64> %e, <8 x i64> %f, <8 x i64> %g, <8 x i64> %h, <8 x i64> %i) {
+; CHECK-LABEL: or_reduce9:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: movq %rsp, %rbp
+; CHECK-NEXT: .cfi_def_cfa_register %rbp
+; CHECK-NEXT: andq $-64, %rsp
+; CHECK-NEXT: subq $64, %rsp
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm2
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm3 | zmm4
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm5 | zmm6
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm7 | mem
+; CHECK-NEXT: movq %rbp, %rsp
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: .cfi_def_cfa %rsp, 8
+; CHECK-NEXT: retq
+ %r1 = or <8 x i64> %b, %a
+ %r2 = or <8 x i64> %r1, %c
+ %r3 = or <8 x i64> %r2, %d
+ %r4 = or <8 x i64> %r3, %e
+ %r5 = or <8 x i64> %r4, %f
+ %r6 = or <8 x i64> %r5, %g
+ %r7 = or <8 x i64> %r6, %h
+ %r8 = or <8 x i64> %r7, %i
+ ret <8 x i64> %r8
+}
+
+; 256-bit vector case - exercises VLX path.
+define <4 x i64> @foo_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
+; CHECK-LABEL: foo_256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} ymm0 = ~(ymm0 | ymm1 | ymm2)
+; CHECK-NEXT: retq
+ %or1 = or <4 x i64> %b, %a
+ %or2 = or <4 x i64> %or1, %c
+ %not = xor <4 x i64> %or2, splat (i64 -1)
+ ret <4 x i64> %not
+}
+
+; 128-bit vector case - exercises VLX path.
+define <2 x i64> @foo_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: foo_128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} xmm0 = ~(xmm0 | xmm1 | xmm2)
+; CHECK-NEXT: retq
+ %or1 = or <2 x i64> %b, %a
+ %or2 = or <2 x i64> %or1, %c
+ %not = xor <2 x i64> %or2, splat (i64 -1)
+ ret <2 x i64> %not
+}
+
+; Balanced OR-4 tree; one leaf OR stays a plain vporq.
+define <8 x i64> @balanced_or4(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
+; CHECK-LABEL: balanced_or4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | zmm2 | zmm3
+; CHECK-NEXT: retq
+ %t1 = or <8 x i64> %a, %b
+ %t2 = or <8 x i64> %c, %d
+ %res = or <8 x i64> %t1, %t2
+ ret <8 x i64> %res
+}
+
+; Multi-use operand - %a feeds both ORs; reuse collapses to one VPTERNLOG.
+define <8 x i64> @shared_operand(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
+; CHECK-LABEL: shared_operand:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & zmm2)
+; CHECK-NEXT: retq
+ %or1 = or <8 x i64> %a, %b
+ %or2 = or <8 x i64> %a, %c
+ %res = and <8 x i64> %or1, %or2
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index a28c23bb8a61e..39fd3fb59995a 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -2731,14 +2731,12 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
@@ -2750,14 +2748,12 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3023,14 +3019,12 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3042,14 +3036,12 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb (%rsi), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
@@ -3314,14 +3306,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = ymm0[0,1,0,1]
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
+; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
@@ -3334,14 +3325,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
+; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
index 21a3df0456de1..4a2873070c0a4 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -2149,13 +2149,11 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2166,13 +2164,11 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogd {{.*#+}} ymm2 = ymm2 ^ (m32bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2399,13 +2395,11 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2416,13 +2410,11 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm3
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpbroadcastb %xmm0, %ymm2
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (m64bcst & (ymm2 ^ ymm1))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2649,14 +2641,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512F-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -2667,14 +2658,13 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpandn %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = mem & (ymm2 | ymm1)
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]
+; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm2 ^ (ymm3 & (ymm1 ^ ymm2))
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
More information about the llvm-commits
mailing list