[llvm] [LLVM][SelectionDAG] Fix incorrect splitting of VECTOR_COMPRESS when passthrough operand is not undef. (PR #215772)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 03:11:32 PDT 2026
https://github.com/paulwalker-arm updated https://github.com/llvm/llvm-project/pull/215772
>From 59d4e1ee1e535e6736e5a597d6dae4e80100fbf7 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Tue, 11 Aug 2026 16:25:38 +0000
Subject: [PATCH] [LLVM][SelectionDAG] Fix incorrect splitting of
VECTOR_COMPRESS when passthrough operand is not undef.
When splitting VECTOR_COMPRESS the original mask is used for the final
step when inactive elements are set to the passthough value. This is
incorrect because only the inactive lanes of the result should be set
to their passthrough value.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 18 +-
llvm/test/CodeGen/X86/vector-compress.ll | 286 +++++++++++-------
2 files changed, 190 insertions(+), 114 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 0215f8f5e00c0..b7f0a8d48e4b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2907,8 +2907,22 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
SDValue Compressed = DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
if (!Passthru.isUndef()) {
- Compressed =
- DAG.getNode(ISD::VSELECT, DL, VecVT, Mask, Compressed, Passthru);
+ // Compress the input mask so only inactive lanes of the result are replaced
+ // by their passthrough value.
+ EVT MaskVT = Mask.getValueType();
+ EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+ MaskVT.getVectorElementCount());
+ SDValue WideMask = DAG.getNode(ISD::ZERO_EXTEND, DL, WideMaskVT, Mask);
+ SDValue NumActiveElts =
+ DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, WideMask);
+
+ SDValue StepVector = DAG.getStepVector(DL, WideMaskVT);
+ SDValue SplatNumActiveElts = DAG.getSplat(WideMaskVT, DL, NumActiveElts);
+ SDValue CompressedMask =
+ DAG.getSetCC(DL, MaskVT, StepVector, SplatNumActiveElts, ISD::SETULT);
+
+ Compressed = DAG.getNode(ISD::VSELECT, DL, VecVT, CompressedMask,
+ Compressed, Passthru);
}
std::tie(Lo, Hi) = DAG.SplitVector(Compressed, DL);
}
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 7f19806114894..d5872b1a2a351 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -1569,33 +1569,56 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512F-NEXT: movq %rsp, %rbp
; AVX512F-NEXT: andq $-32, %rsp
; AVX512F-NEXT: subq $64, %rsp
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
-; AVX512F-NEXT: vpslld $31, %zmm3, %zmm3
-; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm4
-; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
-; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k1
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm4, (%rsp)
-; AVX512F-NEXT: vptestmd %zmm3, %zmm3, %k1
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: vmovd %xmm3, %ecx
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm5
+; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
+; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k1
+; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm5
+; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
+; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k2
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
+; AVX512F-NEXT: vpmovdb %zmm5, (%rsp)
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm6
+; AVX512F-NEXT: vpaddd %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX512F-NEXT: vpaddd %xmm6, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm6, %xmm1, %xmm1
+; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rcx)
-; AVX512F-NEXT: vpsllw $7, %ymm1, %ymm0
+; AVX512F-NEXT: vpand %xmm5, %xmm3, %xmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm1
+; AVX512F-NEXT: vpsrad $31, %zmm1, %zmm1
+; AVX512F-NEXT: vpsubd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512F-NEXT: vmovd %xmm0, %ecx
+; AVX512F-NEXT: addl %eax, %ecx
+; AVX512F-NEXT: vpbroadcastd %ecx, %zmm0
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm0
; AVX512F-NEXT: movq %rbp, %rsp
; AVX512F-NEXT: popq %rbp
@@ -2667,68 +2690,68 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
; AVX512F-NEXT: vpmovqw %zmm3, %xmm3
; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vptestmd %zmm4, %zmm2, %k1
; AVX512F-NEXT: vmovdqu64 224(%rbp), %zmm2
-; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm4
+; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm3
; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
-; AVX512F-NEXT: vpmovqw %zmm4, %xmm4
-; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm4
+; AVX512F-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm3
; AVX512F-NEXT: vmovdqu64 96(%rbp), %zmm2
-; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm6
+; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm5
; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
-; AVX512F-NEXT: vpmovqw %zmm6, %xmm6
-; AVX512F-NEXT: vinserti128 $1, %xmm6, %ymm2, %ymm2
-; AVX512F-NEXT: vmovd %edi, %xmm6
-; AVX512F-NEXT: vpinsrb $1, %esi, %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $2, %edx, %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm7 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k2
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm6
-; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm8
-; AVX512F-NEXT: vpaddd %ymm6, %ymm8, %ymm6
-; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm8
-; AVX512F-NEXT: vpaddd %xmm6, %xmm8, %xmm6
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm6, %xmm8, %xmm6
-; AVX512F-NEXT: vptestmd %zmm3, %zmm5, %k1
+; AVX512F-NEXT: vpmovqw %zmm5, %xmm5
+; AVX512F-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT: vmovd %edi, %xmm5
+; AVX512F-NEXT: vpinsrb $1, %esi, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $2, %edx, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k3
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
+; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k3} {z}
; AVX512F-NEXT: vpmovdb %zmm5, (%rsp)
-; AVX512F-NEXT: vptestmd %zmm3, %zmm7, %k2
-; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
-; AVX512F-NEXT: vpextrd $1, %xmm6, %eax
-; AVX512F-NEXT: vmovd %xmm6, %ecx
+; AVX512F-NEXT: vptestmd %zmm4, %zmm2, %k2
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k3} {z} = -1
+; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm5
+; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm6
+; AVX512F-NEXT: vpaddd %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm6
+; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT: vpextrd $1, %xmm5, %eax
+; AVX512F-NEXT: vmovd %xmm5, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm5
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
; AVX512F-NEXT: vpmovdb %zmm5, (%rsp,%rcx)
-; AVX512F-NEXT: vptestmd %zmm3, %zmm4, %k3
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k3} {z}
-; AVX512F-NEXT: vpmovdb %zmm3, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT: vptestmd %zmm4, %zmm3, %k3
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k3} {z}
+; AVX512F-NEXT: vpmovdb %zmm0, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k3} {z} = -1
+; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT: vpaddd %ymm4, %ymm0, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
@@ -2737,14 +2760,14 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vmovd %xmm4, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, 32(%rsp,%rcx)
-; AVX512F-NEXT: vmovdqa (%rsp), %ymm0
-; AVX512F-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k1} {z}
+; AVX512F-NEXT: vpmovdb %zmm3, 32(%rsp,%rcx)
+; AVX512F-NEXT: vmovdqa (%rsp), %ymm3
+; AVX512F-NEXT: vmovdqa %ymm3, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1
+; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
; AVX512F-NEXT: vpsubd %zmm2, %zmm4, %zmm4
; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
@@ -2758,17 +2781,39 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: andl $63, %ecx
; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %ymm4
; AVX512F-NEXT: vmovdqa %ymm4, 64(%rsp,%rcx)
-; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
-; AVX512F-NEXT: vpmovdb %zmm4, %xmm4
-; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; AVX512F-NEXT: vpblendvb %ymm3, {{[0-9]+}}(%rsp), %ymm4, %ymm3
+; AVX512F-NEXT: vpsrld $31, %zmm4, %zmm4
+; AVX512F-NEXT: vpsubd %zmm3, %zmm4, %zmm3
+; AVX512F-NEXT: vpsubd %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512F-NEXT: vmovd %xmm0, %ecx
+; AVX512F-NEXT: addl %eax, %ecx
+; AVX512F-NEXT: vpbroadcastd %ecx, %zmm0
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
; AVX512F-NEXT: vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT: vpblendvb %ymm2, {{[0-9]+}}(%rsp), %ymm3, %ymm2
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, {{[0-9]+}}(%rsp), %ymm1, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512F-NEXT: movq %rbp, %rsp
; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
@@ -3576,42 +3621,59 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX512F-NEXT: movq %rsp, %rbp
; AVX512F-NEXT: andq $-64, %rsp
; AVX512F-NEXT: addq $-128, %rsp
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
-; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
-; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm5
; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k1
+; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm5
+; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
+; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k2
; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k1} {z}
+; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
; AVX512F-NEXT: vpmovdw %zmm5, (%rsp)
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm5
-; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k1
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm6
+; AVX512F-NEXT: vpaddd %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX512F-NEXT: vpaddd %xmm6, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm6, %xmm1, %xmm1
+; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rcx,2)
-; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
-; AVX512F-NEXT: vpsllw $15, %ymm3, %ymm3
-; AVX512F-NEXT: vpsraw $15, %ymm3, %ymm3
-; AVX512F-NEXT: vpblendvb %ymm3, {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512F-NEXT: vpsllw $15, %ymm1, %ymm1
-; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm1
-; AVX512F-NEXT: vpblendvb %ymm1, (%rsp), %ymm2, %ymm1
-; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vpand %xmm5, %xmm3, %xmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm1
+; AVX512F-NEXT: vpsrad $31, %zmm1, %zmm1
+; AVX512F-NEXT: vpsubd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512F-NEXT: vmovd %xmm0, %ecx
+; AVX512F-NEXT: addl %eax, %ecx
+; AVX512F-NEXT: vpbroadcastd %ecx, %zmm0
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512F-NEXT: vpblendvb %ymm1, {{[0-9]+}}(%rsp), %ymm3, %ymm1
+; AVX512F-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512F-NEXT: movq %rbp, %rsp
; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
More information about the llvm-commits
mailing list