[llvm] [SelectionDAG] Freeze dynamic vector indices lowered through the stack (PR #225031)
Mahmoud Naderi via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 06:29:01 PDT 2026
https://github.com/slant14 updated https://github.com/llvm/llvm-project/pull/225031
>From 9ad6123b9be8185872a756d5abe3ce7789230357 Mon Sep 17 00:00:00 2001
From: Mahmoud <mahmoud at Mahmouds-MacBook-Pro.local>
Date: Mon, 21 Sep 2026 11:48:53 +0300
Subject: [PATCH 1/2] [SelectionDAG] Freeze dynamic vector indices lowered
through the stack
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 6 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 3 +
llvm/test/CodeGen/X86/freeze-vector.ll | 181 ++++++++++++++++++
llvm/test/CodeGen/X86/var-permute-128.ll | 68 ++++---
llvm/test/CodeGen/X86/vecloadextract.ll | 6 +-
.../CodeGen/X86/vector-extract-last-active.ll | 12 +-
7 files changed, 243 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d4a7403a6b25f..c6f842a1d2568 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -25358,6 +25358,9 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
if (!IsByteSized && Elmnt.getValueType().bitsLT(EltVT))
Elmnt = DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Elmnt);
+ // Freeze the index, as clamping a poison index would be meaningless.
+ Index = DAG.getFreeze(Index);
+
// Store the new element. This may be larger than the vector element
// type, so use a truncating store.
SDValue EltPtr =
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index dc3b589087421..844ffe455d8a1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2295,6 +2295,9 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
}
}
+ // Freeze the index, as clamping a poison index would be meaningless.
+ Idx = DAG.getFreeze(Idx);
+
// Make the vector elements byte-addressable if they aren't already.
EVT VecVT = Vec.getValueType();
EVT EltVT = VecVT.getVectorElementType();
@@ -4358,6 +4361,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
if (CustomLowerNode(N, N->getValueType(0), true))
return SDValue();
+ // Freeze the index, as clamping a poison index would be meaningless.
+ Idx = DAG.getFreeze(Idx);
+
// Make the vector elements byte-addressable if they aren't already.
SDLoc dl(N);
EVT EltVT = VecVT.getVectorElementType();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 6e559d91952f2..248302669ed1c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -14284,6 +14284,9 @@ SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
!IsFast)
return SDValue();
+ // Freeze EltNo, as clamping a poison index would be meaningless.
+ EltNo = DAG.getFreeze(EltNo);
+
// The original DAG loaded the entire vector from memory, so arithmetic
// within it must be inbounds.
SDValue NewPtr = getInboundsVectorElementPointer(
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 7062ace700512..54a9bdc851cdb 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -944,3 +944,184 @@ define <2 x i64> @freeze_vselect_knownbits(<8 x i32> %csrc, <4 x i32> %a, <4 x i
%add = add <2 x i64> %ext, splat (i64 16)
ret <2 x i64> %add
}
+
+; A dynamic insert/extract that is legalized through a stack slot must freeze
+; the index before clamping it, or the clamp can be optimized away and the
+; store/load uses an unbounded offset.
+; https://github.com/llvm/llvm-project/issues/224200
+define void @freeze_dynamic_insertelement_wide_vector(ptr %vp, i32 %x, ptr %dst) nounwind {
+; X86-LABEL: freeze_dynamic_insertelement_wide_vector:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: andl $-32, %esp
+; X86-NEXT: subl $160, %esp
+; X86-NEXT: movl 16(%ebp), %eax
+; X86-NEXT: movl 8(%ebp), %ecx
+; X86-NEXT: vmovaps (%ecx), %ymm0
+; X86-NEXT: vmovaps 32(%ecx), %ymm1
+; X86-NEXT: vmovaps 64(%ecx), %ymm2
+; X86-NEXT: vmovaps 96(%ecx), %ymm3
+; X86-NEXT: bsrl 12(%ebp), %ecx
+; X86-NEXT: notl %ecx
+; X86-NEXT: andl $31, %ecx
+; X86-NEXT: vmovaps %ymm3, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm2, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm0, (%esp)
+; X86-NEXT: movl $5, (%esp,%ecx,4)
+; X86-NEXT: vmovaps (%esp), %ymm0
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm1
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm2
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm3
+; X86-NEXT: vmovaps %ymm3, 64(%eax)
+; X86-NEXT: vmovaps %ymm2, 96(%eax)
+; X86-NEXT: vmovaps %ymm0, (%eax)
+; X86-NEXT: vmovaps %ymm1, 32(%eax)
+; X86-NEXT: movl %ebp, %esp
+; X86-NEXT: popl %ebp
+; X86-NEXT: vzeroupper
+; X86-NEXT: retl
+;
+; X64-LABEL: freeze_dynamic_insertelement_wide_vector:
+; X64: # %bb.0:
+; X64-NEXT: pushq %rbp
+; X64-NEXT: movq %rsp, %rbp
+; X64-NEXT: andq $-32, %rsp
+; X64-NEXT: subq $160, %rsp
+; X64-NEXT: vmovaps (%rdi), %ymm0
+; X64-NEXT: vmovaps 32(%rdi), %ymm1
+; X64-NEXT: vmovaps 64(%rdi), %ymm2
+; X64-NEXT: vmovaps 96(%rdi), %ymm3
+; X64-NEXT: bsrl %esi, %eax
+; X64-NEXT: notl %eax
+; X64-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm2, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm0, (%rsp)
+; X64-NEXT: andl $31, %eax
+; X64-NEXT: movl $5, (%rsp,%rax,4)
+; X64-NEXT: vmovaps (%rsp), %ymm0
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm2
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm3
+; X64-NEXT: vmovaps %ymm2, 64(%rdx)
+; X64-NEXT: vmovaps %ymm3, 96(%rdx)
+; X64-NEXT: vmovaps %ymm0, (%rdx)
+; X64-NEXT: vmovaps %ymm1, 32(%rdx)
+; X64-NEXT: movq %rbp, %rsp
+; X64-NEXT: popq %rbp
+; X64-NEXT: vzeroupper
+; X64-NEXT: retq
+ %v = load <32 x i32>, ptr %vp
+ %idx = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+ %ins = insertelement <32 x i32> %v, i32 5, i32 %idx
+ store <32 x i32> %ins, ptr %dst
+ ret void
+}
+
+define i32 @freeze_dynamic_extractelement_wide_vector(ptr %vp, i32 %x) nounwind {
+; X86-LABEL: freeze_dynamic_extractelement_wide_vector:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: bsrl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: notl %ecx
+; X86-NEXT: andl $31, %ecx
+; X86-NEXT: movl (%eax,%ecx,4), %eax
+; X86-NEXT: retl
+;
+; X64-LABEL: freeze_dynamic_extractelement_wide_vector:
+; X64: # %bb.0:
+; X64-NEXT: bsrl %esi, %eax
+; X64-NEXT: notl %eax
+; X64-NEXT: andl $31, %eax
+; X64-NEXT: movl (%rdi,%rax,4), %eax
+; X64-NEXT: retq
+ %v = load <32 x i32>, ptr %vp
+ %idx = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+ %r = extractelement <32 x i32> %v, i32 %idx
+ ret i32 %r
+}
+
+define void @freeze_dynamic_insertelement_chain_wide_vector(ptr %vp, i32 %x, i32 %y, ptr %dst) nounwind {
+; X86-LABEL: freeze_dynamic_insertelement_chain_wide_vector:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: andl $-32, %esp
+; X86-NEXT: subl $160, %esp
+; X86-NEXT: movl 20(%ebp), %eax
+; X86-NEXT: movl 8(%ebp), %ecx
+; X86-NEXT: vmovaps (%ecx), %ymm0
+; X86-NEXT: vmovaps 32(%ecx), %ymm1
+; X86-NEXT: vmovaps 64(%ecx), %ymm2
+; X86-NEXT: vmovaps 96(%ecx), %ymm3
+; X86-NEXT: bsrl 12(%ebp), %ecx
+; X86-NEXT: notl %ecx
+; X86-NEXT: andl $31, %ecx
+; X86-NEXT: bsrl 16(%ebp), %edx
+; X86-NEXT: notl %edx
+; X86-NEXT: andl $31, %edx
+; X86-NEXT: vmovaps %ymm3, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm2, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
+; X86-NEXT: vmovaps %ymm0, (%esp)
+; X86-NEXT: movl $5, (%esp,%ecx,4)
+; X86-NEXT: movl $7, (%esp,%edx,4)
+; X86-NEXT: vmovaps (%esp), %ymm0
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm1
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm2
+; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm3
+; X86-NEXT: vmovaps %ymm3, 64(%eax)
+; X86-NEXT: vmovaps %ymm2, 96(%eax)
+; X86-NEXT: vmovaps %ymm0, (%eax)
+; X86-NEXT: vmovaps %ymm1, 32(%eax)
+; X86-NEXT: movl %ebp, %esp
+; X86-NEXT: popl %ebp
+; X86-NEXT: vzeroupper
+; X86-NEXT: retl
+;
+; X64-LABEL: freeze_dynamic_insertelement_chain_wide_vector:
+; X64: # %bb.0:
+; X64-NEXT: pushq %rbp
+; X64-NEXT: movq %rsp, %rbp
+; X64-NEXT: andq $-32, %rsp
+; X64-NEXT: subq $160, %rsp
+; X64-NEXT: vmovaps (%rdi), %ymm0
+; X64-NEXT: vmovaps 32(%rdi), %ymm1
+; X64-NEXT: vmovaps 64(%rdi), %ymm2
+; X64-NEXT: vmovaps 96(%rdi), %ymm3
+; X64-NEXT: bsrl %esi, %eax
+; X64-NEXT: notl %eax
+; X64-NEXT: bsrl %edx, %edx
+; X64-NEXT: notl %edx
+; X64-NEXT: andl $31, %eax
+; X64-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm2, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm1, {{[0-9]+}}(%rsp)
+; X64-NEXT: vmovaps %ymm0, (%rsp)
+; X64-NEXT: movl $5, (%rsp,%rax,4)
+; X64-NEXT: andl $31, %edx
+; X64-NEXT: movl $7, (%rsp,%rdx,4)
+; X64-NEXT: vmovaps (%rsp), %ymm0
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm2
+; X64-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm3
+; X64-NEXT: vmovaps %ymm2, 64(%rcx)
+; X64-NEXT: vmovaps %ymm3, 96(%rcx)
+; X64-NEXT: vmovaps %ymm0, (%rcx)
+; X64-NEXT: vmovaps %ymm1, 32(%rcx)
+; X64-NEXT: movq %rbp, %rsp
+; X64-NEXT: popq %rbp
+; X64-NEXT: vzeroupper
+; X64-NEXT: retq
+ %v = load <32 x i32>, ptr %vp
+ %idx0 = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+ %idx1 = call i32 @llvm.ctlz.i32(i32 %y, i1 true)
+ %ins0 = insertelement <32 x i32> %v, i32 5, i32 %idx0
+ %ins1 = insertelement <32 x i32> %ins0, i32 7, i32 %idx1
+ store <32 x i32> %ins1, ptr %dst
+ ret void
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1)
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index a66c074425374..2b3abf2de5ba9 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -2026,46 +2026,56 @@ define <16 x i8> @var_shuffle_v16i8_from_v32i8_v16i8(<32 x i8> %v, <16 x i8> %in
define void @indices_convert() {
; SSE3-LABEL: indices_convert:
; SSE3: # %bb.0: # %bb
-; SSE3-NEXT: movaps (%rax), %xmm0
-; SSE3-NEXT: movaps %xmm0, -24(%rsp)
-; SSE3-NEXT: movaps %xmm0, -40(%rsp)
-; SSE3-NEXT: movl (%rax), %eax
-; SSE3-NEXT: movaps %xmm0, -56(%rsp)
-; SSE3-NEXT: movaps %xmm0, -72(%rsp)
+; SSE3-NEXT: movdqa (%rax), %xmm0
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE3-NEXT: movd %xmm1, %eax
+; SSE3-NEXT: movdqa %xmm0, -24(%rsp)
+; SSE3-NEXT: movdqa %xmm0, -40(%rsp)
; SSE3-NEXT: andl $3, %eax
-; SSE3-NEXT: shll $3, %eax
-; SSE3-NEXT: movsd -72(%rsp,%rax), %xmm0 # xmm0 = mem[0],zero
-; SSE3-NEXT: movsd -40(%rsp,%rax), %xmm1 # xmm1 = mem[0],zero
-; SSE3-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE3-NEXT: movups %xmm1, (%rax)
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE3-NEXT: movd %xmm1, %ecx
+; SSE3-NEXT: movdqa %xmm0, -56(%rsp)
+; SSE3-NEXT: movdqa %xmm0, -72(%rsp)
+; SSE3-NEXT: andl $3, %ecx
+; SSE3-NEXT: movsd -40(%rsp,%rax,8), %xmm0 # xmm0 = mem[0],zero
+; SSE3-NEXT: movsd -72(%rsp,%rcx,8), %xmm1 # xmm1 = mem[0],zero
+; SSE3-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE3-NEXT: movups %xmm0, (%rax)
; SSE3-NEXT: retq
;
; SSSE3-LABEL: indices_convert:
; SSSE3: # %bb.0: # %bb
-; SSSE3-NEXT: movaps (%rax), %xmm0
-; SSSE3-NEXT: movaps %xmm0, -24(%rsp)
-; SSSE3-NEXT: movaps %xmm0, -40(%rsp)
-; SSSE3-NEXT: movl (%rax), %eax
-; SSSE3-NEXT: movaps %xmm0, -56(%rsp)
-; SSSE3-NEXT: movaps %xmm0, -72(%rsp)
+; SSSE3-NEXT: movdqa (%rax), %xmm0
+; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movd %xmm1, %eax
+; SSSE3-NEXT: movdqa %xmm0, -24(%rsp)
+; SSSE3-NEXT: movdqa %xmm0, -40(%rsp)
; SSSE3-NEXT: andl $3, %eax
-; SSSE3-NEXT: shll $3, %eax
-; SSSE3-NEXT: movsd -72(%rsp,%rax), %xmm0 # xmm0 = mem[0],zero
-; SSSE3-NEXT: movsd -40(%rsp,%rax), %xmm1 # xmm1 = mem[0],zero
-; SSSE3-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT: movups %xmm1, (%rax)
+; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSSE3-NEXT: movd %xmm1, %ecx
+; SSSE3-NEXT: movdqa %xmm0, -56(%rsp)
+; SSSE3-NEXT: movdqa %xmm0, -72(%rsp)
+; SSSE3-NEXT: andl $3, %ecx
+; SSSE3-NEXT: movsd -40(%rsp,%rax,8), %xmm0 # xmm0 = mem[0],zero
+; SSSE3-NEXT: movsd -72(%rsp,%rcx,8), %xmm1 # xmm1 = mem[0],zero
+; SSSE3-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT: movups %xmm0, (%rax)
; SSSE3-NEXT: retq
;
; SSE41-LABEL: indices_convert:
; SSE41: # %bb.0: # %bb
-; SSE41-NEXT: movaps (%rax), %xmm0
-; SSE41-NEXT: extractps $2, %xmm0, %eax
-; SSE41-NEXT: movaps %xmm0, -24(%rsp)
-; SSE41-NEXT: movaps %xmm0, -40(%rsp)
+; SSE41-NEXT: movdqa (%rax), %xmm0
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: movd %xmm1, %eax
+; SSE41-NEXT: movdqa %xmm0, -24(%rsp)
+; SSE41-NEXT: movdqa %xmm0, -40(%rsp)
; SSE41-NEXT: andl $3, %eax
-; SSE41-NEXT: extractps $3, %xmm0, %ecx
-; SSE41-NEXT: movaps %xmm0, -56(%rsp)
-; SSE41-NEXT: movaps %xmm0, -72(%rsp)
+; SSE41-NEXT: pextrd $1, %xmm1, %ecx
+; SSE41-NEXT: movdqa %xmm0, -56(%rsp)
+; SSE41-NEXT: movdqa %xmm0, -72(%rsp)
; SSE41-NEXT: andl $3, %ecx
; SSE41-NEXT: movsd -72(%rsp,%rcx,8), %xmm0 # xmm0 = mem[0],zero
; SSE41-NEXT: movsd -40(%rsp,%rax,8), %xmm1 # xmm1 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/vecloadextract.ll b/llvm/test/CodeGen/X86/vecloadextract.ll
index ad5fc22b1e6af..90c44efb9234a 100644
--- a/llvm/test/CodeGen/X86/vecloadextract.ll
+++ b/llvm/test/CodeGen/X86/vecloadextract.ll
@@ -20,7 +20,8 @@ define i32 @const_index(ptr %v) {
; CHECK: name: variable_index
; CHECK: bb.0 (%ir-block.0):
; CHECK: [[INDEX:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
-; CHECK: [[MASKED_INDEX:%[0-9]+]]:gr32_nosp = AND32ri [[INDEX]], 7, implicit-def dead $eflags
+; CHECK: [[FROZEN_INDEX:%[0-9]+]]:gr32 = COPY killed [[INDEX]]
+; CHECK: [[MASKED_INDEX:%[0-9]+]]:gr32_nosp = AND32ri [[FROZEN_INDEX]], 7, implicit-def dead $eflags
; CHECK: [[POINTER:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.1)
; CHECK: [[LOAD:%[0-9]+]]:gr32 = MOV32rm killed [[POINTER]], 4, killed [[MASKED_INDEX]], 0, $noreg :: (load (s32))
; CHECK: $eax = COPY [[LOAD]]
@@ -34,7 +35,8 @@ define i32 @variable_index(ptr %v, i32 %i) {
; CHECK: name: variable_index_with_addrspace
; CHECK: bb.0 (%ir-block.0):
; CHECK: [[INDEX:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
-; CHECK: [[MASKED_INDEX:%[0-9]+]]:gr32_nosp = AND32ri [[INDEX]], 7, implicit-def dead $eflags
+; CHECK: [[FROZEN_INDEX:%[0-9]+]]:gr32 = COPY killed [[INDEX]]
+; CHECK: [[MASKED_INDEX:%[0-9]+]]:gr32_nosp = AND32ri [[FROZEN_INDEX]], 7, implicit-def dead $eflags
; CHECK: [[POINTER:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.1)
; CHECK: [[LOAD:%[0-9]+]]:gr32 = MOV32rm killed [[POINTER]], 4, killed [[MASKED_INDEX]], 0, $noreg :: (load (s32), addrspace 1)
; CHECK: $eax = COPY [[LOAD]]
diff --git a/llvm/test/CodeGen/X86/vector-extract-last-active.ll b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
index 16a2de9783893..fcf9c295c3852 100644
--- a/llvm/test/CodeGen/X86/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
@@ -654,6 +654,7 @@ define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passt
; SSE2-NEXT: psrlw $8, %xmm5
; SSE2-NEXT: pmaxub %xmm1, %xmm5
; SSE2-NEXT: movd %xmm5, %eax
+; SSE2-NEXT: movzbl %al, %eax
; SSE2-NEXT: pmovmskb %xmm3, %ecx
; SSE2-NEXT: pandn %xmm0, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
@@ -667,7 +668,8 @@ define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passt
; SSE2-NEXT: psrlw $8, %xmm1
; SSE2-NEXT: pmaxub %xmm0, %xmm1
; SSE2-NEXT: movd %xmm1, %edx
-; SSE2-NEXT: addl $16, %edx
+; SSE2-NEXT: movzbl %dl, %edx
+; SSE2-NEXT: addq $16, %rdx
; SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; SSE2-NEXT: cmoveq %rax, %rdx
; SSE2-NEXT: andl $31, %edx
@@ -682,9 +684,9 @@ define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passt
; SSE42-LABEL: extract_last_active_split:
; SSE42: # %bb.0:
; SSE42-NEXT: pxor %xmm4, %xmm4
-; SSE42-NEXT: movdqa %xmm2, %xmm5
+; SSE42-NEXT: movdqa %xmm3, %xmm5
; SSE42-NEXT: pcmpeqb %xmm4, %xmm5
-; SSE42-NEXT: pcmpeqb %xmm3, %xmm4
+; SSE42-NEXT: pcmpeqb %xmm2, %xmm4
; SSE42-NEXT: pcmpeqd %xmm6, %xmm6
; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -698,7 +700,6 @@ define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passt
; SSE42-NEXT: movd %xmm1, %eax
; SSE42-NEXT: notb %al
; SSE42-NEXT: movzbl %al, %eax
-; SSE42-NEXT: addl $16, %eax
; SSE42-NEXT: pandn %xmm0, %xmm5
; SSE42-NEXT: pxor %xmm6, %xmm5
; SSE42-NEXT: movdqa %xmm5, %xmm0
@@ -708,8 +709,9 @@ define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passt
; SSE42-NEXT: movd %xmm0, %ecx
; SSE42-NEXT: notb %cl
; SSE42-NEXT: movzbl %cl, %ecx
+; SSE42-NEXT: addq $16, %rcx
; SSE42-NEXT: ptest %xmm3, %xmm3
-; SSE42-NEXT: cmovneq %rax, %rcx
+; SSE42-NEXT: cmoveq %rax, %rcx
; SSE42-NEXT: andl $31, %ecx
; SSE42-NEXT: por %xmm3, %xmm2
; SSE42-NEXT: ptest %xmm2, %xmm2
>From 321b039df60bffc764589960ddd925277c0f5968 Mon Sep 17 00:00:00 2001
From: Mahmoud <mahmoud at Mahmouds-MacBook-Pro.local>
Date: Mon, 21 Sep 2026 16:28:31 +0300
Subject: [PATCH 2/2] [SelectionDAG] Freeze the vector index in
getVectorSubVecPointer
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 -
.../SelectionDAG/LegalizeVectorTypes.cpp | 6 -
.../CodeGen/SelectionDAG/TargetLowering.cpp | 10 +-
.../CodeGen/AArch64/sve-vector-compress.ll | 22 +-
llvm/test/CodeGen/AArch64/vector-compress.ll | 179 +--
.../test/CodeGen/Mips/msa/basic_operations.ll | 8 +-
llvm/test/CodeGen/NVPTX/f32x2-instructions.ll | 15 +-
llvm/test/CodeGen/NVPTX/i32x2-instructions.ll | 15 +-
llvm/test/CodeGen/NVPTX/insertelt-dynamic.ll | 96 +-
.../WebAssembly/vector-extract-last-active.ll | 4 +
...ractelement-legalization-store-ordering.ll | 10 +-
llvm/test/CodeGen/X86/var-permute-128.ll | 61 +-
llvm/test/CodeGen/X86/vector-compress.ll | 1010 +++++++++--------
.../CodeGen/X86/vector-extract-last-active.ll | 3 +-
.../X86/vector-shuffle-variable-128.ll | 80 +-
15 files changed, 814 insertions(+), 708 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index c6f842a1d2568..d4a7403a6b25f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -25358,9 +25358,6 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
if (!IsByteSized && Elmnt.getValueType().bitsLT(EltVT))
Elmnt = DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Elmnt);
- // Freeze the index, as clamping a poison index would be meaningless.
- Index = DAG.getFreeze(Index);
-
// Store the new element. This may be larger than the vector element
// type, so use a truncating store.
SDValue EltPtr =
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 844ffe455d8a1..dc3b589087421 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2295,9 +2295,6 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
}
}
- // Freeze the index, as clamping a poison index would be meaningless.
- Idx = DAG.getFreeze(Idx);
-
// Make the vector elements byte-addressable if they aren't already.
EVT VecVT = Vec.getValueType();
EVT EltVT = VecVT.getVectorElementType();
@@ -4361,9 +4358,6 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
if (CustomLowerNode(N, N->getValueType(0), true))
return SDValue();
- // Freeze the index, as clamping a poison index would be meaningless.
- Idx = DAG.getFreeze(Idx);
-
// Make the vector elements byte-addressable if they aren't already.
SDLoc dl(N);
EVT EltVT = VecVT.getVectorElementType();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 248302669ed1c..314e94e0ce8de 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12185,6 +12185,13 @@ TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
"Converting bits to bytes lost precision");
assert(SubVecVT.getVectorElementType() == EltVT &&
"Sub-vector must be a vector with matching element type");
+
+ // An out-of-range index only makes the vector operation return poison, but
+ // a load/store through the pointer computed below would be immediate UB, so
+ // freeze the index before clamping it into range.
+ if (!DAG.isGuaranteedNotToBePoison(Index))
+ Index = DAG.getFreeze(Index);
+
Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
SubVecVT.getVectorElementCount());
@@ -14284,9 +14291,6 @@ SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
!IsFast)
return SDValue();
- // Freeze EltNo, as clamping a poison index would be meaningless.
- EltNo = DAG.getFreeze(EltNo);
-
// The original DAG loaded the entire vector from memory, so arithmetic
// within it must be inbounds.
SDValue NewPtr = getInboundsVectorElementPointer(
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-compress.ll b/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
index dd3269963595f..836265a2a7b6f 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
@@ -182,16 +182,17 @@ define <vscale x 8 x i32> @test_compress_large(<vscale x 8 x i32> %vec, <vscale
; CHECK-SVE-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
; CHECK-SVE-NEXT: .cfi_offset w29, -16
; CHECK-SVE-NEXT: punpklo p1.h, p0.b
-; CHECK-SVE-NEXT: mov x8, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov x9, #-1 // =0xffffffffffffffff
; CHECK-SVE-NEXT: punpkhi p0.h, p0.b
-; CHECK-SVE-NEXT: inch x8
-; CHECK-SVE-NEXT: cntp x9, p1, p1.s
+; CHECK-SVE-NEXT: inch x9
+; CHECK-SVE-NEXT: cntp x8, p1, p1.s
; CHECK-SVE-NEXT: compact z0.s, p1, z0.s
; CHECK-SVE-NEXT: compact z1.s, p0, z1.s
; CHECK-SVE-NEXT: ptrue p0.s
-; CHECK-SVE-NEXT: cmp x9, x8
+; CHECK-SVE-NEXT: mov w8, w8
; CHECK-SVE-NEXT: str z0, [sp]
-; CHECK-SVE-NEXT: csel x8, x9, x8, lo
+; CHECK-SVE-NEXT: cmp x8, x9
+; CHECK-SVE-NEXT: csel x8, x8, x9, lo
; CHECK-SVE-NEXT: mov x9, sp
; CHECK-SVE-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
; CHECK-SVE-NEXT: ldr z0, [sp]
@@ -209,17 +210,18 @@ define <vscale x 8 x i32> @test_compress_large(<vscale x 8 x i32> %vec, <vscale
; CHECK-SME2p2-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-SME2p2-NEXT: .cfi_offset w29, -16
; CHECK-SME2p2-NEXT: punpklo p1.h, p0.b
-; CHECK-SME2p2-NEXT: mov x8, #-1 // =0xffffffffffffffff
+; CHECK-SME2p2-NEXT: mov x9, #-1 // =0xffffffffffffffff
; CHECK-SME2p2-NEXT: ptrue pn8.s
; CHECK-SME2p2-NEXT: punpkhi p0.h, p0.b
-; CHECK-SME2p2-NEXT: inch x8
-; CHECK-SME2p2-NEXT: cntp x9, p1, p1.s
+; CHECK-SME2p2-NEXT: inch x9
+; CHECK-SME2p2-NEXT: cntp x8, p1, p1.s
; CHECK-SME2p2-NEXT: compact z0.s, p1, z0.s
; CHECK-SME2p2-NEXT: compact z1.s, p0, z1.s
; CHECK-SME2p2-NEXT: ptrue p0.s
-; CHECK-SME2p2-NEXT: cmp x9, x8
+; CHECK-SME2p2-NEXT: mov w8, w8
; CHECK-SME2p2-NEXT: str z0, [sp]
-; CHECK-SME2p2-NEXT: csel x8, x9, x8, lo
+; CHECK-SME2p2-NEXT: cmp x8, x9
+; CHECK-SME2p2-NEXT: csel x8, x8, x9, lo
; CHECK-SME2p2-NEXT: mov x9, sp
; CHECK-SME2p2-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
; CHECK-SME2p2-NEXT: ld1w { z0.s, z1.s }, pn8/z, [sp]
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 55c343164a1b8..56f6a1cd341d2 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -107,94 +107,97 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask) {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: shl.16b v1, v1, #7
-; CHECK-NEXT: mov x12, sp
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: str b0, [sp]
+; CHECK-NEXT: mov x10, sp
+; CHECK-NEXT: mov x11, sp
; CHECK-NEXT: mov x13, sp
+; CHECK-NEXT: str b0, [sp]
; CHECK-NEXT: cmlt.16b v1, v1, #0
-; CHECK-NEXT: umov.b w9, v1[0]
-; CHECK-NEXT: umov.b w10, v1[1]
-; CHECK-NEXT: umov.b w11, v1[2]
+; CHECK-NEXT: umov.b w8, v1[0]
+; CHECK-NEXT: umov.b w9, v1[1]
+; CHECK-NEXT: umov.b w12, v1[2]
; CHECK-NEXT: umov.b w14, v1[3]
-; CHECK-NEXT: bfxil x12, x9, #0, #1
-; CHECK-NEXT: and x10, x10, #0x1
+; CHECK-NEXT: bfxil x10, x8, #0, #1
; CHECK-NEXT: and x9, x9, #0x1
-; CHECK-NEXT: add x9, x9, x10
-; CHECK-NEXT: umov.b w10, v1[4]
-; CHECK-NEXT: and x11, x11, #0x1
-; CHECK-NEXT: st1.b { v0 }[1], [x12]
-; CHECK-NEXT: orr x12, x8, x9
-; CHECK-NEXT: add x9, x9, x11
-; CHECK-NEXT: umov.b w11, v1[5]
+; CHECK-NEXT: and x8, x8, #0x1
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: umov.b w9, v1[4]
+; CHECK-NEXT: and x12, x12, #0x1
+; CHECK-NEXT: bfxil x11, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x12
+; CHECK-NEXT: umov.b w12, v1[5]
+; CHECK-NEXT: st1.b { v0 }[1], [x10]
; CHECK-NEXT: and x14, x14, #0x1
-; CHECK-NEXT: st1.b { v0 }[2], [x12]
-; CHECK-NEXT: add x14, x9, x14
-; CHECK-NEXT: umov.b w12, v1[6]
-; CHECK-NEXT: orr x9, x8, x9
-; CHECK-NEXT: and x10, x10, #0x1
-; CHECK-NEXT: st1.b { v0 }[3], [x9]
-; CHECK-NEXT: orr x9, x8, x14
-; CHECK-NEXT: add x10, x14, x10
-; CHECK-NEXT: umov.b w14, v1[7]
-; CHECK-NEXT: st1.b { v0 }[4], [x9]
-; CHECK-NEXT: and x11, x11, #0x1
-; CHECK-NEXT: bfxil x13, x10, #0, #4
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x10, x10, x11
-; CHECK-NEXT: umov.b w11, v1[8]
+; CHECK-NEXT: bfxil x13, x8, #0, #4
+; CHECK-NEXT: st1.b { v0 }[2], [x11]
+; CHECK-NEXT: umov.b w11, v1[6]
+; CHECK-NEXT: mov x10, sp
+; CHECK-NEXT: add x8, x8, x14
+; CHECK-NEXT: and x9, x9, #0x1
+; CHECK-NEXT: mov x14, sp
+; CHECK-NEXT: bfxil x10, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: st1.b { v0 }[3], [x13]
+; CHECK-NEXT: umov.b w13, v1[7]
; CHECK-NEXT: and x12, x12, #0x1
-; CHECK-NEXT: bfxil x9, x10, #0, #4
-; CHECK-NEXT: st1.b { v0 }[5], [x13]
-; CHECK-NEXT: umov.b w13, v1[9]
-; CHECK-NEXT: add x10, x10, x12
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: bfxil x14, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x12
+; CHECK-NEXT: and x11, x11, #0x1
; CHECK-NEXT: mov x12, sp
-; CHECK-NEXT: and x14, x14, #0x1
+; CHECK-NEXT: st1.b { v0 }[4], [x10]
+; CHECK-NEXT: umov.b w10, v1[8]
+; CHECK-NEXT: bfxil x9, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x11
+; CHECK-NEXT: st1.b { v0 }[5], [x14]
+; CHECK-NEXT: umov.b w14, v1[9]
+; CHECK-NEXT: bfxil x12, x8, #0, #4
+; CHECK-NEXT: and x13, x13, #0x1
; CHECK-NEXT: st1.b { v0 }[6], [x9]
; CHECK-NEXT: umov.b w9, v1[10]
-; CHECK-NEXT: bfxil x12, x10, #0, #4
-; CHECK-NEXT: add x10, x10, x14
-; CHECK-NEXT: mov x14, sp
-; CHECK-NEXT: and x11, x11, #0x1
-; CHECK-NEXT: bfxil x14, x10, #0, #4
-; CHECK-NEXT: add x10, x10, x11
; CHECK-NEXT: mov x11, sp
-; CHECK-NEXT: and x13, x13, #0x1
+; CHECK-NEXT: add x8, x8, x13
; CHECK-NEXT: st1.b { v0 }[7], [x12]
-; CHECK-NEXT: mov x12, sp
-; CHECK-NEXT: bfxil x11, x10, #0, #4
-; CHECK-NEXT: add x10, x10, x13
-; CHECK-NEXT: umov.b w13, v1[11]
-; CHECK-NEXT: st1.b { v0 }[8], [x14]
-; CHECK-NEXT: umov.b w14, v1[12]
-; CHECK-NEXT: and x9, x9, #0x1
-; CHECK-NEXT: bfxil x12, x10, #0, #4
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: mov x10, sp
-; CHECK-NEXT: st1.b { v0 }[9], [x11]
-; CHECK-NEXT: umov.b w11, v1[13]
-; CHECK-NEXT: bfxil x10, x9, #0, #4
-; CHECK-NEXT: st1.b { v0 }[10], [x12]
-; CHECK-NEXT: umov.b w12, v1[14]
-; CHECK-NEXT: and x13, x13, #0x1
+; CHECK-NEXT: umov.b w12, v1[11]
+; CHECK-NEXT: and x10, x10, #0x1
+; CHECK-NEXT: bfxil x11, x8, #0, #4
+; CHECK-NEXT: mov x13, sp
+; CHECK-NEXT: add x8, x8, x10
; CHECK-NEXT: and x14, x14, #0x1
-; CHECK-NEXT: add x9, x9, x13
-; CHECK-NEXT: st1.b { v0 }[11], [x10]
; CHECK-NEXT: mov x10, sp
-; CHECK-NEXT: add x13, x9, x14
+; CHECK-NEXT: bfxil x13, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x14
+; CHECK-NEXT: st1.b { v0 }[8], [x11]
+; CHECK-NEXT: umov.b w11, v1[12]
+; CHECK-NEXT: and x9, x9, #0x1
; CHECK-NEXT: mov x14, sp
-; CHECK-NEXT: bfxil x10, x9, #0, #4
-; CHECK-NEXT: and x9, x11, #0x1
+; CHECK-NEXT: bfxil x10, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: and x9, x12, #0x1
+; CHECK-NEXT: umov.b w12, v1[13]
+; CHECK-NEXT: st1.b { v0 }[9], [x13]
+; CHECK-NEXT: bfxil x14, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: umov.b w13, v1[14]
+; CHECK-NEXT: bfxil x9, x8, #0, #4
+; CHECK-NEXT: st1.b { v0 }[10], [x10]
+; CHECK-NEXT: and x10, x11, #0x1
+; CHECK-NEXT: st1.b { v0 }[11], [x14]
+; CHECK-NEXT: add x8, x8, x10
+; CHECK-NEXT: mov x10, sp
+; CHECK-NEXT: st1.b { v0 }[12], [x9]
+; CHECK-NEXT: and x9, x12, #0x1
+; CHECK-NEXT: bfxil x10, x8, #0, #4
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: and w11, w13, #0x1
+; CHECK-NEXT: bfxil x9, x8, #0, #4
+; CHECK-NEXT: add w8, w8, w11
; CHECK-NEXT: mov x11, sp
-; CHECK-NEXT: add x9, x13, x9
-; CHECK-NEXT: and w12, w12, #0x1
-; CHECK-NEXT: bfxil x14, x13, #0, #4
-; CHECK-NEXT: bfxil x11, x9, #0, #4
-; CHECK-NEXT: add w9, w9, w12
-; CHECK-NEXT: st1.b { v0 }[12], [x10]
-; CHECK-NEXT: bfxil x8, x9, #0, #4
-; CHECK-NEXT: st1.b { v0 }[13], [x14]
-; CHECK-NEXT: st1.b { v0 }[14], [x11]
-; CHECK-NEXT: st1.b { v0 }[15], [x8]
+; CHECK-NEXT: bfxil x11, x8, #0, #4
+; CHECK-NEXT: st1.b { v0 }[13], [x10]
+; CHECK-NEXT: st1.b { v0 }[14], [x9]
+; CHECK-NEXT: st1.b { v0 }[15], [x11]
; CHECK-NEXT: ldr q0, [sp], #16
; CHECK-NEXT: ret
%out = call <16 x i8> @llvm.experimental.vector.compress(<16 x i8> %vec, <16 x i1> %mask, <16 x i8> undef)
@@ -358,14 +361,15 @@ define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) {
; CHECK-NEXT: and x10, x10, #0x1
; CHECK-NEXT: and x9, x9, #0x1
; CHECK-NEXT: add x9, x9, x10
-; CHECK-NEXT: and w11, w11, #0x1
; CHECK-NEXT: add x10, sp, #8
-; CHECK-NEXT: add w11, w9, w11
-; CHECK-NEXT: orr x9, x10, x9, lsl #1
+; CHECK-NEXT: and x11, x11, #0x1
+; CHECK-NEXT: bfi x10, x9, #1, #2
+; CHECK-NEXT: add x9, x9, x11
+; CHECK-NEXT: add x11, sp, #8
+; CHECK-NEXT: bfi x11, x9, #1, #2
; CHECK-NEXT: st1.h { v0 }[1], [x8]
-; CHECK-NEXT: bfi x10, x11, #1, #2
-; CHECK-NEXT: st1.h { v0 }[2], [x9]
-; CHECK-NEXT: st1.h { v0 }[3], [x10]
+; CHECK-NEXT: st1.h { v0 }[2], [x10]
+; CHECK-NEXT: st1.h { v0 }[3], [x11]
; CHECK-NEXT: ldr d0, [sp, #8]
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
@@ -390,14 +394,15 @@ define <4 x i4> @test_compress_illegal_element_type(<4 x i4> %vec, <4 x i1> %mas
; CHECK-NEXT: and x10, x10, #0x1
; CHECK-NEXT: and x9, x9, #0x1
; CHECK-NEXT: add x9, x9, x10
-; CHECK-NEXT: and w11, w11, #0x1
; CHECK-NEXT: add x10, sp, #8
-; CHECK-NEXT: add w11, w9, w11
-; CHECK-NEXT: orr x9, x10, x9, lsl #1
+; CHECK-NEXT: and x11, x11, #0x1
+; CHECK-NEXT: bfi x10, x9, #1, #2
+; CHECK-NEXT: add x9, x9, x11
+; CHECK-NEXT: add x11, sp, #8
+; CHECK-NEXT: bfi x11, x9, #1, #2
; CHECK-NEXT: st1.h { v0 }[1], [x8]
-; CHECK-NEXT: bfi x10, x11, #1, #2
-; CHECK-NEXT: st1.h { v0 }[2], [x9]
-; CHECK-NEXT: st1.h { v0 }[3], [x10]
+; CHECK-NEXT: st1.h { v0 }[2], [x10]
+; CHECK-NEXT: st1.h { v0 }[3], [x11]
; CHECK-NEXT: ldr d0, [sp, #8]
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
@@ -459,9 +464,9 @@ define <3 x i3> @test_compress_narrow_illegal_element_type(<3 x i3> %vec, <3 x i
; CHECK-NEXT: bfi x10, x8, #1, #1
; CHECK-NEXT: add x8, x11, x9
; CHECK-NEXT: add x9, sp, #8
-; CHECK-NEXT: orr x8, x9, x8, lsl #1
+; CHECK-NEXT: bfi x9, x8, #1, #2
; CHECK-NEXT: strh w1, [x10]
-; CHECK-NEXT: strh w2, [x8]
+; CHECK-NEXT: strh w2, [x9]
; CHECK-NEXT: ldrh w0, [sp, #8]
; CHECK-NEXT: ldrh w1, [sp, #10]
; CHECK-NEXT: ldrh w2, [sp, #12]
diff --git a/llvm/test/CodeGen/Mips/msa/basic_operations.ll b/llvm/test/CodeGen/Mips/msa/basic_operations.ll
index fea44035e42f9..e59cfd7cc750e 100644
--- a/llvm/test/CodeGen/Mips/msa/basic_operations.ll
+++ b/llvm/test/CodeGen/Mips/msa/basic_operations.ll
@@ -1904,7 +1904,7 @@ define void @insert_v16i8_vidx(i32 signext %a) nounwind {
; N64-NEXT: daddu $1, $1, $25
; N64-NEXT: daddiu $1, $1, %lo(%neg(%gp_rel(insert_v16i8_vidx)))
; N64-NEXT: ld $2, %got_disp(i32)($1)
-; N64-NEXT: lwu $2, 0($2)
+; N64-NEXT: lw $2, 0($2)
; N64-NEXT: andi $2, $2, 15
; N64-NEXT: ld $1, %got_disp(v16i8)($1)
; N64-NEXT: daddu $1, $1, $2
@@ -1953,7 +1953,7 @@ define void @insert_v8i16_vidx(i32 signext %a) nounwind {
; N64-NEXT: daddu $1, $1, $25
; N64-NEXT: daddiu $1, $1, %lo(%neg(%gp_rel(insert_v8i16_vidx)))
; N64-NEXT: ld $2, %got_disp(i32)($1)
-; N64-NEXT: lwu $2, 0($2)
+; N64-NEXT: lw $2, 0($2)
; N64-NEXT: andi $2, $2, 7
; N64-NEXT: ld $1, %got_disp(v8i16)($1)
; N64-NEXT: dlsa $1, $2, $1, 1
@@ -2002,7 +2002,7 @@ define void @insert_v4i32_vidx(i32 signext %a) nounwind {
; N64-NEXT: daddu $1, $1, $25
; N64-NEXT: daddiu $1, $1, %lo(%neg(%gp_rel(insert_v4i32_vidx)))
; N64-NEXT: ld $2, %got_disp(i32)($1)
-; N64-NEXT: lwu $2, 0($2)
+; N64-NEXT: lw $2, 0($2)
; N64-NEXT: andi $2, $2, 3
; N64-NEXT: ld $1, %got_disp(v4i32)($1)
; N64-NEXT: dlsa $1, $2, $1, 2
@@ -2053,7 +2053,7 @@ define void @insert_v2i64_vidx(i64 signext %a) nounwind {
; N64-NEXT: daddu $1, $1, $25
; N64-NEXT: daddiu $1, $1, %lo(%neg(%gp_rel(insert_v2i64_vidx)))
; N64-NEXT: ld $2, %got_disp(i32)($1)
-; N64-NEXT: lwu $2, 0($2)
+; N64-NEXT: lw $2, 0($2)
; N64-NEXT: andi $2, $2, 1
; N64-NEXT: ld $1, %got_disp(v2i64)($1)
; N64-NEXT: dlsa $1, $2, $1, 3
diff --git a/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
index 2779cf265c58a..e95c250f0434d 100644
--- a/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
@@ -88,19 +88,20 @@ define float @test_extract_i(<2 x float> %a, i64 %idx) #0 {
; CHECK-NOF32X2-NEXT: .reg .b64 %SP;
; CHECK-NOF32X2-NEXT: .reg .b64 %SPL;
; CHECK-NOF32X2-NEXT: .reg .b32 %r<4>;
-; CHECK-NOF32X2-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOF32X2-NEXT: .reg .b64 %rd<7>;
; CHECK-NOF32X2-EMPTY:
; CHECK-NOF32X2-NEXT: // %bb.0:
; CHECK-NOF32X2-NEXT: mov.b64 %SPL, __local_depot3;
; CHECK-NOF32X2-NEXT: cvta.local.u64 %SP, %SPL;
; CHECK-NOF32X2-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_extract_i_param_0];
-; CHECK-NOF32X2-NEXT: ld.param.b64 %rd1, [test_extract_i_param_1];
+; CHECK-NOF32X2-NEXT: ld.param.b64 %rd2, [test_extract_i_param_1];
+; CHECK-NOF32X2-NEXT: mov.b64 %rd1, %rd2;
; CHECK-NOF32X2-NEXT: st.v2.b32 [%SP], {%r1, %r2};
-; CHECK-NOF32X2-NEXT: and.b64 %rd2, %rd1, 1;
-; CHECK-NOF32X2-NEXT: shl.b64 %rd3, %rd2, 2;
-; CHECK-NOF32X2-NEXT: add.u64 %rd4, %SP, 0;
-; CHECK-NOF32X2-NEXT: or.b64 %rd5, %rd4, %rd3;
-; CHECK-NOF32X2-NEXT: ld.b32 %r3, [%rd5];
+; CHECK-NOF32X2-NEXT: and.b64 %rd3, %rd1, 1;
+; CHECK-NOF32X2-NEXT: shl.b64 %rd4, %rd3, 2;
+; CHECK-NOF32X2-NEXT: add.u64 %rd5, %SP, 0;
+; CHECK-NOF32X2-NEXT: or.b64 %rd6, %rd5, %rd4;
+; CHECK-NOF32X2-NEXT: ld.b32 %r3, [%rd6];
; CHECK-NOF32X2-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NOF32X2-NEXT: ret;
;
diff --git a/llvm/test/CodeGen/NVPTX/i32x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i32x2-instructions.ll
index 01bcfcbda0d53..37fc1f140cd20 100644
--- a/llvm/test/CodeGen/NVPTX/i32x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i32x2-instructions.ll
@@ -87,19 +87,20 @@ define i32 @test_extract_i(<2 x i32> %a, i64 %idx) #0 {
; CHECK-NOI32X2-NEXT: .reg .b64 %SP;
; CHECK-NOI32X2-NEXT: .reg .b64 %SPL;
; CHECK-NOI32X2-NEXT: .reg .b32 %r<4>;
-; CHECK-NOI32X2-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOI32X2-NEXT: .reg .b64 %rd<7>;
; CHECK-NOI32X2-EMPTY:
; CHECK-NOI32X2-NEXT: // %bb.0:
; CHECK-NOI32X2-NEXT: mov.b64 %SPL, __local_depot3;
; CHECK-NOI32X2-NEXT: cvta.local.u64 %SP, %SPL;
; CHECK-NOI32X2-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_extract_i_param_0];
-; CHECK-NOI32X2-NEXT: ld.param.b64 %rd1, [test_extract_i_param_1];
+; CHECK-NOI32X2-NEXT: ld.param.b64 %rd2, [test_extract_i_param_1];
+; CHECK-NOI32X2-NEXT: mov.b64 %rd1, %rd2;
; CHECK-NOI32X2-NEXT: st.v2.b32 [%SP], {%r1, %r2};
-; CHECK-NOI32X2-NEXT: and.b64 %rd2, %rd1, 1;
-; CHECK-NOI32X2-NEXT: shl.b64 %rd3, %rd2, 2;
-; CHECK-NOI32X2-NEXT: add.u64 %rd4, %SP, 0;
-; CHECK-NOI32X2-NEXT: or.b64 %rd5, %rd4, %rd3;
-; CHECK-NOI32X2-NEXT: ld.b32 %r3, [%rd5];
+; CHECK-NOI32X2-NEXT: and.b64 %rd3, %rd1, 1;
+; CHECK-NOI32X2-NEXT: shl.b64 %rd4, %rd3, 2;
+; CHECK-NOI32X2-NEXT: add.u64 %rd5, %SP, 0;
+; CHECK-NOI32X2-NEXT: or.b64 %rd6, %rd5, %rd4;
+; CHECK-NOI32X2-NEXT: ld.b32 %r3, [%rd6];
; CHECK-NOI32X2-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NOI32X2-NEXT: ret;
;
diff --git a/llvm/test/CodeGen/NVPTX/insertelt-dynamic.ll b/llvm/test/CodeGen/NVPTX/insertelt-dynamic.ll
index f2ccf3ed65c02..25bccb1113718 100644
--- a/llvm/test/CodeGen/NVPTX/insertelt-dynamic.ll
+++ b/llvm/test/CodeGen/NVPTX/insertelt-dynamic.ll
@@ -103,23 +103,24 @@ define <4 x i32> @repeated_same_index(i32 %idx) {
; CHECK-NEXT: .local .align 4 .b8 __local_depot3[16];
; CHECK-NEXT: .reg .b64 %SP;
; CHECK-NEXT: .reg .b64 %SPL;
-; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: mov.b64 %SPL, __local_depot3;
; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
-; CHECK-NEXT: ld.param.b32 %rd1, [repeated_same_index_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [repeated_same_index_param_0];
+; CHECK-NEXT: cvt.u64.u32 %rd1, %r1;
; CHECK-NEXT: and.b64 %rd2, %rd1, 3;
; CHECK-NEXT: shl.b64 %rd3, %rd2, 2;
; CHECK-NEXT: add.u64 %rd4, %SP, 0;
; CHECK-NEXT: add.s64 %rd5, %rd4, %rd3;
; CHECK-NEXT: st.b32 [%rd5], 20;
-; CHECK-NEXT: ld.b32 %r1, [%SP+12];
-; CHECK-NEXT: ld.b32 %r2, [%SP+8];
-; CHECK-NEXT: ld.b32 %r3, [%SP+4];
-; CHECK-NEXT: ld.b32 %r4, [%SP];
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ld.b32 %r2, [%SP+12];
+; CHECK-NEXT: ld.b32 %r3, [%SP+8];
+; CHECK-NEXT: ld.b32 %r4, [%SP+4];
+; CHECK-NEXT: ld.b32 %r5, [%SP];
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r4, %r3, %r2};
; CHECK-NEXT: ret;
%v0 = insertelement <4 x i32> poison, i32 10, i32 %idx
%v1 = insertelement <4 x i32> %v0, i32 20, i32 %idx
@@ -133,28 +134,30 @@ define <4 x i32> @multiple_dynamic(i32 %idx0, i32 %idx1) {
; CHECK-NEXT: .local .align 4 .b8 __local_depot4[16];
; CHECK-NEXT: .reg .b64 %SP;
; CHECK-NEXT: .reg .b64 %SPL;
-; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-NEXT: .reg .b64 %rd<10>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: mov.b64 %SPL, __local_depot4;
; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
-; CHECK-NEXT: ld.param.b32 %rd1, [multiple_dynamic_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [multiple_dynamic_param_0];
+; CHECK-NEXT: cvt.u64.u32 %rd1, %r1;
; CHECK-NEXT: and.b64 %rd2, %rd1, 3;
; CHECK-NEXT: shl.b64 %rd3, %rd2, 2;
; CHECK-NEXT: add.u64 %rd4, %SP, 0;
; CHECK-NEXT: add.s64 %rd5, %rd4, %rd3;
+; CHECK-NEXT: ld.param.b32 %r2, [multiple_dynamic_param_1];
; CHECK-NEXT: st.b32 [%rd5], 10;
-; CHECK-NEXT: ld.param.b32 %rd6, [multiple_dynamic_param_1];
+; CHECK-NEXT: cvt.u64.u32 %rd6, %r2;
; CHECK-NEXT: and.b64 %rd7, %rd6, 3;
; CHECK-NEXT: shl.b64 %rd8, %rd7, 2;
; CHECK-NEXT: add.s64 %rd9, %rd4, %rd8;
; CHECK-NEXT: st.b32 [%rd9], 20;
-; CHECK-NEXT: ld.b32 %r1, [%SP+12];
-; CHECK-NEXT: ld.b32 %r2, [%SP+8];
-; CHECK-NEXT: ld.b32 %r3, [%SP+4];
-; CHECK-NEXT: ld.b32 %r4, [%SP];
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ld.b32 %r3, [%SP+12];
+; CHECK-NEXT: ld.b32 %r4, [%SP+8];
+; CHECK-NEXT: ld.b32 %r5, [%SP+4];
+; CHECK-NEXT: ld.b32 %r6, [%SP];
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r5, %r4, %r3};
; CHECK-NEXT: ret;
%v0 = insertelement <4 x i32> poison, i32 10, i32 %idx0
%v1 = insertelement <4 x i32> %v0, i32 20, i32 %idx1
@@ -168,38 +171,42 @@ define <4 x i32> @all_dynamic(i32 %idx0, i32 %idx1, i32 %idx2, i32 %idx3) {
; CHECK-NEXT: .local .align 4 .b8 __local_depot5[16];
; CHECK-NEXT: .reg .b64 %SP;
; CHECK-NEXT: .reg .b64 %SPL;
-; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b32 %r<9>;
; CHECK-NEXT: .reg .b64 %rd<18>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: mov.b64 %SPL, __local_depot5;
; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
-; CHECK-NEXT: ld.param.b32 %rd1, [all_dynamic_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [all_dynamic_param_0];
+; CHECK-NEXT: cvt.u64.u32 %rd1, %r1;
; CHECK-NEXT: and.b64 %rd2, %rd1, 3;
; CHECK-NEXT: shl.b64 %rd3, %rd2, 2;
; CHECK-NEXT: add.u64 %rd4, %SP, 0;
; CHECK-NEXT: add.s64 %rd5, %rd4, %rd3;
-; CHECK-NEXT: ld.param.b32 %rd6, [all_dynamic_param_1];
+; CHECK-NEXT: ld.param.b32 %r2, [all_dynamic_param_1];
+; CHECK-NEXT: cvt.u64.u32 %rd6, %r2;
; CHECK-NEXT: and.b64 %rd7, %rd6, 3;
; CHECK-NEXT: shl.b64 %rd8, %rd7, 2;
; CHECK-NEXT: add.s64 %rd9, %rd4, %rd8;
-; CHECK-NEXT: ld.param.b32 %rd10, [all_dynamic_param_2];
+; CHECK-NEXT: ld.param.b32 %r3, [all_dynamic_param_2];
+; CHECK-NEXT: cvt.u64.u32 %rd10, %r3;
; CHECK-NEXT: and.b64 %rd11, %rd10, 3;
; CHECK-NEXT: shl.b64 %rd12, %rd11, 2;
; CHECK-NEXT: add.s64 %rd13, %rd4, %rd12;
+; CHECK-NEXT: ld.param.b32 %r4, [all_dynamic_param_3];
; CHECK-NEXT: st.b32 [%rd5], 10;
; CHECK-NEXT: st.b32 [%rd9], 20;
; CHECK-NEXT: st.b32 [%rd13], 30;
-; CHECK-NEXT: ld.param.b32 %rd14, [all_dynamic_param_3];
+; CHECK-NEXT: cvt.u64.u32 %rd14, %r4;
; CHECK-NEXT: and.b64 %rd15, %rd14, 3;
; CHECK-NEXT: shl.b64 %rd16, %rd15, 2;
; CHECK-NEXT: add.s64 %rd17, %rd4, %rd16;
; CHECK-NEXT: st.b32 [%rd17], 40;
-; CHECK-NEXT: ld.b32 %r1, [%SP+12];
-; CHECK-NEXT: ld.b32 %r2, [%SP+8];
-; CHECK-NEXT: ld.b32 %r3, [%SP+4];
-; CHECK-NEXT: ld.b32 %r4, [%SP];
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ld.b32 %r5, [%SP+12];
+; CHECK-NEXT: ld.b32 %r6, [%SP+8];
+; CHECK-NEXT: ld.b32 %r7, [%SP+4];
+; CHECK-NEXT: ld.b32 %r8, [%SP];
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
; CHECK-NEXT: ret;
%v0 = insertelement <4 x i32> poison, i32 10, i32 %idx0
%v1 = insertelement <4 x i32> %v0, i32 20, i32 %idx1
@@ -216,29 +223,31 @@ define <4 x i32> @mix_dynamic_constant(i32 %idx0, i32 %idx1) {
; CHECK-NEXT: .local .align 4 .b8 __local_depot6[16];
; CHECK-NEXT: .reg .b64 %SP;
; CHECK-NEXT: .reg .b64 %SPL;
-; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-NEXT: .reg .b64 %rd<10>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: mov.b64 %SPL, __local_depot6;
; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
-; CHECK-NEXT: ld.param.b32 %rd1, [mix_dynamic_constant_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [mix_dynamic_constant_param_0];
+; CHECK-NEXT: cvt.u64.u32 %rd1, %r1;
; CHECK-NEXT: and.b64 %rd2, %rd1, 3;
; CHECK-NEXT: shl.b64 %rd3, %rd2, 2;
; CHECK-NEXT: add.u64 %rd4, %SP, 0;
; CHECK-NEXT: add.s64 %rd5, %rd4, %rd3;
+; CHECK-NEXT: ld.param.b32 %r2, [mix_dynamic_constant_param_1];
; CHECK-NEXT: st.b32 [%rd5], 10;
-; CHECK-NEXT: ld.param.b32 %rd6, [mix_dynamic_constant_param_1];
+; CHECK-NEXT: cvt.u64.u32 %rd6, %r2;
; CHECK-NEXT: and.b64 %rd7, %rd6, 3;
; CHECK-NEXT: shl.b64 %rd8, %rd7, 2;
; CHECK-NEXT: add.s64 %rd9, %rd4, %rd8;
; CHECK-NEXT: st.b32 [%SP+4], 20;
; CHECK-NEXT: st.b32 [%rd9], 30;
-; CHECK-NEXT: ld.b32 %r1, [%SP+12];
-; CHECK-NEXT: ld.b32 %r2, [%SP+8];
-; CHECK-NEXT: ld.b32 %r3, [%SP+4];
-; CHECK-NEXT: ld.b32 %r4, [%SP];
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ld.b32 %r3, [%SP+12];
+; CHECK-NEXT: ld.b32 %r4, [%SP+8];
+; CHECK-NEXT: ld.b32 %r5, [%SP+4];
+; CHECK-NEXT: ld.b32 %r6, [%SP];
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r5, %r4, %r3};
; CHECK-NEXT: ret;
%v0 = insertelement <4 x i32> poison, i32 10, i32 %idx0
%v1 = insertelement <4 x i32> %v0, i32 20, i32 1
@@ -302,7 +311,7 @@ define void @overlapping_chains(i32 %idx0, i32 %idx1, ptr %out0, ptr %out1) {
; CHECK-NEXT: .local .align 4 .b8 __local_depot8[32];
; CHECK-NEXT: .reg .b64 %SP;
; CHECK-NEXT: .reg .b64 %SPL;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-NEXT: .reg .b64 %rd<14>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
@@ -316,23 +325,24 @@ define void @overlapping_chains(i32 %idx0, i32 %idx1, ptr %out0, ptr %out1) {
; CHECK-NEXT: st.b32 [%rd5], 10;
; CHECK-NEXT: add.u64 %rd6, %SP, 0;
; CHECK-NEXT: add.s64 %rd7, %rd6, %rd3;
-; CHECK-NEXT: ld.b32 %r1, [%SP+28];
-; CHECK-NEXT: ld.b32 %r2, [%SP+16];
+; CHECK-NEXT: ld.param.b32 %r1, [overlapping_chains_param_1];
+; CHECK-NEXT: ld.b32 %r2, [%SP+28];
+; CHECK-NEXT: ld.b32 %r3, [%SP+16];
; CHECK-NEXT: ld.param.b64 %rd8, [overlapping_chains_param_2];
; CHECK-NEXT: st.b32 [%rd7], 10;
-; CHECK-NEXT: ld.param.b32 %rd9, [overlapping_chains_param_1];
+; CHECK-NEXT: cvt.u64.u32 %rd9, %r1;
; CHECK-NEXT: and.b64 %rd10, %rd9, 3;
; CHECK-NEXT: shl.b64 %rd11, %rd10, 2;
; CHECK-NEXT: add.s64 %rd12, %rd6, %rd11;
; CHECK-NEXT: st.b32 [%SP+4], 20;
; CHECK-NEXT: st.b32 [%rd12], 30;
; CHECK-NEXT: ld.param.b64 %rd13, [overlapping_chains_param_3];
-; CHECK-NEXT: ld.b32 %r3, [%SP+12];
-; CHECK-NEXT: ld.b32 %r4, [%SP+8];
-; CHECK-NEXT: ld.b32 %r5, [%SP+4];
-; CHECK-NEXT: ld.b32 %r6, [%SP];
-; CHECK-NEXT: st.v4.b32 [%rd8], {%r2, 20, 40, %r1};
-; CHECK-NEXT: st.v4.b32 [%rd13], {%r6, %r5, %r4, %r3};
+; CHECK-NEXT: ld.b32 %r4, [%SP+12];
+; CHECK-NEXT: ld.b32 %r5, [%SP+8];
+; CHECK-NEXT: ld.b32 %r6, [%SP+4];
+; CHECK-NEXT: ld.b32 %r7, [%SP];
+; CHECK-NEXT: st.v4.b32 [%rd8], {%r3, 20, 40, %r2};
+; CHECK-NEXT: st.v4.b32 [%rd13], {%r7, %r6, %r5, %r4};
; CHECK-NEXT: ret;
%v0 = insertelement <4 x i32> poison, i32 10, i32 %idx0
%v1 = insertelement <4 x i32> %v0, i32 20, i32 1
diff --git a/llvm/test/CodeGen/WebAssembly/vector-extract-last-active.ll b/llvm/test/CodeGen/WebAssembly/vector-extract-last-active.ll
index 927424dc649d5..e3cd76db74d47 100644
--- a/llvm/test/CodeGen/WebAssembly/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/WebAssembly/vector-extract-last-active.ll
@@ -38,6 +38,8 @@ define i32 @extract_last_active_v4i32(<4 x i32> %a, <4 x i1> %c) {
; CHECK-NEXT: i32.const 1
; CHECK-NEXT: i32.and
; CHECK-NEXT: i32.select
+; CHECK-NEXT: i32.const 3
+; CHECK-NEXT: i32.and
; CHECK-NEXT: i32.const 2
; CHECK-NEXT: i32.shl
; CHECK-NEXT: i32.or
@@ -93,6 +95,8 @@ define i32 @extract_last_active_v4i32_no_default(<4 x i32> %a, <4 x i1> %c) {
; CHECK-NEXT: i32.const 1
; CHECK-NEXT: i32.and
; CHECK-NEXT: i32.select
+; CHECK-NEXT: i32.const 3
+; CHECK-NEXT: i32.and
; CHECK-NEXT: i32.const 2
; CHECK-NEXT: i32.shl
; CHECK-NEXT: i32.or
diff --git a/llvm/test/CodeGen/X86/extractelement-legalization-store-ordering.ll b/llvm/test/CodeGen/X86/extractelement-legalization-store-ordering.ll
index 0e0cfc64af9ee..aa69675e8bbb6 100644
--- a/llvm/test/CodeGen/X86/extractelement-legalization-store-ordering.ll
+++ b/llvm/test/CodeGen/X86/extractelement-legalization-store-ordering.ll
@@ -20,13 +20,13 @@ define void @test_extractelement_legalization_storereuse(<4 x i32> %a, ptr nocap
; CHECK-NEXT: movdqa %xmm0, (%edx)
; CHECK-NEXT: movl (%edx), %esi
; CHECK-NEXT: movl 4(%edx), %edi
-; CHECK-NEXT: shll $4, %ecx
+; CHECK-NEXT: shll $4, %eax
; CHECK-NEXT: movl 8(%edx), %ebx
; CHECK-NEXT: movl 12(%edx), %edx
-; CHECK-NEXT: movl %esi, 12(%eax,%ecx)
-; CHECK-NEXT: movl %edi, (%eax,%ecx)
-; CHECK-NEXT: movl %ebx, 8(%eax,%ecx)
-; CHECK-NEXT: movl %edx, 4(%eax,%ecx)
+; CHECK-NEXT: movl %esi, 12(%ecx,%eax)
+; CHECK-NEXT: movl %edi, (%ecx,%eax)
+; CHECK-NEXT: movl %ebx, 8(%ecx,%eax)
+; CHECK-NEXT: movl %edx, 4(%ecx,%eax)
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
; CHECK-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index 2b3abf2de5ba9..edd6d8e87be9a 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -818,74 +818,97 @@ define <16 x i8> @var_shuffle_v16i8(<16 x i8> %v, <16 x i8> %indices) nounwind {
define <16 x i8> @var_shuffle_zero_v16i8(<16 x i8> %v, <16 x i8> %indices) nounwind {
; SSE3-LABEL: var_shuffle_zero_v16i8:
; SSE3: # %bb.0:
+; SSE3-NEXT: pushq %rbp
+; SSE3-NEXT: pushq %r15
+; SSE3-NEXT: pushq %r14
+; SSE3-NEXT: pushq %r13
+; SSE3-NEXT: pushq %r12
+; SSE3-NEXT: pushq %rbx
; SSE3-NEXT: movaps %xmm0, %xmm2
; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
; SSE3-NEXT: pmaxub %xmm1, %xmm0
; SSE3-NEXT: pcmpeqb %xmm1, %xmm0
; SSE3-NEXT: por %xmm0, %xmm1
; SSE3-NEXT: movdqa %xmm1, -40(%rsp)
-; SSE3-NEXT: movaps %xmm2, -24(%rsp)
+; SSE3-NEXT: movzbl -40(%rsp), %eax
+; SSE3-NEXT: movb %al, -41(%rsp) # 1-byte Spill
+; SSE3-NEXT: movzbl -39(%rsp), %ecx
+; SSE3-NEXT: movzbl -38(%rsp), %edx
+; SSE3-NEXT: movzbl -37(%rsp), %esi
+; SSE3-NEXT: movzbl -36(%rsp), %edi
+; SSE3-NEXT: movzbl -35(%rsp), %r8d
+; SSE3-NEXT: movzbl -34(%rsp), %r9d
+; SSE3-NEXT: movzbl -33(%rsp), %r10d
+; SSE3-NEXT: movzbl -32(%rsp), %r11d
+; SSE3-NEXT: movzbl -31(%rsp), %ebx
+; SSE3-NEXT: movzbl -30(%rsp), %ebp
+; SSE3-NEXT: movzbl -29(%rsp), %r14d
+; SSE3-NEXT: movzbl -28(%rsp), %r15d
+; SSE3-NEXT: movzbl -27(%rsp), %r12d
+; SSE3-NEXT: movzbl -26(%rsp), %r13d
; SSE3-NEXT: movzbl -25(%rsp), %eax
+; SSE3-NEXT: movaps %xmm2, -24(%rsp)
+; SSE3-NEXT: movzbl %al, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm1
-; SSE3-NEXT: movzbl -26(%rsp), %eax
+; SSE3-NEXT: movzbl %r13b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm2
-; SSE3-NEXT: movzbl -27(%rsp), %eax
+; SSE3-NEXT: movzbl %r12b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm4
-; SSE3-NEXT: movzbl -28(%rsp), %eax
+; SSE3-NEXT: movzbl %r15b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm3
-; SSE3-NEXT: movzbl -29(%rsp), %eax
+; SSE3-NEXT: movzbl %r14b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm6
-; SSE3-NEXT: movzbl -30(%rsp), %eax
+; SSE3-NEXT: movzbl %bpl, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm7
-; SSE3-NEXT: movzbl -31(%rsp), %eax
+; SSE3-NEXT: movzbl %bl, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm8
-; SSE3-NEXT: movzbl -32(%rsp), %eax
+; SSE3-NEXT: movzbl %r11b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm5
-; SSE3-NEXT: movzbl -33(%rsp), %eax
+; SSE3-NEXT: movzbl %r10b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm9
-; SSE3-NEXT: movzbl -34(%rsp), %eax
+; SSE3-NEXT: movzbl %r9b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm10
-; SSE3-NEXT: movzbl -35(%rsp), %eax
+; SSE3-NEXT: movzbl %r8b, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm12
-; SSE3-NEXT: movzbl -36(%rsp), %eax
+; SSE3-NEXT: movzbl %dil, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm11
-; SSE3-NEXT: movzbl -37(%rsp), %eax
+; SSE3-NEXT: movzbl %sil, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm13
-; SSE3-NEXT: movzbl -38(%rsp), %eax
+; SSE3-NEXT: movzbl %dl, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm14
-; SSE3-NEXT: movzbl -39(%rsp), %eax
+; SSE3-NEXT: movzbl %cl, %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
; SSE3-NEXT: movd %eax, %xmm15
-; SSE3-NEXT: movzbl -40(%rsp), %eax
+; SSE3-NEXT: movzbl -41(%rsp), %eax # 1-byte Folded Reload
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
@@ -905,6 +928,12 @@ define <16 x i8> @var_shuffle_zero_v16i8(<16 x i8> %v, <16 x i8> %indices) nounw
; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
; SSE3-NEXT: pandn %xmm1, %xmm0
+; SSE3-NEXT: popq %rbx
+; SSE3-NEXT: popq %r12
+; SSE3-NEXT: popq %r13
+; SSE3-NEXT: popq %r14
+; SSE3-NEXT: popq %r15
+; SSE3-NEXT: popq %rbp
; SSE3-NEXT: retq
;
; SSSE3-LABEL: var_shuffle_zero_v16i8:
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index d5872b1a2a351..e82cd72665bed 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -37,6 +37,7 @@ define <4 x i32> @test_compress_v4i32(<4 x i32> %vec, <4 x i1> %mask, <4 x i32>
; AVX2-NEXT: cmpq $3, %rcx
; AVX2-NEXT: movl $3, %eax
; AVX2-NEXT: cmovbq %rcx, %rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: movl %r8d, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -94,13 +95,15 @@ define <4 x float> @test_compress_v4f32(<4 x float> %vec, <4 x i1> %mask, <4 x f
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $3, %rax
-; AVX2-NEXT: movl $3, %ecx
-; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: ja .LBB1_2
+; AVX2-NEXT: cmpq $4, %rax
+; AVX2-NEXT: jae .LBB1_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm1, %xmm0
; AVX2-NEXT: .LBB1_2:
+; AVX2-NEXT: cmpq $3, %rax
+; AVX2-NEXT: movl $3, %ecx
+; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -151,7 +154,7 @@ define <2 x i64> @test_compress_v2i64(<2 x i64> %vec, <2 x i1> %mask, <2 x i64>
; AVX2-NEXT: cmpq $1, %rdx
; AVX2-NEXT: movl $1, %eax
; AVX2-NEXT: cmovbq %rdx, %rax
-; AVX2-NEXT: movl %eax, %eax
+; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: movq %rsi, -24(%rsp,%rax,8)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -205,8 +208,8 @@ define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x
; AVX2-NEXT: cmpq $1, %rax
; AVX2-NEXT: movl $1, %ecx
; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: vmovsd %xmm1, -24(%rsp,%rax,8)
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vmovsd %xmm1, -24(%rsp,%rcx,8)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
@@ -300,7 +303,7 @@ define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32>
; AVX2-NEXT: cmpq $7, %r11
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r11, %rax
-; AVX2-NEXT: movl %eax, %eax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: movl %ebx, (%rsp,%rax,4)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: leaq -8(%rbp), %rsp
@@ -404,8 +407,8 @@ define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x f
; AVX2-NEXT: cmpq $7, %rax
; AVX2-NEXT: movl $7, %ecx
; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
; AVX2-NEXT: popq %rbp
@@ -449,7 +452,7 @@ define <4 x i64> @test_compress_v4i64(<4 x i64> %vec, <4 x i1> %mask, <4 x i64>
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rax
; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: vpextrq $1, %xmm1, %rax
; AVX2-NEXT: vmovq %xmm1, %rdx
@@ -470,14 +473,14 @@ define <4 x i64> @test_compress_v4i64(<4 x i64> %vec, <4 x i1> %mask, <4 x i64>
; AVX2-NEXT: vmovq %xmm0, (%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rcx,8)
-; AVX2-NEXT: movl %esi, %ecx
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%rcx,8)
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%rsi,8)
; AVX2-NEXT: andl $3, %edi
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%rdi,8)
; AVX2-NEXT: cmpq $3, %r8
; AVX2-NEXT: movl $3, %ecx
; AVX2-NEXT: cmovbq %r8, %rcx
-; AVX2-NEXT: movl %ecx, %ecx
+; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: movq %rax, (%rsp,%rcx,8)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
@@ -523,7 +526,7 @@ define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x
; AVX2-NEXT: vpaddq %xmm3, %xmm1, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rax
; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
; AVX2-NEXT: vmovlpd %xmm0, (%rsp)
@@ -534,6 +537,7 @@ define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
; AVX2-NEXT: subq %rcx, %rax
; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovlpd %xmm0, (%rsp,%rcx,8)
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
@@ -552,8 +556,8 @@ define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x
; AVX2-NEXT: cmpq $3, %rax
; AVX2-NEXT: movl $3, %ecx
; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: vmovsd %xmm1, (%rsp,%rax,8)
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovsd %xmm1, (%rsp,%rcx,8)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
; AVX2-NEXT: popq %rbp
@@ -608,9 +612,11 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrd $1, %xmm3, %eax
; AVX2-NEXT: vmovd %xmm3, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrb $1, %xmm2, %eax
+; AVX2-NEXT: vpextrb $4, %xmm2, %edx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -621,19 +627,17 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm2, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $4, %xmm2, %r8d
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rax, %r8
-; AVX2-NEXT: vpextrb $5, %xmm2, %r9d
+; AVX2-NEXT: vpextrb $3, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addq %rcx, %r9
+; AVX2-NEXT: addq %r9, %rdx
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $5, %xmm2, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rdx, %rdi
; AVX2-NEXT: vpextrb $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addq %rdi, %r10
; AVX2-NEXT: vpextrb $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -658,29 +662,30 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrb $15, %xmm2, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rcx, %rdi
+; AVX2-NEXT: vpextrb $15, %xmm2, %esi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rcx, %rsi
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX2-NEXT: vextractps $3, %xmm2, %eax
-; AVX2-NEXT: cmpq $16, %rdi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: cmovbl 32(%rsp,%rsi,4), %eax
+; AVX2-NEXT: cmpq $16, %rsi
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX2-NEXT: cmovbl 32(%rsp,%r8,4), %eax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; AVX2-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%rsi,4)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%rsi,4)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%rsi,4)
-; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: vmovss %xmm0, 32(%rsp,%rax,4)
-; AVX2-NEXT: andl $15, %r8d
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%r8,4)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%r8,4)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%r8,4)
; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%r9,4)
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovss %xmm0, 32(%rsp,%r9,4)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%rax,4)
+; AVX2-NEXT: andl $15, %edi
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%rdi,4)
; AVX2-NEXT: andl $15, %r10d
; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%r10,4)
; AVX2-NEXT: andl $15, %r11d
@@ -699,12 +704,12 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vextractps $2, %xmm2, 32(%rsp,%rdx,4)
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vextractps $3, %xmm2, 32(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $15, %rdi
+; AVX2-NEXT: cmpq $15, %rsi
; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rdi, %rax
-; AVX2-NEXT: movl %eax, %eax
+; AVX2-NEXT: cmovbq %rsi, %rax
+; AVX2-NEXT: shll $2, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
-; AVX2-NEXT: movl %ecx, 32(%rsp,%rax,4)
+; AVX2-NEXT: movl %ecx, 32(%rsp,%rax)
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -40(%rbp), %rsp
@@ -774,11 +779,13 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: vpextrb $3, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrb $4, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrb $5, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
@@ -821,42 +828,42 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rax,4)
-; AVX2-NEXT: vpextrb $12, %xmm2, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: vpextrb $12, %xmm2, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: vpextrb $13, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: vpextrb $14, %xmm2, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: vpextrb $15, %xmm2, %eax
-; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: vpextrb $13, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rdx, %rax
; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: vpextrb $14, %xmm2, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: vpextrb $15, %xmm2, %ecx
+; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)
-; AVX2-NEXT: cmpq $16, %rax
+; AVX2-NEXT: cmpq $16, %rcx
; AVX2-NEXT: jae .LBB9_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm3, %xmm0
; AVX2-NEXT: .LBB9_2:
-; AVX2-NEXT: cmpq $15, %rax
-; AVX2-NEXT: movl $15, %ecx
-; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: cmpq $15, %rcx
+; AVX2-NEXT: movl $15, %eax
+; AVX2-NEXT: cmovbq %rcx, %rax
+; AVX2-NEXT: shll $2, %eax
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: movq %rbp, %rsp
@@ -900,55 +907,55 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT: vmovd %xmm3, %eax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: vmovd %xmm2, %edx
+; AVX2-NEXT: vpextrq $1, %xmm3, %rax
+; AVX2-NEXT: vmovd %xmm3, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vpextrw $4, %xmm2, %eax
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vpextrw $1, %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrw $2, %xmm2, %esi
+; AVX2-NEXT: vmovd %xmm2, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
-; AVX2-NEXT: vpextrw $3, %xmm2, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
-; AVX2-NEXT: vpextrw $4, %xmm2, %r8d
+; AVX2-NEXT: addq %rsi, %rdx
+; AVX2-NEXT: vpextrw $2, %xmm2, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
-; AVX2-NEXT: vpextrw $5, %xmm2, %r9d
+; AVX2-NEXT: addq %rdx, %r8
+; AVX2-NEXT: vpextrw $3, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: vpextrw $5, %xmm2, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rax, %rdi
; AVX2-NEXT: vpextrw $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addq %rdi, %r10
; AVX2-NEXT: vpextrw $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vmovq %xmm0, (%rsp)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, (%rsp,%rcx,8)
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
-; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r8,8)
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm2, (%rsp,%r9,8)
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%rax,8)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vmovq %xmm2, (%rsp,%rdi,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r11, %rax
-; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)
+; AVX2-NEXT: shll $3, %eax
+; AVX2-NEXT: movq %rbx, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -8(%rbp), %rsp
@@ -994,7 +1001,7 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vpextrq $1, %xmm3, %rax
; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: andl $7, %ecx
; AVX2-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-NEXT: vmovlps %xmm0, (%rsp)
@@ -1046,8 +1053,8 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: cmpq $7, %rax
; AVX2-NEXT: movl $7, %ecx
; AVX2-NEXT: cmovbq %rax, %rcx
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: vmovsd %xmm3, (%rsp,%rax,8)
+; AVX2-NEXT: shll $3, %ecx
+; AVX2-NEXT: vmovsd %xmm3, (%rsp,%rcx)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: movq %rbp, %rsp
@@ -1251,63 +1258,82 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16> %passthru) nounwind {
; AVX2-LABEL: test_compress_v8i16:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vpsllw $15, %xmm1, %xmm1
; AVX2-NEXT: vpsraw $15, %xmm1, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrw $1, %xmm1, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: leal (%rcx,%rax), %esi
-; AVX2-NEXT: vpextrw $2, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: leal (%rdi,%rdx), %r10d
-; AVX2-NEXT: addl %esi, %r10d
-; AVX2-NEXT: vpextrw $4, %xmm1, %r9d
+; AVX2-NEXT: vpextrw $1, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
+; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: leal (%r9,%rsi), %r11d
-; AVX2-NEXT: vpextrw $6, %xmm1, %r8d
+; AVX2-NEXT: leaq (%rsi,%r9), %rcx
+; AVX2-NEXT: vpextrw $2, %xmm1, %r10d
+; AVX2-NEXT: movl %r10d, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
+; AVX2-NEXT: movl %r8d, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: vpextrw $4, %xmm1, %r11d
+; AVX2-NEXT: movl %r11d, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rdx, %rdi
+; AVX2-NEXT: movl %esi, %ebx
+; AVX2-NEXT: addl %r9d, %ebx
+; AVX2-NEXT: vpextrw $5, %xmm1, %ebp
+; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addl %r8d, %r11d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: vpextrw $7, %xmm1, %r10d
+; AVX2-NEXT: addl %r10d, %r8d
+; AVX2-NEXT: movl %ebp, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: addq %rdi, %r9
+; AVX2-NEXT: addl %ebx, %r8d
+; AVX2-NEXT: vpextrw $6, %xmm1, %ebx
+; AVX2-NEXT: andl $1, %r11d
+; AVX2-NEXT: andl $1, %ebp
+; AVX2-NEXT: addl %r11d, %ebp
+; AVX2-NEXT: movl %ebx, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: andl $7, %r11d
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: addq %rax, %rdi
-; AVX2-NEXT: addq %rdi, %rdx
-; AVX2-NEXT: addq %rdx, %r9
-; AVX2-NEXT: addq %r9, %rsi
-; AVX2-NEXT: addq %rsi, %r8
-; AVX2-NEXT: addq %r8, %r10
-; AVX2-NEXT: vpextrw $7, %xmm0, %ebx
-; AVX2-NEXT: cmpq $8, %r10
-; AVX2-NEXT: cmovbw -16(%rsp,%r11,2), %bx
+; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: andl $1, %ebx
+; AVX2-NEXT: addl %ebp, %ebx
+; AVX2-NEXT: vpextrw $7, %xmm1, %r14d
+; AVX2-NEXT: addl %r8d, %ebx
+; AVX2-NEXT: movl %r14d, %r8d
+; AVX2-NEXT: andl $1, %r8d
+; AVX2-NEXT: addq %r10, %r8
+; AVX2-NEXT: andl $1, %r14d
+; AVX2-NEXT: addl %ebx, %r14d
+; AVX2-NEXT: andl $7, %r14d
+; AVX2-NEXT: vpextrw $7, %xmm0, %r11d
+; AVX2-NEXT: cmpq $8, %r8
+; AVX2-NEXT: cmovbw -16(%rsp,%r14,2), %r11w
; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rcx,2)
-; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdi,2)
+; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rcx,2)
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rax,2)
; AVX2-NEXT: andl $7, %edx
; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%rdi,2)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r9,2)
-; AVX2-NEXT: andl $7, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rsi,2)
-; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r8,2)
-; AVX2-NEXT: cmpq $7, %r10
+; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%r9,2)
+; AVX2-NEXT: andl $7, %r10d
+; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r10,2)
+; AVX2-NEXT: cmpq $7, %r8
; AVX2-NEXT: movl $7, %eax
-; AVX2-NEXT: cmovbq %r10, %rax
-; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movw %bx, -16(%rsp,%rax,2)
+; AVX2-NEXT: cmovbq %r8, %rax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: movw %r11w, -16(%rsp,%rax,2)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_compress_v8i16:
@@ -1631,18 +1657,17 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andq $-32, %rsp
; AVX512VL-ONLY-NEXT: subq $64, %rsp
; AVX512VL-ONLY-NEXT: vpsllw $7, %ymm1, %ymm1
-; AVX512VL-ONLY-NEXT: vpmovb2m %ymm1, %k6
-; AVX512VL-ONLY-NEXT: kshiftrd $31, %k6, %k0
-; AVX512VL-ONLY-NEXT: kshiftrd $30, %k6, %k1
-; AVX512VL-ONLY-NEXT: kshiftrd $29, %k6, %k2
-; AVX512VL-ONLY-NEXT: kshiftrd $28, %k6, %k3
-; AVX512VL-ONLY-NEXT: kshiftrd $27, %k6, %k4
-; AVX512VL-ONLY-NEXT: kshiftrd $26, %k6, %k5
-; AVX512VL-ONLY-NEXT: kshiftrd $1, %k6, %k7
+; AVX512VL-ONLY-NEXT: vpmovb2m %ymm1, %k5
+; AVX512VL-ONLY-NEXT: kshiftrd $31, %k5, %k0
+; AVX512VL-ONLY-NEXT: kshiftrd $30, %k5, %k1
+; AVX512VL-ONLY-NEXT: kshiftrd $29, %k5, %k2
+; AVX512VL-ONLY-NEXT: kshiftrd $28, %k5, %k3
+; AVX512VL-ONLY-NEXT: kshiftrd $27, %k5, %k4
+; AVX512VL-ONLY-NEXT: kshiftrd $2, %k5, %k6
+; AVX512VL-ONLY-NEXT: kshiftrd $1, %k5, %k7
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $2, %k6, %k7
; AVX512VL-ONLY-NEXT: vmovaps %ymm2, (%rsp)
-; AVX512VL-ONLY-NEXT: vmovdqu8 {{.*#+}} ymm1 {%k6} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-ONLY-NEXT: vmovdqu8 {{.*#+}} ymm1 {%k5} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512VL-ONLY-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512VL-ONLY-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; AVX512VL-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
@@ -1658,39 +1683,44 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %eax
; AVX512VL-ONLY-NEXT: movzbl (%rsp,%rax), %eax
; AVX512VL-ONLY-NEXT: vpextrb $0, %xmm0, (%rsp)
-; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: vpextrb $1, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
-; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $3, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: movl %ecx, %edx
+; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)
+; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
+; AVX512VL-ONLY-NEXT: movl %edx, %ecx
+; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: kshiftrd $26, %k5, %k6
+; AVX512VL-ONLY-NEXT: kshiftrd $3, %k5, %k7
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $4, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
+; AVX512VL-ONLY-NEXT: kshiftrd $4, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
+; AVX512VL-ONLY-NEXT: movl %ecx, %edx
+; AVX512VL-ONLY-NEXT: vpextrb $4, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $5, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: kshiftrd $5, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
+; AVX512VL-ONLY-NEXT: movl %edx, %ecx
+; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rdx)
-; AVX512VL-ONLY-NEXT: kshiftrd $6, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $6, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $7, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $7, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1701,12 +1731,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $8, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $8, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $9, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $9, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1714,7 +1744,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $10, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $10, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1725,12 +1755,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $10, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $11, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $11, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $12, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $12, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1738,7 +1768,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $11, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $13, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $13, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1749,12 +1779,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $14, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $14, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $15, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $15, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1762,7 +1792,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $16, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $16, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1770,7 +1800,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $15, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $17, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $17, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1779,7 +1809,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512VL-ONLY-NEXT: vpextrb $0, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $18, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $18, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1790,12 +1820,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $19, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $19, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $20, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $20, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1803,7 +1833,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $21, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $21, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1814,12 +1844,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $22, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $22, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrd $23, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $23, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1827,8 +1857,8 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: andl $31, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrd $25, %k6, %k7
-; AVX512VL-ONLY-NEXT: kshiftrd $24, %k6, %k6
+; AVX512VL-ONLY-NEXT: kshiftrd $25, %k5, %k7
+; AVX512VL-ONLY-NEXT: kshiftrd $24, %k5, %k5
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -1838,7 +1868,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: movl %ecx, %edx
; AVX512VL-ONLY-NEXT: andl $31, %edx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm0, (%rsp,%rdx)
-; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1849,7 +1879,7 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
-; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -1940,9 +1970,11 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $1, %xmm2, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: vpextrw $6, %xmm1, %edx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -1953,26 +1985,24 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrw $3, %xmm1, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rax, %r8
-; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
+; AVX2-NEXT: addq %rcx, %r8
+; AVX2-NEXT: vpextrw $4, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
-; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
+; AVX2-NEXT: addq %r8, %r10
+; AVX2-NEXT: vpextrw $5, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
+; AVX2-NEXT: addq %r11, %rdx
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrw $7, %xmm1, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: addq %rdx, %r9
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX2-NEXT: vmovd %xmm1, %ebx
; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addq %r11, %rbx
+; AVX2-NEXT: addq %r9, %rbx
; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
; AVX2-NEXT: andl $1, %r14d
; AVX2-NEXT: addq %rbx, %r14
@@ -1991,32 +2021,32 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $6, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrw $7, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rcx, %rdi
+; AVX2-NEXT: vpextrw $7, %xmm1, %esi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rcx, %rsi
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrw $7, %xmm1, %eax
-; AVX2-NEXT: cmpq $16, %rdi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: cmovbw (%rsp,%rsi,2), %ax
+; AVX2-NEXT: cmpq $16, %rsi
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: cmovbw (%rsp,%rdi,2), %ax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; AVX2-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vpextrw $1, %xmm0, (%rsp,%rsi,2)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%rsi,2)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%rsi,2)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%rax,2)
-; AVX2-NEXT: andl $15, %r8d
-; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r8,2)
-; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r9,2)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vpextrw $1, %xmm0, (%rsp,%rdi,2)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%rdi,2)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%rdi,2)
+; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%r8,2)
; AVX2-NEXT: andl $15, %r10d
-; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%r10,2)
+; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r10,2)
; AVX2-NEXT: andl $15, %r11d
-; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r11,2)
+; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r11,2)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%rax,2)
+; AVX2-NEXT: andl $15, %r9d
+; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r9,2)
; AVX2-NEXT: andl $15, %ebx
; AVX2-NEXT: vpextrw $1, %xmm1, (%rsp,%rbx,2)
; AVX2-NEXT: andl $15, %r14d
@@ -2031,10 +2061,10 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%rdx,2)
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%rcx,2)
-; AVX2-NEXT: cmpq $15, %rdi
+; AVX2-NEXT: cmpq $15, %rsi
; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rdi, %rax
-; AVX2-NEXT: movl %eax, %eax
+; AVX2-NEXT: cmovbq %rsi, %rax
+; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; AVX2-NEXT: movw %cx, (%rsp,%rax,2)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
@@ -2825,18 +2855,17 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andq $-64, %rsp
; AVX512VL-ONLY-NEXT: addq $-128, %rsp
; AVX512VL-ONLY-NEXT: vpsllw $7, %zmm1, %zmm1
-; AVX512VL-ONLY-NEXT: vpmovb2m %zmm1, %k6
-; AVX512VL-ONLY-NEXT: kshiftrq $63, %k6, %k0
-; AVX512VL-ONLY-NEXT: kshiftrq $62, %k6, %k1
-; AVX512VL-ONLY-NEXT: kshiftrq $61, %k6, %k2
-; AVX512VL-ONLY-NEXT: kshiftrq $60, %k6, %k3
-; AVX512VL-ONLY-NEXT: kshiftrq $59, %k6, %k4
-; AVX512VL-ONLY-NEXT: kshiftrq $58, %k6, %k5
-; AVX512VL-ONLY-NEXT: kshiftrq $1, %k6, %k7
-; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
+; AVX512VL-ONLY-NEXT: vpmovb2m %zmm1, %k5
+; AVX512VL-ONLY-NEXT: kshiftrq $63, %k5, %k0
+; AVX512VL-ONLY-NEXT: kshiftrq $62, %k5, %k1
+; AVX512VL-ONLY-NEXT: kshiftrq $61, %k5, %k2
+; AVX512VL-ONLY-NEXT: kshiftrq $60, %k5, %k3
+; AVX512VL-ONLY-NEXT: kshiftrq $59, %k5, %k4
+; AVX512VL-ONLY-NEXT: kshiftrq $2, %k5, %k6
; AVX512VL-ONLY-NEXT: vmovaps %zmm2, (%rsp)
-; AVX512VL-ONLY-NEXT: kshiftrq $2, %k6, %k7
-; AVX512VL-ONLY-NEXT: vmovdqu8 {{.*#+}} zmm1 {%k6} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-ONLY-NEXT: kshiftrq $1, %k5, %k7
+; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
+; AVX512VL-ONLY-NEXT: vmovdqu8 {{.*#+}} zmm1 {%k5} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512VL-ONLY-NEXT: vextracti64x4 $1, %zmm1, %ymm2
; AVX512VL-ONLY-NEXT: vpaddb %ymm2, %ymm1, %ymm1
; AVX512VL-ONLY-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -2854,40 +2883,46 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %eax
; AVX512VL-ONLY-NEXT: movzbl (%rsp,%rax), %eax
; AVX512VL-ONLY-NEXT: vpextrb $0, %xmm0, (%rsp)
-; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: vpextrb $1, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
-; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $3, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: movl %ecx, %edx
+; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)
+; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
+; AVX512VL-ONLY-NEXT: movl %edx, %ecx
+; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: kshiftrq $5, %k5, %k6
+; AVX512VL-ONLY-NEXT: kshiftrq $3, %k5, %k7
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $4, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
+; AVX512VL-ONLY-NEXT: kshiftrq $4, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
+; AVX512VL-ONLY-NEXT: movl %ecx, %edx
+; AVX512VL-ONLY-NEXT: vpextrb $4, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
-; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rdx)
-; AVX512VL-ONLY-NEXT: kshiftrq $5, %k6, %k7
-; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $6, %k6, %k7
+; AVX512VL-ONLY-NEXT: movl %edx, %ecx
+; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: kmovd %k6, %ecx
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
+; AVX512VL-ONLY-NEXT: movl %ecx, %edx
+; AVX512VL-ONLY-NEXT: vpextrb $6, %xmm0, (%rsp,%rdx)
+; AVX512VL-ONLY-NEXT: kshiftrq $58, %k5, %k6
+; AVX512VL-ONLY-NEXT: kshiftrq $6, %k5, %k7
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $7, %k6, %k7
-; AVX512VL-ONLY-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)
+; AVX512VL-ONLY-NEXT: kshiftrq $7, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -2895,12 +2930,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $8, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $8, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $9, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $9, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -2908,7 +2943,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $10, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $10, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -2919,12 +2954,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $10, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $11, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $11, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $12, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $12, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -2932,7 +2967,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $11, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $13, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $13, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -2943,12 +2978,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $14, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $14, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $15, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $15, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -2956,7 +2991,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $16, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $16, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -2964,7 +2999,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $15, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $17, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $17, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -2973,7 +3008,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512VL-ONLY-NEXT: vpextrb $0, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $18, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $18, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -2984,12 +3019,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $19, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $19, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $20, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $20, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -2997,7 +3032,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $21, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $21, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3008,12 +3043,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $22, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $22, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $23, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $23, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3021,7 +3056,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $6, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $24, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $24, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3032,12 +3067,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $25, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $25, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $26, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $26, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3045,7 +3080,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $9, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $27, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $27, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3056,12 +3091,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $11, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $28, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $28, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $29, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $29, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3069,7 +3104,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $12, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $30, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $30, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3080,12 +3115,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $14, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $31, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $31, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $32, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $32, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3093,7 +3128,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $15, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $33, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $33, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3105,12 +3140,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $1, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $34, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $34, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $35, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $35, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3118,7 +3153,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $36, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $36, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3129,12 +3164,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $4, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $37, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $37, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $38, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $38, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3142,7 +3177,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $39, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $39, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3153,12 +3188,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $7, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $40, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $40, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $41, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $41, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3166,7 +3201,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $42, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $42, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3177,12 +3212,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $10, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $43, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $43, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $44, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $44, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3190,7 +3225,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $11, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $45, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $45, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3201,12 +3236,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $13, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $46, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $46, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $47, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $47, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3214,7 +3249,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $14, %xmm1, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $48, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $48, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3222,7 +3257,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $15, %xmm1, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $49, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $49, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3231,7 +3266,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm0
; AVX512VL-ONLY-NEXT: vpextrb $0, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $50, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $50, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3242,12 +3277,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $51, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $51, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $52, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $52, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3255,7 +3290,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $53, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $53, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3266,12 +3301,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $54, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $54, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
; AVX512VL-ONLY-NEXT: kmovd %k7, %edx
-; AVX512VL-ONLY-NEXT: kshiftrq $55, %k6, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $55, %k5, %k7
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3279,8 +3314,8 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: andl $63, %ecx
; AVX512VL-ONLY-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)
; AVX512VL-ONLY-NEXT: kmovd %k7, %ecx
-; AVX512VL-ONLY-NEXT: kshiftrq $57, %k6, %k7
-; AVX512VL-ONLY-NEXT: kshiftrq $56, %k6, %k6
+; AVX512VL-ONLY-NEXT: kshiftrq $57, %k5, %k7
+; AVX512VL-ONLY-NEXT: kshiftrq $56, %k5, %k5
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
@@ -3290,7 +3325,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: movl %ecx, %edx
; AVX512VL-ONLY-NEXT: andl $63, %edx
; AVX512VL-ONLY-NEXT: vpextrb $8, %xmm0, (%rsp,%rdx)
-; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3301,7 +3336,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512VL-ONLY-NEXT: movzbl %cl, %ecx
; AVX512VL-ONLY-NEXT: andl $1, %ecx
; AVX512VL-ONLY-NEXT: addq %rdx, %rcx
-; AVX512VL-ONLY-NEXT: kmovd %k5, %edx
+; AVX512VL-ONLY-NEXT: kmovd %k6, %edx
; AVX512VL-ONLY-NEXT: movzbl %dl, %edx
; AVX512VL-ONLY-NEXT: andl $1, %edx
; AVX512VL-ONLY-NEXT: addq %rcx, %rdx
@@ -3398,87 +3433,88 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $1, %xmm2, %eax
+; AVX2-NEXT: vpextrb $4, %xmm2, %eax
+; AVX2-NEXT: vpextrb $1, %xmm2, %ecx
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vmovd %xmm2, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: addq %rdx, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $2, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $2, %xmm2, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $3, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addq %rdx, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm2, %eax
-; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $4, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $5, %xmm2, %eax
+; AVX2-NEXT: vpextrb $6, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $6, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $7, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $7, %xmm2, %eax
+; AVX2-NEXT: vpextrb $8, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $8, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $9, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $9, %xmm2, %eax
+; AVX2-NEXT: vpextrb $10, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $10, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $11, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $11, %xmm2, %eax
+; AVX2-NEXT: vpextrb $12, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $12, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $13, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $13, %xmm2, %eax
+; AVX2-NEXT: vpextrb $14, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $15, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $15, %xmm2, %eax
+; AVX2-NEXT: vmovd %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vmovd %xmm3, %ecx
+; AVX2-NEXT: vpextrb $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $1, %xmm3, %eax
+; AVX2-NEXT: vpextrb $2, %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $2, %xmm3, %ecx
+; AVX2-NEXT: vpextrb $3, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm3, %r12d
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rcx, %r12
; AVX2-NEXT: vpextrb $4, %xmm3, %r15d
; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %r12, %r15
+; AVX2-NEXT: addq %rcx, %r15
; AVX2-NEXT: vpextrb $5, %xmm3, %r14d
; AVX2-NEXT: andl $1, %r14d
; AVX2-NEXT: addq %r15, %r14
@@ -3509,13 +3545,12 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpextrb $14, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrb $15, %xmm3, %r13d
-; AVX2-NEXT: andl $1, %r13d
-; AVX2-NEXT: addq %rcx, %r13
-; AVX2-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $15, %xmm3, %r12d
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addq %rcx, %r12
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: cmpq $32, %r13
+; AVX2-NEXT: cmpq $32, %r12
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; AVX2-NEXT: cmovbw (%rsp,%r13,2), %ax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -3527,8 +3562,10 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%r13,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: andl $31, %r13d
; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%r13,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: andl $31, %r13d
; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r13,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; AVX2-NEXT: andl $31, %r13d
@@ -3570,11 +3607,12 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; AVX2-NEXT: andl $31, %r13d
; AVX2-NEXT: vpextrw $2, %xmm1, (%rsp,%r13,2)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: andl $31, %r13d
+; AVX2-NEXT: vpextrw $3, %xmm1, (%rsp,%r13,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX2-NEXT: andl $31, %eax
-; AVX2-NEXT: vpextrw $3, %xmm1, (%rsp,%rax,2)
-; AVX2-NEXT: andl $31, %r12d
-; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%r12,2)
+; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%rax,2)
; AVX2-NEXT: andl $31, %r15d
; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r15,2)
; AVX2-NEXT: andl $31, %r14d
@@ -3597,13 +3635,12 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpextrw $6, %xmm2, (%rsp,%rdx,2)
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: vpextrw $7, %xmm2, (%rsp,%rcx,2)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT: cmpq $31, %rcx
+; AVX2-NEXT: cmpq $31, %r12
; AVX2-NEXT: movl $31, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: movl %eax, %eax
+; AVX2-NEXT: cmovbq %r12, %rax
+; AVX2-NEXT: addl %eax, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
-; AVX2-NEXT: movw %cx, (%rsp,%rax,2)
+; AVX2-NEXT: movw %cx, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -40(%rbp), %rsp
@@ -3985,7 +4022,7 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX512VL-ONLY-NEXT: cmpq $31, %r8
; AVX512VL-ONLY-NEXT: movl $31, %eax
; AVX512VL-ONLY-NEXT: cmovbq %r8, %rax
-; AVX512VL-ONLY-NEXT: movl %eax, %eax
+; AVX512VL-ONLY-NEXT: andl $31, %eax
; AVX512VL-ONLY-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; AVX512VL-ONLY-NEXT: movw %cx, (%rsp,%rax,2)
; AVX512VL-ONLY-NEXT: vmovaps (%rsp), %zmm0
@@ -4023,29 +4060,33 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addl %esi, %edx
+; AVX2-NEXT: addq %rsi, %rdx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rdx,4)
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addl %edx, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: # kill: def $r9d killed $r9d def $r9
; AVX2-NEXT: # kill: def $r8d killed $r8d def $r8
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addl %ecx, %r8d
+; AVX2-NEXT: addq %rcx, %r8
+; AVX2-NEXT: movzbl 16(%rbp), %eax
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%r8,4)
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addl %r8d, %r9d
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r9,4)
-; AVX2-NEXT: movzbl 16(%rbp), %eax
+; AVX2-NEXT: addq %r8, %r9
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addl %r9d, %eax
-; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
+; AVX2-NEXT: andl $63, %r9d
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r9,4)
; AVX2-NEXT: movzbl 24(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
@@ -4057,35 +4098,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm1, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 48(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 56(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 64(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 72(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4093,14 +4134,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 88(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4114,35 +4155,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm2, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 112(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm2, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 120(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm2, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 128(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 136(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4150,14 +4191,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 152(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4171,35 +4212,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm3, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 176(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm3, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 184(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm3, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 192(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm3, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 200(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm3, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4207,14 +4248,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 216(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4228,35 +4269,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm4, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 240(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm4, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 248(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm4, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 256(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm4, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 264(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm4, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4264,14 +4305,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 280(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4285,35 +4326,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm5, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 304(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm5, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 312(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm5, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 320(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm5, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 328(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm5, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4321,14 +4362,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 344(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4342,35 +4383,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm6, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 368(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm6, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 376(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm6, (%rsp,%rax,4)
; AVX2-NEXT: movzbl 384(%rbp), %eax
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm6, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 392(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm6, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
@@ -4378,14 +4419,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 408(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
@@ -4400,35 +4441,35 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $63, %edx
; AVX2-NEXT: vmovss %xmm7, (%rsp,%rdx,4)
; AVX2-NEXT: movzbl 432(%rbp), %edx
; AVX2-NEXT: movzbl %dl, %edx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm7, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 440(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $63, %edx
; AVX2-NEXT: vextractps $2, %xmm7, (%rsp,%rdx,4)
; AVX2-NEXT: movzbl 448(%rbp), %edx
; AVX2-NEXT: movzbl %dl, %edx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $3, %xmm7, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 456(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $63, %edx
; AVX2-NEXT: vextractf128 $1, %ymm7, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)
@@ -4436,14 +4477,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: movzbl %dl, %edx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: movzbl 472(%rbp), %ecx
; AVX2-NEXT: movzbl %cl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $63, %edx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rdx,4)
; AVX2-NEXT: andl $63, %ecx
@@ -4673,56 +4714,56 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: vmovd %xmm1, %edx
+; AVX2-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-NEXT: vmovd %xmm2, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vpextrw $4, %xmm1, %eax
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vpextrw $1, %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrw $2, %xmm1, %esi
+; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX2-NEXT: addq %rsi, %rdx
+; AVX2-NEXT: vpextrw $2, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
-; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
+; AVX2-NEXT: addq %rdx, %r8
+; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: vpextrw $5, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rax, %rdi
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addq %rdi, %r10
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
; AVX2-NEXT: vmovq %xmm0, (%rsp)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, (%rsp,%rcx,8)
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm4, (%rsp,%rdi,8)
-; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%r8,8)
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
+; AVX2-NEXT: vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rax,8)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r11, %rax
-; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)
+; AVX2-NEXT: shll $3, %eax
+; AVX2-NEXT: movq %rbx, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -8(%rbp), %rsp
@@ -4777,56 +4818,56 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: vmovd %xmm1, %edx
+; AVX2-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-NEXT: vmovd %xmm2, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vpextrw $4, %xmm1, %eax
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vpextrw $1, %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrw $2, %xmm1, %esi
+; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX2-NEXT: addq %rsi, %rdx
+; AVX2-NEXT: vpextrw $2, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
-; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
+; AVX2-NEXT: addq %rdx, %r8
+; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: vpextrw $5, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rax, %rdi
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addq %rdi, %r10
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vpmovsxwq %xmm0, %ymm0
; AVX2-NEXT: vmovq %xmm0, (%rsp)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, (%rsp,%rcx,8)
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm4, (%rsp,%rdi,8)
-; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%r8,8)
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
+; AVX2-NEXT: vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rax,8)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r11, %rax
-; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)
+; AVX2-NEXT: shll $3, %eax
+; AVX2-NEXT: movq %rbx, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -8(%rbp), %rsp
@@ -4973,6 +5014,18 @@ define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) nounwind {
; AVX2-NEXT: vpextrb $3, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vpextrb $4, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $6, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $7, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $8, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $9, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $10, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $11, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $12, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $13, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $14, %xmm0, -24(%rsp,%rcx)
; AVX2-NEXT: vpextrb $15, %xmm0, -24(%rsp,%rcx)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -5019,8 +5072,9 @@ define <4 x i4> @test_compress_illegal_element_type(<4 x i4> %vec, <4 x i1> %mas
; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
; AVX2-NEXT: subl %ecx, %eax
-; AVX2-NEXT: leal (,%rax,4), %ecx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
; AVX2-NEXT: subl %ecx, %eax
; AVX2-NEXT: andl $3, %eax
@@ -5062,13 +5116,16 @@ define <3 x i32> @test_compress_narrow(<3 x i32> %vec, <3 x i1> %mask) nounwind
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %eax
-; AVX2-NEXT: leal (,%rax,4), %ecx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx)
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %eax
-; AVX2-NEXT: andl $3, %eax
-; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpextrd $2, %xmm1, %edx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addl %ecx, %edx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: andl $3, %edx
+; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rdx,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
@@ -5115,9 +5172,10 @@ define <3 x i3> @test_compress_narrow_illegal_element_type(<3 x i3> %vec, <3 x i
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: movl %esi, -24(%rsp,%rax,4)
; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: subl %ecx, %eax
-; AVX2-NEXT: shll $2, %eax
-; AVX2-NEXT: movl %edx, -24(%rsp,%rax)
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: movl %edx, -24(%rsp,%rcx,4)
; AVX2-NEXT: vmovdqa -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: vpextrb $4, %xmm0, %edx
@@ -5191,20 +5249,22 @@ define <4 x i32> @test_compress_v4i32_zero_passthru(<4 x i32> %vec, <4 x i1> %ma
; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
; AVX2-NEXT: vpextrd $2, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vpextrd $3, %xmm1, %edx
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rax, %rdx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: cmpq $3, %rcx
-; AVX2-NEXT: vextractps $3, %xmm0, %edx
-; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: cmpq $3, %rdx
; AVX2-NEXT: movl $3, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: movl %edx, -24(%rsp,%rax,4)
+; AVX2-NEXT: cmovbq %rdx, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vextractps $3, %xmm0, %ecx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: cmpq $4, %rdx
+; AVX2-NEXT: cmovael %ecx, %esi
+; AVX2-NEXT: movl %esi, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-extract-last-active.ll b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
index fcf9c295c3852..07f5eaf8f6813 100644
--- a/llvm/test/CodeGen/X86/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
@@ -179,16 +179,15 @@ define i32 @extract_last_active_v4i32_no_default(<4 x i32> %a, <4 x i1> %c) noun
define i32 @extract_last_active_v2i32(<2 x i32> %a, <2 x i1> %c) nounwind {
; SSE2-LABEL: extract_last_active_v2i32:
; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: psllq $63, %xmm1
; SSE2-NEXT: movmskpd %xmm1, %ecx
; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: cmpl $1, %ecx
; SSE2-NEXT: sbbl %eax, %eax
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
; SSE2-NEXT: psrld $31, %xmm0
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: movl %ecx, %ecx
; SSE2-NEXT: orl -24(%rsp,%rcx,4), %eax
; SSE2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-variable-128.ll b/llvm/test/CodeGen/X86/vector-shuffle-variable-128.ll
index ce8d2acd035f6..3ca0e2121e0d1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-variable-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-variable-128.ll
@@ -255,28 +255,28 @@ define <8 x i16> @var_shuffle_v8i16_v8i16_xxxxxxxx_i16(<8 x i16> %x, i16 %i0, i1
; SSE2-NEXT: andl $7, %r8d
; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: andl $7, %r9d
-; SSE2-NEXT: movzwl -24(%rsp,%r10,2), %r10d
-; SSE2-NEXT: movd %r10d, %xmm0
-; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movzwl -24(%rsp,%r9,2), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzwl -24(%rsp,%r8,2), %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: movzwl -24(%rsp,%rcx,2), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzwl -24(%rsp,%rdx,2), %eax
-; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm0
+; SSE2-NEXT: movzwl -24(%rsp,%rdx,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm1
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movzwl -24(%rsp,%rsi,2), %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movzwl -24(%rsp,%rdi,2), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: movzwl -24(%rsp,%rsi,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: movzwl -24(%rsp,%rdi,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movzwl -24(%rsp,%r9,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm1
+; SSE2-NEXT: movzwl -24(%rsp,%r8,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: movzwl -24(%rsp,%r10,2), %ecx
+; SSE2-NEXT: movd %ecx, %xmm1
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSE2-NEXT: retq
;
@@ -299,28 +299,28 @@ define <8 x i16> @var_shuffle_v8i16_v8i16_xxxxxxxx_i16(<8 x i16> %x, i16 %i0, i1
; SSSE3-NEXT: andl $7, %r8d
; SSSE3-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSSE3-NEXT: andl $7, %r9d
-; SSSE3-NEXT: movzwl -24(%rsp,%r10,2), %r10d
-; SSSE3-NEXT: movd %r10d, %xmm0
-; SSSE3-NEXT: movzwl -24(%rsp,%rax,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm1
-; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT: movzwl -24(%rsp,%r9,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm0
-; SSSE3-NEXT: movzwl -24(%rsp,%r8,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm2
-; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT: movzwl -24(%rsp,%rcx,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm0
-; SSSE3-NEXT: movzwl -24(%rsp,%rdx,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm1
+; SSSE3-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm0
+; SSSE3-NEXT: movzwl -24(%rsp,%rdx,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm1
; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT: movzwl -24(%rsp,%rsi,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm3
-; SSSE3-NEXT: movzwl -24(%rsp,%rdi,2), %eax
-; SSSE3-NEXT: movd %eax, %xmm0
-; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT: movzwl -24(%rsp,%rsi,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm2
+; SSSE3-NEXT: movzwl -24(%rsp,%rdi,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm0
+; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT: movzwl -24(%rsp,%r9,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm1
+; SSSE3-NEXT: movzwl -24(%rsp,%r8,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm2
+; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT: movzwl -24(%rsp,%r10,2), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm1
+; SSSE3-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSSE3-NEXT: movd %eax, %xmm3
+; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-NEXT: retq
;
More information about the llvm-commits
mailing list