[llvm] [DAGCombiner] `visitSIGN_EXTEND_INREG` - fold `(sext_inreg (freeze (extload x)))` -> `(freeze (sextload x))` (PR #226208)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 02:01:29 PDT 2026
https://github.com/VachanVY updated https://github.com/llvm/llvm-project/pull/226208
>From daf557c81c5600a4be98eb0453bf1bc2cd8fa6c7 Mon Sep 17 00:00:00 2001
From: Vachan V Y <vachanvy05 at gmail.com>
Date: Fri, 25 Sep 2026 22:05:45 +0530
Subject: [PATCH 1/2] Squashed commit of the following:
commit 3eb925780c594130fe63027bef094d4d3377c0b8
Author: Vachan V Y <vachanvy05 at gmail.com>
Date: Wed Aug 26 22:51:50 2026 +0530
[DAGCombine] Freeze = Op == Freeze && Load is used once
`visitANY_EXTEND` - fold `(aext (freeze (load x)))` -> `(aext (truncate (freeze (extload x))))`
commit 4f5b1e7f3b2441bb75bdeac00a793a2ff3bd0242
Author: Vachan V Y <vachanvy05 at gmail.com>
Date: Wed Aug 26 22:16:02 2026 +0530
Revert "[DAGCombine] add tests"
This reverts commit 0dfaba5979dc6be8c542c9b0d62692b05bd96a8c.
commit 035bced366cb5e7b4583c78fefdc881d4d8ac862
Author: Vachan V Y <vachanvy05 at gmail.com>
Date: Mon Aug 24 21:46:16 2026 +0530
[DAGCombine] run llc through merge conflict resolved files; fix tests
commit d8aae06f636381b09892481dea92d1da217a8b41
Merge: c42649171d48 2ed0f89b61bb
Author: Vachan <vachanvy05 at gmail.com>
Date: Tue Aug 25 00:54:38 2026 +0530
Merge branch 'main' into dagcombiner-anyextend-fold
commit c42649171d48be5b1923eaca0d9c4490940cad0c
Author: Vachan V Y <vachanvy05 at gmail.com>
Date: Mon Aug 24 21:49:37 2026 +0530
[DAGCombine] check diff + optimization
`visitANY_EXTEND` - fold `(aext (freeze (load x)))` -> `(aext (truncate (freeze (extload x))))`
commit 0dfaba5979dc6be8c542c9b0d62692b05bd96a8c
Author: Vachan V Y <vachanvy05 at gmail.com>
Date: Mon Aug 24 21:46:16 2026 +0530
[DAGCombine] add tests
`visitANY_EXTEND` - fold `(aext (freeze (load x)))` -> `(aext (truncate (freeze (extload x))))`
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 62 +-
llvm/test/CodeGen/X86/bit-manip-i512.ll | 28 +-
llvm/test/CodeGen/X86/iabs.ll | 4 +-
llvm/test/CodeGen/X86/icmp-abs-C.ll | 4 +-
.../test/CodeGen/X86/icmp-pow2-logic-npow2.ll | 4 +-
llvm/test/CodeGen/X86/midpoint-int.ll | 22 +-
llvm/test/CodeGen/X86/neg-abs.ll | 8 +-
llvm/test/CodeGen/X86/shift-i512.ll | 50 +-
llvm/test/CodeGen/X86/var-permute-128.ll | 36 +-
llvm/test/CodeGen/X86/vector-compress.ll | 964 ++++++++----------
.../Inputs/basic.ll.expected | 4 +-
11 files changed, 541 insertions(+), 645 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 28a8cb94096488..46c39f2669b4a7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -16710,6 +16710,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
}
// fold (aext (load x)) -> (aext (truncate (extload x)))
+ // fold (aext (freeze (load x))) -> (aext (truncate (freeze (extload x))))
// None of the supported targets knows how to perform load and any_ext
// on vectors in one instruction, so attempt to fold to zext instead.
if (VT.isVector()) {
@@ -16718,33 +16719,42 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
return foldedExt;
- } else if (ISD::isNON_EXTLoad(N0.getNode()) &&
- ISD::isUNINDEXEDLoad(N0.getNode())) {
- LoadSDNode *LN0 = cast<LoadSDNode>(N0);
- if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(),
- LN0->getAddressSpace(), ISD::EXTLOAD, false)) {
- bool DoXform = true;
- SmallVector<SDNode *, 4> SetCCs;
- if (!N0.hasOneUse())
- DoXform =
- ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
- if (DoXform) {
- SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
- LN0->getBasePtr(), N0.getValueType(),
- LN0->getMemOperand());
- ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
- // If the load value is used only by N, replace it via CombineTo N.
- bool NoReplaceTrunc = N0.hasOneUse();
- CombineTo(N, ExtLoad);
- if (NoReplaceTrunc) {
- DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
- recursivelyDeleteUnusedNodes(LN0);
- } else {
- SDValue Trunc =
- DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
- CombineTo(LN0, Trunc, ExtLoad.getValue(1));
+ } else {
+ // TODO: Support multiple uses of the load when frozen.
+ bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.getOperand(0).hasOneUse();
+ SDValue LoadOp = Frozen ? N0.getOperand(0) : N0;
+ EVT LoadVT = LoadOp.getValueType();
+ if (ISD::isNON_EXTLoad(LoadOp.getNode()) &&
+ ISD::isUNINDEXEDLoad(LoadOp.getNode())) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(LoadOp);
+ if (TLI.isLoadLegalOrCustom(VT, LoadVT, LN0->getAlign(),
+ LN0->getAddressSpace(), ISD::EXTLOAD,
+ false)) {
+ bool DoXform = true;
+ SmallVector<SDNode *, 4> SetCCs;
+ // N0 is a Load and has multiple uses.
+ if (!Frozen && !N0.hasOneUse())
+ DoXform =
+ ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
+ if (DoXform) {
+ SDValue ExtLoad =
+ DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
+ LN0->getBasePtr(), LoadVT, LN0->getMemOperand());
+ SDValue Res = Frozen ? DAG.getFreeze(ExtLoad) : ExtLoad;
+ ExtendSetCCUses(SetCCs, N0, Res, ISD::ANY_EXTEND);
+ // If the load or freeze value is used only by N, replace it via
+ // CombineTo N.
+ bool NoReplaceTrunc = N0.hasOneUse();
+ CombineTo(N, Res);
+ if (NoReplaceTrunc) {
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+ recursivelyDeleteUnusedNodes(N0.getNode());
+ } else {
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), LoadVT, Res);
+ CombineTo(LN0, Trunc, ExtLoad.getValue(1));
+ }
+ return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
- return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
}
}
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 9e4d00650e6137..fb1fd173a7d226 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -459,31 +459,31 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %edi, %ecx
+; AVX512VBMI-NEXT: vpbroadcastq %rdi, %zmm1
+; AVX512VBMI-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi
; AVX512VBMI-NEXT: shrl $3, %edi
; AVX512VBMI-NEXT: andl $56, %edi
-; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm1[1,2,3,4,5,6,7],zmm0[0]
-; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm3
-; AVX512VBMI-NEXT: vpshrdvq %zmm3, %zmm0, %zmm1
+; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm3
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm3[1,2,3,4,5,6,7],zmm0[0]
+; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm0, %zmm3
; AVX512VBMI-NEXT: vextracti128 $1, %ymm2, %xmm0
; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX512VBMI-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm3, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512VBMI-NEXT: vmovq %xmm2, %rsi
; AVX512VBMI-NEXT: andq %r11, %rcx
-; AVX512VBMI-NEXT: vmovq %xmm3, %rdi
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm1, %xmm2
-; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %r11
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdi
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm3, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r11
; AVX512VBMI-NEXT: andq %r10, %rdi
; AVX512VBMI-NEXT: andq %rbx, %r11
-; AVX512VBMI-NEXT: vmovq %xmm2, %r10
+; AVX512VBMI-NEXT: vmovq %xmm1, %r10
; AVX512VBMI-NEXT: andq %r9, %r10
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r9
+; AVX512VBMI-NEXT: vpextrq $1, %xmm3, %r9
; AVX512VBMI-NEXT: andq %r8, %r9
-; AVX512VBMI-NEXT: vmovq %xmm1, %r8
+; AVX512VBMI-NEXT: vmovq %xmm3, %r8
; AVX512VBMI-NEXT: andq %rsi, %r8
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm3, %xmm1
; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
; AVX512VBMI-NEXT: vmovq %xmm0, %rbx
; AVX512VBMI-NEXT: andq %rdx, %rsi
diff --git a/llvm/test/CodeGen/X86/iabs.ll b/llvm/test/CodeGen/X86/iabs.ll
index 3c3171032451a9..f18f49e342962b 100644
--- a/llvm/test/CodeGen/X86/iabs.ll
+++ b/llvm/test/CodeGen/X86/iabs.ll
@@ -37,8 +37,8 @@ define i8 @test_i8(i8 %a) nounwind {
define i16 @test_i16(i16 %a) nounwind {
; X86-NO-CMOV-LABEL: test_i16:
; X86-NO-CMOV: # %bb.0:
-; X86-NO-CMOV-NEXT: movswl {{[0-9]+}}(%esp), %eax
-; X86-NO-CMOV-NEXT: movl %eax, %ecx
+; X86-NO-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-CMOV-NEXT: movswl %ax, %ecx
; X86-NO-CMOV-NEXT: sarl $15, %ecx
; X86-NO-CMOV-NEXT: xorl %ecx, %eax
; X86-NO-CMOV-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/icmp-abs-C.ll b/llvm/test/CodeGen/X86/icmp-abs-C.ll
index 4773a369326f7f..5c88909de9eb64 100644
--- a/llvm/test/CodeGen/X86/icmp-abs-C.ll
+++ b/llvm/test/CodeGen/X86/icmp-abs-C.ll
@@ -161,8 +161,8 @@ define i16 @ne_and_with_dom_abs(i16 %x) nounwind {
; X86-LABEL: ne_and_with_dom_abs:
; X86: # %bb.0:
; X86-NEXT: pushl %esi
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movswl %ax, %ecx
; X86-NEXT: sarl $15, %ecx
; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
index b14fedeaae57d1..0a85a395895dc1 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
@@ -196,8 +196,8 @@ define i1 @abs_ne_pow2(i64 %0) nounwind {
define i1 @abs_ne_nonpow2(i16 %0) nounwind {
; X86-LABEL: abs_ne_nonpow2:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movswl %ax, %ecx
; X86-NEXT: sarl $15, %ecx
; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/midpoint-int.ll b/llvm/test/CodeGen/X86/midpoint-int.ll
index 4de448a099b30a..9d1141e1128ba1 100644
--- a/llvm/test/CodeGen/X86/midpoint-int.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int.ll
@@ -653,9 +653,9 @@ define i16 @scalar_i16_signed_reg_reg(i16 %a1, i16 %a2) nounwind {
; X86: # %bb.0:
; X86-NEXT: pushl %ebx
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: subw %dx, %ax
; X86-NEXT: setle %bl
; X86-NEXT: leal -1(%ebx,%ebx), %edx
@@ -761,9 +761,9 @@ define i16 @scalar_i16_signed_mem_reg(ptr %a1_addr, i16 %a2) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzwl (%eax), %eax
+; X86-NEXT: movzwl (%eax), %ecx
; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: subw %dx, %ax
; X86-NEXT: setle %bl
; X86-NEXT: leal -1(%ebx,%ebx), %edx
@@ -813,11 +813,11 @@ define i16 @scalar_i16_signed_reg_mem(i16 %a1, ptr %a2_addr) nounwind {
; X86-LABEL: scalar_i16_signed_reg_mem:
; X86: # %bb.0:
; X86-NEXT: pushl %ebx
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movzwl (%ecx), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: subw %dx, %ax
; X86-NEXT: setle %bl
; X86-NEXT: leal -1(%ebx,%ebx), %edx
@@ -867,12 +867,12 @@ define i16 @scalar_i16_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
; X86-LABEL: scalar_i16_signed_mem_mem:
; X86: # %bb.0:
; X86-NEXT: pushl %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzwl (%eax), %eax
-; X86-NEXT: movzwl (%ecx), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzwl (%ecx), %ecx
+; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: subw %dx, %ax
; X86-NEXT: setle %bl
; X86-NEXT: leal -1(%ebx,%ebx), %edx
diff --git a/llvm/test/CodeGen/X86/neg-abs.ll b/llvm/test/CodeGen/X86/neg-abs.ll
index 050610426e6bd1..47c95e1c0943c5 100644
--- a/llvm/test/CodeGen/X86/neg-abs.ll
+++ b/llvm/test/CodeGen/X86/neg-abs.ll
@@ -34,8 +34,8 @@ define i8 @neg_abs_i8(i8 %x) nounwind {
define i16 @neg_abs_i16(i16 %x) nounwind {
; X86-LABEL: neg_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movswl %cx, %eax
; X86-NEXT: sarl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
@@ -182,8 +182,8 @@ define i8 @sub_abs_i8(i8 %x, i8 %y) nounwind {
define i16 @sub_abs_i16(i16 %x, i16 %y) nounwind {
; X86-LABEL: sub_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movswl %cx, %eax
; X86-NEXT: sarl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index 4f1adab266822d..c58b576bf05f8e 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -220,17 +220,17 @@ define i512 @shl_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vpbroadcastq %rdi, %zmm1
; AVX512VBMI-NEXT: movl %edi, %ecx
-; AVX512VBMI-NEXT: shrl $3, %edi
-; AVX512VBMI-NEXT: andl $56, %edi
-; AVX512VBMI-NEXT: negl %edi
-; AVX512VBMI-NEXT: movslq %edi, %rdx
-; AVX512VBMI-NEXT: vmovdqu64 -64(%rsp,%rdx), %zmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm0[7],zmm1[0,1,2,3,4,5,6]
-; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm2
-; AVX512VBMI-NEXT: vpshldvq %zmm2, %zmm0, %zmm1
-; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm1, 32(%rax)
-; AVX512VBMI-NEXT: vmovdqu %ymm1, (%rax)
+; AVX512VBMI-NEXT: shrl $3, %ecx
+; AVX512VBMI-NEXT: andl $56, %ecx
+; AVX512VBMI-NEXT: negl %ecx
+; AVX512VBMI-NEXT: movslq %ecx, %rcx
+; AVX512VBMI-NEXT: vmovdqu64 -64(%rsp,%rcx), %zmm2
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm0[7],zmm2[0,1,2,3,4,5,6]
+; AVX512VBMI-NEXT: vpshldvq %zmm1, %zmm0, %zmm2
+; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm2, 32(%rax)
+; AVX512VBMI-NEXT: vmovdqu %ymm2, (%rax)
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
@@ -434,15 +434,15 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl {{[0-9]+}}(%rsp), %ecx
-; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1
+; AVX512VBMI-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX512VBMI-NEXT: shrl $3, %ecx
; AVX512VBMI-NEXT: andl $56, %ecx
-; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rcx), %zmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm1[1,2,3,4,5,6,7],zmm0[0]
-; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm2
-; AVX512VBMI-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1
-; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm1, 32(%rax)
-; AVX512VBMI-NEXT: vmovdqu %ymm1, (%rax)
+; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rcx), %zmm2
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm2[1,2,3,4,5,6,7],zmm0[0]
+; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm0, %zmm2
+; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm2, 32(%rax)
+; AVX512VBMI-NEXT: vmovdqu %ymm2, (%rax)
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
@@ -676,16 +676,16 @@ define i512 @ashr_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %edi, %ecx
+; AVX512VBMI-NEXT: vpbroadcastq %rdi, %zmm0
+; AVX512VBMI-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi
; AVX512VBMI-NEXT: shrl $3, %edi
; AVX512VBMI-NEXT: andl $56, %edi
-; AVX512VBMI-NEXT: vpsraq $63, -72(%rsp,%rdi){1to2}, %xmm0
-; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm1[1,2,3,4,5,6,7],zmm0[0]
-; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm2
-; AVX512VBMI-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1
-; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm1, 32(%rax)
-; AVX512VBMI-NEXT: vmovdqu %ymm1, (%rax)
+; AVX512VBMI-NEXT: vpsraq $63, -72(%rsp,%rdi){1to2}, %xmm1
+; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm2
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm1 = zmm2[1,2,3,4,5,6,7],zmm1[0]
+; AVX512VBMI-NEXT: vpshrdvq %zmm0, %zmm1, %zmm2
+; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm2, 32(%rax)
+; AVX512VBMI-NEXT: vmovdqu %ymm2, (%rax)
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index dbfad9562d65a2..d1dc25989b3f54 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -501,39 +501,39 @@ define <8 x i16> @var_shuffle_zero_v8i16(<8 x i16> %v, <8 x i16> %indices) nounw
; SSE3-NEXT: pextrw $0, %xmm1, %eax
; SSE3-NEXT: pextrw $1, %xmm1, %ecx
; SSE3-NEXT: pextrw $2, %xmm1, %edx
-; SSE3-NEXT: pextrw $3, %xmm1, %edi
-; SSE3-NEXT: pextrw $4, %xmm1, %r8d
-; SSE3-NEXT: pextrw $5, %xmm1, %r9d
-; SSE3-NEXT: pextrw $6, %xmm1, %r10d
-; SSE3-NEXT: pextrw $7, %xmm1, %esi
+; SSE3-NEXT: pextrw $3, %xmm1, %esi
+; SSE3-NEXT: pextrw $4, %xmm1, %edi
+; SSE3-NEXT: pextrw $5, %xmm1, %r8d
+; SSE3-NEXT: pextrw $6, %xmm1, %r9d
+; SSE3-NEXT: pextrw $7, %xmm1, %r10d
; SSE3-NEXT: movdqa %xmm2, -24(%rsp)
; SSE3-NEXT: andl $7, %eax
-; SSE3-NEXT: movzwl -24(%rsp,%rax,2), %eax
; SSE3-NEXT: andl $7, %ecx
-; SSE3-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
; SSE3-NEXT: andl $7, %edx
-; SSE3-NEXT: movzwl -24(%rsp,%rdx,2), %edx
+; SSE3-NEXT: andl $7, %esi
; SSE3-NEXT: andl $7, %edi
-; SSE3-NEXT: movzwl -24(%rsp,%rdi,2), %edi
; SSE3-NEXT: andl $7, %r8d
-; SSE3-NEXT: movzwl -24(%rsp,%r8,2), %r8d
; SSE3-NEXT: andl $7, %r9d
-; SSE3-NEXT: movzwl -24(%rsp,%r9,2), %r9d
; SSE3-NEXT: andl $7, %r10d
; SSE3-NEXT: movzwl -24(%rsp,%r10,2), %r10d
-; SSE3-NEXT: andl $7, %esi
-; SSE3-NEXT: movzwl -24(%rsp,%rsi,2), %esi
-; SSE3-NEXT: movd %esi, %xmm1
-; SSE3-NEXT: movd %r10d, %xmm2
+; SSE3-NEXT: movd %r10d, %xmm1
+; SSE3-NEXT: movzwl -24(%rsp,%r9,2), %r9d
+; SSE3-NEXT: movd %r9d, %xmm2
; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE3-NEXT: movd %r9d, %xmm1
-; SSE3-NEXT: movd %r8d, %xmm3
+; SSE3-NEXT: movzwl -24(%rsp,%r8,2), %r8d
+; SSE3-NEXT: movd %r8d, %xmm1
+; SSE3-NEXT: movzwl -24(%rsp,%rdi,2), %edi
+; SSE3-NEXT: movd %edi, %xmm3
; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE3-NEXT: movd %edi, %xmm1
+; SSE3-NEXT: movzwl -24(%rsp,%rsi,2), %esi
+; SSE3-NEXT: movd %esi, %xmm1
+; SSE3-NEXT: movzwl -24(%rsp,%rdx,2), %edx
; SSE3-NEXT: movd %edx, %xmm2
; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
; SSE3-NEXT: movd %ecx, %xmm1
+; SSE3-NEXT: movzwl -24(%rsp,%rax,2), %eax
; SSE3-NEXT: movd %eax, %xmm4
; SSE3-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
; SSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 39ef06af993404..90e3a81385112d 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -2068,74 +2068,74 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: andq $-32, %rsp
; AVX2-NEXT: subq $96, %rsp
-; AVX2-NEXT: movzbl 360(%rbp), %eax
-; AVX2-NEXT: movzbl 352(%rbp), %r10d
+; AVX2-NEXT: movl 360(%rbp), %eax
+; AVX2-NEXT: movl 352(%rbp), %r10d
; AVX2-NEXT: vmovd %r10d, %xmm4
; AVX2-NEXT: vpinsrb $1, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 368(%rbp), %eax
+; AVX2-NEXT: movl 368(%rbp), %eax
; AVX2-NEXT: vpinsrb $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 376(%rbp), %eax
+; AVX2-NEXT: movl 376(%rbp), %eax
; AVX2-NEXT: vpinsrb $3, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 384(%rbp), %eax
+; AVX2-NEXT: movl 384(%rbp), %eax
; AVX2-NEXT: vpinsrb $4, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 392(%rbp), %eax
+; AVX2-NEXT: movl 392(%rbp), %eax
; AVX2-NEXT: vpinsrb $5, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 400(%rbp), %eax
+; AVX2-NEXT: movl 400(%rbp), %eax
; AVX2-NEXT: vpinsrb $6, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 408(%rbp), %eax
+; AVX2-NEXT: movl 408(%rbp), %eax
; AVX2-NEXT: vpinsrb $7, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 416(%rbp), %eax
+; AVX2-NEXT: movl 416(%rbp), %eax
; AVX2-NEXT: vpinsrb $8, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 424(%rbp), %eax
+; AVX2-NEXT: movl 424(%rbp), %eax
; AVX2-NEXT: vpinsrb $9, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 432(%rbp), %eax
+; AVX2-NEXT: movl 432(%rbp), %eax
; AVX2-NEXT: vpinsrb $10, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 440(%rbp), %eax
+; AVX2-NEXT: movl 440(%rbp), %eax
; AVX2-NEXT: vpinsrb $11, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 448(%rbp), %eax
+; AVX2-NEXT: movl 448(%rbp), %eax
; AVX2-NEXT: vpinsrb $12, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 456(%rbp), %eax
+; AVX2-NEXT: movl 456(%rbp), %eax
; AVX2-NEXT: vpinsrb $13, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 464(%rbp), %eax
+; AVX2-NEXT: movl 464(%rbp), %eax
; AVX2-NEXT: vpinsrb $14, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 472(%rbp), %eax
+; AVX2-NEXT: movl 472(%rbp), %eax
; AVX2-NEXT: vpinsrb $15, %eax, %xmm4, %xmm4
-; AVX2-NEXT: movzbl 224(%rbp), %eax
+; AVX2-NEXT: movl 224(%rbp), %eax
; AVX2-NEXT: vmovd %eax, %xmm5
-; AVX2-NEXT: movzbl 232(%rbp), %eax
+; AVX2-NEXT: movl 232(%rbp), %eax
; AVX2-NEXT: vpinsrb $1, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 240(%rbp), %eax
+; AVX2-NEXT: movl 240(%rbp), %eax
; AVX2-NEXT: vpinsrb $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 248(%rbp), %eax
+; AVX2-NEXT: movl 248(%rbp), %eax
; AVX2-NEXT: vpinsrb $3, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 256(%rbp), %eax
+; AVX2-NEXT: movl 256(%rbp), %eax
; AVX2-NEXT: vpinsrb $4, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 264(%rbp), %eax
+; AVX2-NEXT: movl 264(%rbp), %eax
; AVX2-NEXT: vpinsrb $5, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 272(%rbp), %eax
+; AVX2-NEXT: movl 272(%rbp), %eax
; AVX2-NEXT: vpinsrb $6, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 280(%rbp), %eax
+; AVX2-NEXT: movl 280(%rbp), %eax
; AVX2-NEXT: vpinsrb $7, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 288(%rbp), %eax
+; AVX2-NEXT: movl 288(%rbp), %eax
; AVX2-NEXT: vpinsrb $8, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 296(%rbp), %eax
+; AVX2-NEXT: movl 296(%rbp), %eax
; AVX2-NEXT: vpinsrb $9, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 304(%rbp), %eax
+; AVX2-NEXT: movl 304(%rbp), %eax
; AVX2-NEXT: vpinsrb $10, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 312(%rbp), %eax
+; AVX2-NEXT: movl 312(%rbp), %eax
; AVX2-NEXT: vpinsrb $11, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 320(%rbp), %eax
+; AVX2-NEXT: movl 320(%rbp), %eax
; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 328(%rbp), %eax
+; AVX2-NEXT: movl 328(%rbp), %eax
; AVX2-NEXT: vpinsrb $13, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 336(%rbp), %eax
+; AVX2-NEXT: movl 336(%rbp), %eax
; AVX2-NEXT: vpinsrb $14, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 344(%rbp), %eax
+; AVX2-NEXT: movl 344(%rbp), %eax
; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm5
; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: # kill: def $esi killed $esi def $rsi
-; AVX2-NEXT: # kill: def $edi killed $edi def $rdi
+; AVX2-NEXT: movl %edi, %r14d
; AVX2-NEXT: # kill: def $r9d killed $r9d def $r9
; AVX2-NEXT: # kill: def $r8d killed $r8d def $r8
; AVX2-NEXT: vmovd %edi, %xmm6
@@ -2144,58 +2144,58 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpinsrb $3, %ecx, %xmm6, %xmm6
; AVX2-NEXT: vpinsrb $4, %r8d, %xmm6, %xmm6
; AVX2-NEXT: vpinsrb $5, %r9d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 16(%rbp), %r10d
-; AVX2-NEXT: vpinsrb $6, %r10d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 24(%rbp), %r11d
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 32(%rbp), %ebx
-; AVX2-NEXT: vpinsrb $8, %ebx, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 40(%rbp), %r14d
-; AVX2-NEXT: vpinsrb $9, %r14d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 48(%rbp), %r15d
+; AVX2-NEXT: movl 16(%rbp), %edi
+; AVX2-NEXT: vpinsrb $6, %edi, %xmm6, %xmm6
+; AVX2-NEXT: movl 24(%rbp), %r10d
+; AVX2-NEXT: vpinsrb $7, %r10d, %xmm6, %xmm6
+; AVX2-NEXT: movl 32(%rbp), %r11d
+; AVX2-NEXT: vpinsrb $8, %r11d, %xmm6, %xmm6
+; AVX2-NEXT: movl 40(%rbp), %ebx
+; AVX2-NEXT: vpinsrb $9, %ebx, %xmm6, %xmm6
+; AVX2-NEXT: movl 48(%rbp), %r15d
; AVX2-NEXT: vpinsrb $10, %r15d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 56(%rbp), %r12d
+; AVX2-NEXT: movl 56(%rbp), %r12d
; AVX2-NEXT: vpinsrb $11, %r12d, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 64(%rbp), %eax
+; AVX2-NEXT: movl 64(%rbp), %eax
; AVX2-NEXT: vpinsrb $12, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 72(%rbp), %eax
+; AVX2-NEXT: movl 72(%rbp), %eax
; AVX2-NEXT: vpinsrb $13, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 80(%rbp), %eax
+; AVX2-NEXT: movl 80(%rbp), %eax
; AVX2-NEXT: vpinsrb $14, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 88(%rbp), %eax
+; AVX2-NEXT: movl 88(%rbp), %eax
; AVX2-NEXT: vpinsrb $15, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 96(%rbp), %eax
+; AVX2-NEXT: movl 96(%rbp), %eax
; AVX2-NEXT: vmovd %eax, %xmm7
-; AVX2-NEXT: movzbl 104(%rbp), %eax
+; AVX2-NEXT: movl 104(%rbp), %eax
; AVX2-NEXT: vpinsrb $1, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 112(%rbp), %eax
+; AVX2-NEXT: movl 112(%rbp), %eax
; AVX2-NEXT: vpinsrb $2, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 120(%rbp), %eax
+; AVX2-NEXT: movl 120(%rbp), %eax
; AVX2-NEXT: vpinsrb $3, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 128(%rbp), %eax
+; AVX2-NEXT: movl 128(%rbp), %eax
; AVX2-NEXT: vpinsrb $4, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 136(%rbp), %eax
+; AVX2-NEXT: movl 136(%rbp), %eax
; AVX2-NEXT: vpinsrb $5, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 144(%rbp), %eax
+; AVX2-NEXT: movl 144(%rbp), %eax
; AVX2-NEXT: vpinsrb $6, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 152(%rbp), %eax
+; AVX2-NEXT: movl 152(%rbp), %eax
; AVX2-NEXT: vpinsrb $7, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 160(%rbp), %eax
+; AVX2-NEXT: movl 160(%rbp), %eax
; AVX2-NEXT: vpinsrb $8, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 168(%rbp), %eax
+; AVX2-NEXT: movl 168(%rbp), %eax
; AVX2-NEXT: vpinsrb $9, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 176(%rbp), %eax
+; AVX2-NEXT: movl 176(%rbp), %eax
; AVX2-NEXT: vpinsrb $10, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 184(%rbp), %eax
+; AVX2-NEXT: movl 184(%rbp), %eax
; AVX2-NEXT: vpinsrb $11, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 192(%rbp), %eax
+; AVX2-NEXT: movl 192(%rbp), %eax
; AVX2-NEXT: vpinsrb $12, %eax, %xmm7, %xmm7
-; AVX2-NEXT: movzbl 200(%rbp), %eax
+; AVX2-NEXT: movl 200(%rbp), %eax
; AVX2-NEXT: vpinsrb $13, %eax, %xmm7, %xmm7
; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm5, %ymm4
-; AVX2-NEXT: movzbl 208(%rbp), %eax
+; AVX2-NEXT: movl 208(%rbp), %eax
; AVX2-NEXT: vpinsrb $14, %eax, %xmm7, %xmm5
-; AVX2-NEXT: movzbl 216(%rbp), %eax
+; AVX2-NEXT: movl 216(%rbp), %eax
; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm5
; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm5
; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
@@ -2219,10 +2219,10 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: movzbl (%rsp,%rax), %r13d
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)
+; AVX2-NEXT: andl $1, %r14d
+; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%r14)
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rdi, %rsi
+; AVX2-NEXT: addq %r14, %rsi
; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rsi)
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rsi, %rdx
@@ -2236,415 +2236,359 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: movzbl %r10b, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %r9, %rax
-; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
-; AVX2-NEXT: andl $63, %r9d
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r9)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r11b, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl %bl, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r14b, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl %r15b, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r12b, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %r9d, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %r9, %rdi
+; AVX2-NEXT: andl $1, %r10d
+; AVX2-NEXT: addq %rdi, %r10
+; AVX2-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi
+; AVX2-NEXT: andl $63, %edi
+; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rdi)
+; AVX2-NEXT: movl %r10d, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: andl $1, %r11d
+; AVX2-NEXT: addq %r10, %r11
+; AVX2-NEXT: andl $1, %ebx
+; AVX2-NEXT: addq %r11, %rbx
+; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r11
+; AVX2-NEXT: andl $63, %r11d
+; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%r11)
+; AVX2-NEXT: movl %ebx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: andl $1, %r15d
+; AVX2-NEXT: addq %rbx, %r15
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addq %r15, %r12
+; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r15
+; AVX2-NEXT: andl $63, %r15d
+; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%r15)
+; AVX2-NEXT: movl %r12d, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 64(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 72(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: addq %r12, %rax
+; AVX2-NEXT: movl 72(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 80(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 80(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 88(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: movl 88(%rbp), %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rax, %rdx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: movl 96(%rbp), %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 96(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 104(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 104(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 112(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 112(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 120(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 120(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 128(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 128(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 136(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 136(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 144(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 144(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 152(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 152(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 160(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 160(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 168(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 168(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 176(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 176(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 184(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 184(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 192(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 192(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 200(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 200(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 208(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 208(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 216(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 216(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 224(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $0, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $0, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 224(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $1, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 232(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 232(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 240(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $1, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $2, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $2, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 240(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $3, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 248(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 248(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 256(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $3, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $4, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $4, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 256(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $5, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 264(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 264(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 272(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $5, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $6, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $6, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 272(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $7, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 280(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 280(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 288(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $7, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $8, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $8, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 288(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $9, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 296(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 296(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 304(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $9, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $10, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $10, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 304(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $11, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 312(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 312(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 320(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $11, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $12, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $12, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 320(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $13, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 328(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 328(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 336(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $13, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: vpextrb $14, %xmm1, (%rsp,%rax)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $14, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 336(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $15, %xmm1, (%rsp,%rax)
+; AVX2-NEXT: movl 344(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: movzbl 344(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 352(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vpextrb $15, %xmm1, (%rsp,%rax)
-; AVX2-NEXT: movzbl 352(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
-; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 360(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 368(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 360(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 376(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 368(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 384(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 376(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 392(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 384(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 400(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 392(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 408(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 400(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 416(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 408(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 424(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 416(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 432(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 424(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 440(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 432(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 448(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 440(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 456(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 448(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movzbl 464(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 456(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 472(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 464(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: vpextrb $15, %xmm0, %edx
; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: movl 472(%rbp), %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpextrb $15, %xmm0, %eax
+; AVX2-NEXT: addq %rdx, %rcx
; AVX2-NEXT: cmpq $64, %rcx
-; AVX2-NEXT: cmovbl %r13d, %edx
+; AVX2-NEXT: cmovbl %r13d, %eax
; AVX2-NEXT: cmpq $63, %rcx
-; AVX2-NEXT: movl $63, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: movb %dl, (%rsp,%rax)
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: movl $63, %ecx
+; AVX2-NEXT: cmovbq %rdx, %rcx
+; AVX2-NEXT: movb %al, (%rsp,%rcx)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -40(%rbp), %rsp
@@ -3975,413 +3919,355 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r9,4)
-; AVX2-NEXT: movzbl 16(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 16(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %r9d, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 24(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 24(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 32(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 32(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 40(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 40(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm1, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 48(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rax,4)
+; AVX2-NEXT: movl 48(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 56(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 56(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rax,4)
+; AVX2-NEXT: movl 64(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 72(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 72(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 80(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: movl 80(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 88(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 88(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 96(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 96(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 104(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vmovss %xmm2, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 104(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vmovss %xmm2, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 112(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 112(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm2, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 120(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 120(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm2, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 128(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rax,4)
+; AVX2-NEXT: movl 128(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 136(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm0
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 136(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm0
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 144(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 144(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 152(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 152(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 160(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 160(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 168(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 168(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm3, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 176(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm3, (%rsp,%rax,4)
+; AVX2-NEXT: movl 176(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm3, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 184(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 184(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm3, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 192(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm3, (%rsp,%rax,4)
+; AVX2-NEXT: movl 192(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm3, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 200(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 200(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm3, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 208(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: movl 208(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 216(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 216(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 224(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 224(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 232(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vmovss %xmm4, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 232(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vmovss %xmm4, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 240(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 240(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm4, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 248(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 248(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm4, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 256(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm4, (%rsp,%rax,4)
+; AVX2-NEXT: movl 256(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm4, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 264(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: vextractf128 $1, %ymm4, %xmm0
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 264(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractf128 $1, %ymm4, %xmm0
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 272(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 272(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 280(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 280(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 288(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 288(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 296(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 296(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vmovss %xmm5, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 304(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm5, (%rsp,%rax,4)
+; AVX2-NEXT: movl 304(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm5, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 312(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 312(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm5, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 320(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm5, (%rsp,%rax,4)
+; AVX2-NEXT: movl 320(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm5, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 328(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 328(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm5, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 336(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: movl 336(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 344(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 344(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 352(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 352(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: movzbl 360(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: vmovss %xmm6, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 360(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vmovss %xmm6, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 368(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 368(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm6, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 376(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 376(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm6, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 384(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm6, (%rsp,%rax,4)
+; AVX2-NEXT: movl 384(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm6, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 392(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: vextractf128 $1, %ymm6, %xmm0
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: movl 392(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: vextractf128 $1, %ymm6, %xmm0
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movzbl 400(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movl 400(%rbp), %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 408(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 408(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: # kill: def $eax killed $eax def $rax
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movzbl 416(%rbp), %edx
-; AVX2-NEXT: movzbl %dl, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: movzbl 424(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 416(%rbp), %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: movl 424(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
-; AVX2-NEXT: andl $63, %edx
-; AVX2-NEXT: vmovss %xmm7, (%rsp,%rdx,4)
-; AVX2-NEXT: movzbl 432(%rbp), %edx
-; AVX2-NEXT: movzbl %dl, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm7, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 440(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vmovss %xmm7, (%rsp,%rax,4)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm7, (%rsp,%rax,4)
+; AVX2-NEXT: movl 432(%rbp), %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: movl 440(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
-; AVX2-NEXT: andl $63, %edx
-; AVX2-NEXT: vextractps $2, %xmm7, (%rsp,%rdx,4)
-; AVX2-NEXT: movzbl 448(%rbp), %edx
-; AVX2-NEXT: movzbl %dl, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $3, %xmm7, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 456(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $2, %xmm7, (%rsp,%rax,4)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $3, %xmm7, (%rsp,%rax,4)
+; AVX2-NEXT: movl 448(%rbp), %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: movl 456(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
-; AVX2-NEXT: andl $63, %edx
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vextractf128 $1, %ymm7, %xmm0
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)
-; AVX2-NEXT: movzbl 464(%rbp), %edx
-; AVX2-NEXT: movzbl %dl, %edx
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $63, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movl 464(%rbp), %edx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: movzbl 472(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movl 472(%rbp), %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addl %edx, %ecx
-; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
; AVX2-NEXT: andl $63, %edx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rdx,4)
; AVX2-NEXT: andl $63, %ecx
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
index 33dce3ca920a39..dd9b78e0c73510 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
@@ -31,8 +31,8 @@ define i8 @test_i8(i8 %a) nounwind {
define i16 @test_i16(i16 %a) nounwind {
; X86-NO-CMOV-LABEL: test_i16:
; X86-NO-CMOV: # %bb.0:
-; X86-NO-CMOV-NEXT: movswl {{[0-9]+}}(%esp), %eax
-; X86-NO-CMOV-NEXT: movl %eax, %ecx
+; X86-NO-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-CMOV-NEXT: movswl %ax, %ecx
; X86-NO-CMOV-NEXT: sarl $15, %ecx
; X86-NO-CMOV-NEXT: xorl %ecx, %eax
; X86-NO-CMOV-NEXT: subl %ecx, %eax
>From f26bcf313b8f1a0e44655e9fd633c192d755fe42 Mon Sep 17 00:00:00 2001
From: Vachan V Y <vachanvy05 at gmail.com>
Date: Mon, 28 Sep 2026 14:30:58 +0530
Subject: [PATCH 2/2] [DAGCombiner] visitSIGN_EXTEND_INREG - fold (sext_inreg
(freeze (extload x))) -> (freeze (sextload x))
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 21 +++++++++++--------
llvm/test/CodeGen/X86/iabs.ll | 6 +++---
llvm/test/CodeGen/X86/icmp-abs-C.ll | 6 +++---
.../test/CodeGen/X86/icmp-pow2-logic-npow2.ll | 6 +++---
llvm/test/CodeGen/X86/neg-abs.ll | 12 +++++------
.../Inputs/basic.ll.expected | 6 +++---
6 files changed, 30 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 46c39f2669b4a7..49585b979737ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17381,21 +17381,26 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
}
}
+ bool Frozen = N0.getOpcode() == ISD::FREEZE;
+
// fold (sext_inreg (extload x)) -> (sextload x)
+ // fold (sext_inreg (freeze (extload x))) -> (freeze (sextload x))
// If sextload is not supported by target, we can only do the combine when
// load has one use. Doing otherwise can block folding the extload with other
// extends that the target does support.
- if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
- auto *LN0 = cast<LoadSDNode>(N0);
+ SDValue LoadOp = Frozen ? N0.getOperand(0) : N0;
+ if (ISD::isEXTLoad(LoadOp.getNode()) &&
+ ISD::isUNINDEXEDLoad(LoadOp.getNode())) {
+ auto *LN0 = cast<LoadSDNode>(LoadOp);
if (ExtVT == LN0->getMemoryVT() &&
- ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) ||
+ ((!LegalOperations && LN0->isSimple() && LoadOp.hasOneUse()) ||
TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(),
ISD::SEXTLOAD, false))) {
SDValue ExtLoad =
DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
- CombineTo(N, ExtLoad);
- CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
+ CombineTo(N, Frozen ? N0 : ExtLoad);
+ CombineTo(LN0, ExtLoad, ExtLoad.getValue(1));
AddToWorklist(ExtLoad.getNode());
return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
@@ -17419,9 +17424,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
}
// fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
- // ignore it if the masked load is already sign extended
- bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.hasOneUse();
- if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0)) {
+ if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Frozen && N0.hasOneUse() ? N0.getOperand(0) : N0)) {
if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(1, 0) &&
Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
TLI.isLoadLegal(VT, ExtVT, Ld->getAlign(), Ld->getAddressSpace(),
@@ -17430,7 +17433,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
VT, DL, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad());
- CombineTo(N, Frozen ? N0 : ExtMaskedLoad);
+ CombineTo(N, Frozen && N0.hasOneUse() ? N0 : ExtMaskedLoad);
CombineTo(Ld, ExtMaskedLoad, ExtMaskedLoad.getValue(1));
return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
diff --git a/llvm/test/CodeGen/X86/iabs.ll b/llvm/test/CodeGen/X86/iabs.ll
index f18f49e342962b..dcd0ca141fed58 100644
--- a/llvm/test/CodeGen/X86/iabs.ll
+++ b/llvm/test/CodeGen/X86/iabs.ll
@@ -37,9 +37,9 @@ define i8 @test_i8(i8 %a) nounwind {
define i16 @test_i16(i16 %a) nounwind {
; X86-NO-CMOV-LABEL: test_i16:
; X86-NO-CMOV: # %bb.0:
-; X86-NO-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-CMOV-NEXT: movswl %ax, %ecx
-; X86-NO-CMOV-NEXT: sarl $15, %ecx
+; X86-NO-CMOV-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NO-CMOV-NEXT: movl %eax, %ecx
+; X86-NO-CMOV-NEXT: shrl $15, %ecx
; X86-NO-CMOV-NEXT: xorl %ecx, %eax
; X86-NO-CMOV-NEXT: subl %ecx, %eax
; X86-NO-CMOV-NEXT: # kill: def $ax killed $ax killed $eax
diff --git a/llvm/test/CodeGen/X86/icmp-abs-C.ll b/llvm/test/CodeGen/X86/icmp-abs-C.ll
index 5c88909de9eb64..dcbbc1ef84d8e4 100644
--- a/llvm/test/CodeGen/X86/icmp-abs-C.ll
+++ b/llvm/test/CodeGen/X86/icmp-abs-C.ll
@@ -161,9 +161,9 @@ define i16 @ne_and_with_dom_abs(i16 %x) nounwind {
; X86-LABEL: ne_and_with_dom_abs:
; X86: # %bb.0:
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movswl %ax, %ecx
-; X86-NEXT: sarl $15, %ecx
+; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrl $15, %ecx
; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: movl %eax, %edx
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
index 0a85a395895dc1..f2ada080dba0df 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
@@ -196,9 +196,9 @@ define i1 @abs_ne_pow2(i64 %0) nounwind {
define i1 @abs_ne_nonpow2(i16 %0) nounwind {
; X86-LABEL: abs_ne_nonpow2:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movswl %ax, %ecx
-; X86-NEXT: sarl $15, %ecx
+; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrl $15, %ecx
; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: movzwl %ax, %eax
diff --git a/llvm/test/CodeGen/X86/neg-abs.ll b/llvm/test/CodeGen/X86/neg-abs.ll
index 47c95e1c0943c5..47c1541ade8a65 100644
--- a/llvm/test/CodeGen/X86/neg-abs.ll
+++ b/llvm/test/CodeGen/X86/neg-abs.ll
@@ -34,9 +34,9 @@ define i8 @neg_abs_i8(i8 %x) nounwind {
define i16 @neg_abs_i16(i16 %x) nounwind {
; X86-LABEL: neg_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movswl %cx, %eax
-; X86-NEXT: sarl $15, %eax
+; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
@@ -182,9 +182,9 @@ define i8 @sub_abs_i8(i8 %x, i8 %y) nounwind {
define i16 @sub_abs_i16(i16 %x, i16 %y) nounwind {
; X86-LABEL: sub_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movswl %cx, %eax
-; X86-NEXT: sarl $15, %eax
+; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
index dd9b78e0c73510..fcdea7de729472 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
@@ -31,9 +31,9 @@ define i8 @test_i8(i8 %a) nounwind {
define i16 @test_i16(i16 %a) nounwind {
; X86-NO-CMOV-LABEL: test_i16:
; X86-NO-CMOV: # %bb.0:
-; X86-NO-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-CMOV-NEXT: movswl %ax, %ecx
-; X86-NO-CMOV-NEXT: sarl $15, %ecx
+; X86-NO-CMOV-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NO-CMOV-NEXT: movl %eax, %ecx
+; X86-NO-CMOV-NEXT: shrl $15, %ecx
; X86-NO-CMOV-NEXT: xorl %ecx, %eax
; X86-NO-CMOV-NEXT: subl %ecx, %eax
; X86-NO-CMOV-NEXT: # kill: def $ax killed $ax killed $eax
More information about the llvm-commits
mailing list